drivers/char/xillybus/xillyusb.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright 2020 Xillybus Ltd, http://xillybus.com
   4  *
   5  * Driver for the XillyUSB FPGA/host framework.
   6  *
   7  * This driver interfaces with a special IP core in an FPGA, setting up
   8  * a pipe between a hardware FIFO in the programmable logic and a device
   9  * file in the host. The number of such pipes and their attributes are
  10  * set up on the logic. This driver detects these automatically and
  11  * creates the device files accordingly.
  12  */
  13
  14 #include <linux/types.h>
  15 #include <linux/slab.h>
  16 #include <linux/list.h>
  17 #include <linux/device.h>
  18 #include <linux/module.h>
  19 #include <asm/byteorder.h>
  20 #include <linux/io.h>
  21 #include <linux/interrupt.h>
  22 #include <linux/sched.h>
  23 #include <linux/fs.h>
  24 #include <linux/spinlock.h>
  25 #include <linux/mutex.h>
  26 #include <linux/workqueue.h>
  27 #include <linux/crc32.h>
  28 #include <linux/poll.h>
  29 #include <linux/delay.h>
  30 #include <linux/usb.h>
  31
  32 #include "xillybus_class.h"
  33
  34 MODULE_DESCRIPTION("Driver for XillyUSB FPGA IP Core");
  35 MODULE_AUTHOR("Eli Billauer, Xillybus Ltd.");
  36 MODULE_ALIAS("xillyusb");
  37 MODULE_LICENSE("GPL v2");
  38
  39 #define XILLY_RX_TIMEOUT                (10 * HZ / 1000)
  40 #define XILLY_RESPONSE_TIMEOUT          (500 * HZ / 1000)
  41
  42 #define BUF_SIZE_ORDER                  4
  43 #define BUFNUM                          8
  44 #define LOG2_IDT_FIFO_SIZE              16
  45 #define LOG2_INITIAL_FIFO_BUF_SIZE      16
  46
  47 #define MSG_EP_NUM                      1
  48 #define IN_EP_NUM                       1
  49
  50 static const char xillyname[] = "xillyusb";
  51
  52 static unsigned int fifo_buf_order;
  53 static struct workqueue_struct *wakeup_wq;
  54
  55 #define USB_VENDOR_ID_XILINX            0x03fd
  56 #define USB_VENDOR_ID_ALTERA            0x09fb
  57
  58 #define USB_PRODUCT_ID_XILLYUSB         0xebbe
  59
  60 static const struct usb_device_id xillyusb_table[] = {
  61         { USB_DEVICE(USB_VENDOR_ID_XILINX, USB_PRODUCT_ID_XILLYUSB) },
  62         { USB_DEVICE(USB_VENDOR_ID_ALTERA, USB_PRODUCT_ID_XILLYUSB) },
  63         { }
  64 };
  65
  66 MODULE_DEVICE_TABLE(usb, xillyusb_table);
  67
  68 struct xillyusb_dev;
  69
  70 struct xillyfifo {
  71         unsigned int bufsize; /* In bytes, always a power of 2 */
  72         unsigned int bufnum;
  73         unsigned int size; /* Lazy: Equals bufsize * bufnum */
  74         unsigned int buf_order;
  75
  76         int fill; /* Number of bytes in the FIFO */
  77         spinlock_t lock;
  78         wait_queue_head_t waitq;
  79
  80         unsigned int readpos;
  81         unsigned int readbuf;
  82         unsigned int writepos;
  83         unsigned int writebuf;
  84         char **mem;
  85 };
  86
  87 struct xillyusb_channel;
  88
  89 struct xillyusb_endpoint {
  90         struct xillyusb_dev *xdev;
  91
  92         struct mutex ep_mutex; /* serialize operations on endpoint */
  93
  94         struct list_head buffers;
  95         struct list_head filled_buffers;
  96         spinlock_t buffers_lock; /* protect these two lists */
  97
  98         unsigned int order;
  99         unsigned int buffer_size;
 100
 101         unsigned int fill_mask;
 102
 103         int outstanding_urbs;
 104
 105         struct usb_anchor anchor;
 106
 107         struct xillyfifo fifo;
 108
 109         struct work_struct workitem;
 110
 111         bool shutting_down;
 112         bool drained;
 113         bool wake_on_drain;
 114
 115         u8 ep_num;
 116 };
 117
 118 struct xillyusb_channel {
 119         struct xillyusb_dev *xdev;
 120
 121         struct xillyfifo *in_fifo;
 122         struct xillyusb_endpoint *out_ep;
 123         struct mutex lock; /* protect @out_ep, @in_fifo, bit fields below */
 124
 125         struct mutex in_mutex; /* serialize fops on FPGA to host stream */
 126         struct mutex out_mutex; /* serialize fops on host to FPGA stream */
 127         wait_queue_head_t flushq;
 128
 129         int chan_idx;
 130
 131         u32 in_consumed_bytes;
 132         u32 in_current_checkpoint;
 133         u32 out_bytes;
 134
 135         unsigned int in_log2_element_size;
 136         unsigned int out_log2_element_size;
 137         unsigned int in_log2_fifo_size;
 138         unsigned int out_log2_fifo_size;
 139
 140         unsigned int read_data_ok; /* EOF not arrived (yet) */
 141         unsigned int poll_used;
 142         unsigned int flushing;
 143         unsigned int flushed;
 144         unsigned int canceled;
 145
 146         /* Bit fields protected by @lock except for initialization */
 147         unsigned readable:1;
 148         unsigned writable:1;
 149         unsigned open_for_read:1;
 150         unsigned open_for_write:1;
 151         unsigned in_synchronous:1;
 152         unsigned out_synchronous:1;
 153         unsigned in_seekable:1;
 154         unsigned out_seekable:1;
 155 };
 156
 157 struct xillybuffer {
 158         struct list_head entry;
 159         struct xillyusb_endpoint *ep;
 160         void *buf;
 161         unsigned int len;
 162 };
 163
 164 struct xillyusb_dev {
 165         struct xillyusb_channel *channels;
 166
 167         struct usb_device       *udev;
 168         struct device           *dev; /* For dev_err() and such */
 169         struct kref             kref;
 170         struct workqueue_struct *workq;
 171
 172         int error;
 173         spinlock_t error_lock; /* protect @error */
 174         struct work_struct wakeup_workitem;
 175
 176         int num_channels;
 177
 178         struct xillyusb_endpoint *msg_ep;
 179         struct xillyusb_endpoint *in_ep;
 180
 181         struct mutex msg_mutex; /* serialize opcode transmission */
 182         int in_bytes_left;
 183         int leftover_chan_num;
 184         unsigned int in_counter;
 185         struct mutex process_in_mutex; /* synchronize wakeup_all() */
 186 };
 187
 188 /*
 189  * kref_mutex is used in xillyusb_open() to prevent the xillyusb_dev
 190  * struct from being freed during the gap between being found by
 191  * xillybus_find_inode() and having its reference count incremented.
 192  */
 193
 194 static DEFINE_MUTEX(kref_mutex);
 195
 196 /* FPGA to host opcodes */
 197 enum {
 198         OPCODE_DATA = 0,
 199         OPCODE_QUIESCE_ACK = 1,
 200         OPCODE_EOF = 2,
 201         OPCODE_REACHED_CHECKPOINT = 3,
 202         OPCODE_CANCELED_CHECKPOINT = 4,
 203 };
 204
 205 /* Host to FPGA opcodes */
 206 enum {
 207         OPCODE_QUIESCE = 0,
 208         OPCODE_REQ_IDT = 1,
 209         OPCODE_SET_CHECKPOINT = 2,
 210         OPCODE_CLOSE = 3,
 211         OPCODE_SET_PUSH = 4,
 212         OPCODE_UPDATE_PUSH = 5,
 213         OPCODE_CANCEL_CHECKPOINT = 6,
 214         OPCODE_SET_ADDR = 7,
 215 };
 216
 217 /*
 218  * fifo_write() and fifo_read() are NOT reentrant (i.e. concurrent multiple
 219  * calls to each on the same FIFO is not allowed) however it's OK to have
 220  * threads calling each of the two functions once on the same FIFO, and
 221  * at the same time.
 222  */
 223
 224 static int fifo_write(struct xillyfifo *fifo,
 225                       const void *data, unsigned int len,
 226                       int (*copier)(void *, const void *, int))
 227 {
 228         unsigned int done = 0;
 229         unsigned int todo = len;
 230         unsigned int nmax;
 231         unsigned int writepos = fifo->writepos;
 232         unsigned int writebuf = fifo->writebuf;
 233         unsigned long flags;
 234         int rc;
 235
 236         nmax = fifo->size - READ_ONCE(fifo->fill);
 237
 238         while (1) {
 239                 unsigned int nrail = fifo->bufsize - writepos;
 240                 unsigned int n = min(todo, nmax);
 241
 242                 if (n == 0) {
 243                         spin_lock_irqsave(&fifo->lock, flags);
 244                         fifo->fill += done;
 245                         spin_unlock_irqrestore(&fifo->lock, flags);
 246
 247                         fifo->writepos = writepos;
 248                         fifo->writebuf = writebuf;
 249
 250                         return done;
 251                 }
 252
 253                 if (n > nrail)
 254                         n = nrail;
 255
 256                 rc = (*copier)(fifo->mem[writebuf] + writepos, data + done, n);
 257
 258                 if (rc)
 259                         return rc;
 260
 261                 done += n;
 262                 todo -= n;
 263
 264                 writepos += n;
 265                 nmax -= n;
 266
 267                 if (writepos == fifo->bufsize) {
 268                         writepos = 0;
 269                         writebuf++;
 270
 271                         if (writebuf == fifo->bufnum)
 272                                 writebuf = 0;
 273                 }
 274         }
 275 }
 276
 277 static int fifo_read(struct xillyfifo *fifo,
 278                      void *data, unsigned int len,
 279                      int (*copier)(void *, const void *, int))
 280 {
 281         unsigned int done = 0;
 282         unsigned int todo = len;
 283         unsigned int fill;
 284         unsigned int readpos = fifo->readpos;
 285         unsigned int readbuf = fifo->readbuf;
 286         unsigned long flags;
 287         int rc;
 288
 289         /*
 290          * The spinlock here is necessary, because otherwise fifo->fill
 291          * could have been increased by fifo_write() after writing data
 292          * to the buffer, but this data would potentially not have been
 293          * visible on this thread at the time the updated fifo->fill was.
 294          * That could lead to reading invalid data.
 295          */
 296
 297         spin_lock_irqsave(&fifo->lock, flags);
 298         fill = fifo->fill;
 299         spin_unlock_irqrestore(&fifo->lock, flags);
 300
 301         while (1) {
 302                 unsigned int nrail = fifo->bufsize - readpos;
 303                 unsigned int n = min(todo, fill);
 304
 305                 if (n == 0) {
 306                         spin_lock_irqsave(&fifo->lock, flags);
 307                         fifo->fill -= done;
 308                         spin_unlock_irqrestore(&fifo->lock, flags);
 309
 310                         fifo->readpos = readpos;
 311                         fifo->readbuf = readbuf;
 312
 313                         return done;
 314                 }
 315
 316                 if (n > nrail)
 317                         n = nrail;
 318
 319                 rc = (*copier)(data + done, fifo->mem[readbuf] + readpos, n);
 320
 321                 if (rc)
 322                         return rc;
 323
 324                 done += n;
 325                 todo -= n;
 326
 327                 readpos += n;
 328                 fill -= n;
 329
 330                 if (readpos == fifo->bufsize) {
 331                         readpos = 0;
 332                         readbuf++;
 333
 334                         if (readbuf == fifo->bufnum)
 335                                 readbuf = 0;
 336                 }
 337         }
 338 }
 339
 340 /*
 341  * These three wrapper functions are used as the @copier argument to
 342  * fifo_write() and fifo_read(), so that they can work directly with
 343  * user memory as well.
 344  */
 345
 346 static int xilly_copy_from_user(void *dst, const void *src, int n)
 347 {
 348         if (copy_from_user(dst, (const void __user *)src, n))
 349                 return -EFAULT;
 350
 351         return 0;
 352 }
 353
 354 static int xilly_copy_to_user(void *dst, const void *src, int n)
 355 {
 356         if (copy_to_user((void __user *)dst, src, n))
 357                 return -EFAULT;
 358
 359         return 0;
 360 }
 361
 362 static int xilly_memcpy(void *dst, const void *src, int n)
 363 {
 364         memcpy(dst, src, n);
 365
 366         return 0;
 367 }
 368
 369 static int fifo_init(struct xillyfifo *fifo,
 370                      unsigned int log2_size)
 371 {
 372         unsigned int log2_bufnum;
 373         unsigned int buf_order;
 374         int i;
 375
 376         unsigned int log2_fifo_buf_size;
 377
 378 retry:
 379         log2_fifo_buf_size = fifo_buf_order + PAGE_SHIFT;
 380
 381         if (log2_size > log2_fifo_buf_size) {
 382                 log2_bufnum = log2_size - log2_fifo_buf_size;
 383                 buf_order = fifo_buf_order;
 384                 fifo->bufsize = 1 << log2_fifo_buf_size;
 385         } else {
 386                 log2_bufnum = 0;
 387                 buf_order = (log2_size > PAGE_SHIFT) ?
 388                         log2_size - PAGE_SHIFT : 0;
 389                 fifo->bufsize = 1 << log2_size;
 390         }
 391
 392         fifo->bufnum = 1 << log2_bufnum;
 393         fifo->size = fifo->bufnum * fifo->bufsize;
 394         fifo->buf_order = buf_order;
 395
 396         fifo->mem = kmalloc_array(fifo->bufnum, sizeof(void *), GFP_KERNEL);
 397
 398         if (!fifo->mem)
 399                 return -ENOMEM;
 400
 401         for (i = 0; i < fifo->bufnum; i++) {
 402                 fifo->mem[i] = (void *)
 403                         __get_free_pages(GFP_KERNEL, buf_order);
 404
 405                 if (!fifo->mem[i])
 406                         goto memfail;
 407         }
 408
 409         fifo->fill = 0;
 410         fifo->readpos = 0;
 411         fifo->readbuf = 0;
 412         fifo->writepos = 0;
 413         fifo->writebuf = 0;
 414         spin_lock_init(&fifo->lock);
 415         init_waitqueue_head(&fifo->waitq);
 416         return 0;
 417
 418 memfail:
 419         for (i--; i >= 0; i--)
 420                 free_pages((unsigned long)fifo->mem[i], buf_order);
 421
 422         kfree(fifo->mem);
 423         fifo->mem = NULL;
 424
 425         if (fifo_buf_order) {
 426                 fifo_buf_order--;
 427                 goto retry;
 428         } else {
 429                 return -ENOMEM;
 430         }
 431 }
 432
 433 static void fifo_mem_release(struct xillyfifo *fifo)
 434 {
 435         int i;
 436
 437         if (!fifo->mem)
 438                 return;
 439
 440         for (i = 0; i < fifo->bufnum; i++)
 441                 free_pages((unsigned long)fifo->mem[i], fifo->buf_order);
 442
 443         kfree(fifo->mem);
 444 }
 445
 446 /*
 447  * When endpoint_quiesce() returns, the endpoint has no URBs submitted,
 448  * won't accept any new URB submissions, and its related work item doesn't
 449  * and won't run anymore.
 450  */
 451
 452 static void endpoint_quiesce(struct xillyusb_endpoint *ep)
 453 {
 454         mutex_lock(&ep->ep_mutex);
 455         ep->shutting_down = true;
 456         mutex_unlock(&ep->ep_mutex);
 457
 458         usb_kill_anchored_urbs(&ep->anchor);
 459         cancel_work_sync(&ep->workitem);
 460 }
 461
 462 /*
 463  * Note that endpoint_dealloc() also frees fifo memory (if allocated), even
 464  * though endpoint_alloc doesn't allocate that memory.
 465  */
 466
 467 static void endpoint_dealloc(struct xillyusb_endpoint *ep)
 468 {
 469         struct list_head *this, *next;
 470
 471         fifo_mem_release(&ep->fifo);
 472
 473         /* Join @filled_buffers with @buffers to free these entries too */
 474         list_splice(&ep->filled_buffers, &ep->buffers);
 475
 476         list_for_each_safe(this, next, &ep->buffers) {
 477                 struct xillybuffer *xb =
 478                         list_entry(this, struct xillybuffer, entry);
 479
 480                 free_pages((unsigned long)xb->buf, ep->order);
 481                 kfree(xb);
 482         }
 483
 484         kfree(ep);
 485 }
 486
 487 static struct xillyusb_endpoint
 488 *endpoint_alloc(struct xillyusb_dev *xdev,
 489                 u8 ep_num,
 490                 void (*work)(struct work_struct *),
 491                 unsigned int order,
 492                 int bufnum)
 493 {
 494         int i;
 495
 496         struct xillyusb_endpoint *ep;
 497
 498         ep = kzalloc(sizeof(*ep), GFP_KERNEL);
 499
 500         if (!ep)
 501                 return NULL;
 502
 503         INIT_LIST_HEAD(&ep->buffers);
 504         INIT_LIST_HEAD(&ep->filled_buffers);
 505
 506         spin_lock_init(&ep->buffers_lock);
 507         mutex_init(&ep->ep_mutex);
 508
 509         init_usb_anchor(&ep->anchor);
 510         INIT_WORK(&ep->workitem, work);
 511
 512         ep->order = order;
 513         ep->buffer_size =  1 << (PAGE_SHIFT + order);
 514         ep->outstanding_urbs = 0;
 515         ep->drained = true;
 516         ep->wake_on_drain = false;
 517         ep->xdev = xdev;
 518         ep->ep_num = ep_num;
 519         ep->shutting_down = false;
 520
 521         for (i = 0; i < bufnum; i++) {
 522                 struct xillybuffer *xb;
 523                 unsigned long addr;
 524
 525                 xb = kzalloc(sizeof(*xb), GFP_KERNEL);
 526
 527                 if (!xb) {
 528                         endpoint_dealloc(ep);
 529                         return NULL;
 530                 }
 531
 532                 addr = __get_free_pages(GFP_KERNEL, order);
 533
 534                 if (!addr) {
 535                         kfree(xb);
 536                         endpoint_dealloc(ep);
 537                         return NULL;
 538                 }
 539
 540                 xb->buf = (void *)addr;
 541                 xb->ep = ep;
 542                 list_add_tail(&xb->entry, &ep->buffers);
 543         }
 544         return ep;
 545 }
 546
 547 static void cleanup_dev(struct kref *kref)
 548 {
 549         struct xillyusb_dev *xdev =
 550                 container_of(kref, struct xillyusb_dev, kref);
 551
 552         if (xdev->in_ep)
 553                 endpoint_dealloc(xdev->in_ep);
 554
 555         if (xdev->msg_ep)
 556                 endpoint_dealloc(xdev->msg_ep);
 557
 558         if (xdev->workq)
 559                 destroy_workqueue(xdev->workq);
 560
 561         usb_put_dev(xdev->udev);
 562         kfree(xdev->channels); /* Argument may be NULL, and that's fine */
 563         kfree(xdev);
 564 }
 565
 566 /*
 567  * @process_in_mutex is taken to ensure that bulk_in_work() won't call
 568  * process_bulk_in() after wakeup_all()'s execution: The latter zeroes all
 569  * @read_data_ok entries, which will make process_bulk_in() report false
 570  * errors if executed. The mechanism relies on that xdev->error is assigned
 571  * a non-zero value by report_io_error() prior to queueing wakeup_all(),
 572  * which prevents bulk_in_work() from calling process_bulk_in().
 573  */
 574
 575 static void wakeup_all(struct work_struct *work)
 576 {
 577         int i;
 578         struct xillyusb_dev *xdev = container_of(work, struct xillyusb_dev,
 579                                                  wakeup_workitem);
 580
 581         mutex_lock(&xdev->process_in_mutex);
 582
 583         for (i = 0; i < xdev->num_channels; i++) {
 584                 struct xillyusb_channel *chan = &xdev->channels[i];
 585
 586                 mutex_lock(&chan->lock);
 587
 588                 if (chan->in_fifo) {
 589                         /*
 590                          * Fake an EOF: Even if such arrives, it won't be
 591                          * processed.
 592                          */
 593                         chan->read_data_ok = 0;
 594                         wake_up_interruptible(&chan->in_fifo->waitq);
 595                 }
 596
 597                 if (chan->out_ep)
 598                         wake_up_interruptible(&chan->out_ep->fifo.waitq);
 599
 600                 mutex_unlock(&chan->lock);
 601
 602                 wake_up_interruptible(&chan->flushq);
 603         }
 604
 605         mutex_unlock(&xdev->process_in_mutex);
 606
 607         wake_up_interruptible(&xdev->msg_ep->fifo.waitq);
 608
 609         kref_put(&xdev->kref, cleanup_dev);
 610 }
 611
 612 static void report_io_error(struct xillyusb_dev *xdev,
 613                             int errcode)
 614 {
 615         unsigned long flags;
 616         bool do_once = false;
 617
 618         spin_lock_irqsave(&xdev->error_lock, flags);
 619         if (!xdev->error) {
 620                 xdev->error = errcode;
 621                 do_once = true;
 622         }
 623         spin_unlock_irqrestore(&xdev->error_lock, flags);
 624
 625         if (do_once) {
 626                 kref_get(&xdev->kref); /* xdev is used by work item */
 627                 queue_work(wakeup_wq, &xdev->wakeup_workitem);
 628         }
 629 }
 630
 631 /*
 632  * safely_assign_in_fifo() changes the value of chan->in_fifo and ensures
 633  * the previous pointer is never used after its return.
 634  */
 635
 636 static void safely_assign_in_fifo(struct xillyusb_channel *chan,
 637                                   struct xillyfifo *fifo)
 638 {
 639         mutex_lock(&chan->lock);
 640         chan->in_fifo = fifo;
 641         mutex_unlock(&chan->lock);
 642
 643         flush_work(&chan->xdev->in_ep->workitem);
 644 }
 645
 646 static void bulk_in_completer(struct urb *urb)
 647 {
 648         struct xillybuffer *xb = urb->context;
 649         struct xillyusb_endpoint *ep = xb->ep;
 650         unsigned long flags;
 651
 652         if (urb->status) {
 653                 if (!(urb->status == -ENOENT ||
 654                       urb->status == -ECONNRESET ||
 655                       urb->status == -ESHUTDOWN))
 656                         report_io_error(ep->xdev, -EIO);
 657
 658                 spin_lock_irqsave(&ep->buffers_lock, flags);
 659                 list_add_tail(&xb->entry, &ep->buffers);
 660                 ep->outstanding_urbs--;
 661                 spin_unlock_irqrestore(&ep->buffers_lock, flags);
 662
 663                 return;
 664         }
 665
 666         xb->len = urb->actual_length;
 667
 668         spin_lock_irqsave(&ep->buffers_lock, flags);
 669         list_add_tail(&xb->entry, &ep->filled_buffers);
 670         spin_unlock_irqrestore(&ep->buffers_lock, flags);
 671
 672         if (!ep->shutting_down)
 673                 queue_work(ep->xdev->workq, &ep->workitem);
 674 }
 675
 676 static void bulk_out_completer(struct urb *urb)
 677 {
 678         struct xillybuffer *xb = urb->context;
 679         struct xillyusb_endpoint *ep = xb->ep;
 680         unsigned long flags;
 681
 682         if (urb->status &&
 683             (!(urb->status == -ENOENT ||
 684                urb->status == -ECONNRESET ||
 685                urb->status == -ESHUTDOWN)))
 686                 report_io_error(ep->xdev, -EIO);
 687
 688         spin_lock_irqsave(&ep->buffers_lock, flags);
 689         list_add_tail(&xb->entry, &ep->buffers);
 690         ep->outstanding_urbs--;
 691         spin_unlock_irqrestore(&ep->buffers_lock, flags);
 692
 693         if (!ep->shutting_down)
 694                 queue_work(ep->xdev->workq, &ep->workitem);
 695 }
 696
 697 static void try_queue_bulk_in(struct xillyusb_endpoint *ep)
 698 {
 699         struct xillyusb_dev *xdev = ep->xdev;
 700         struct xillybuffer *xb;
 701         struct urb *urb;
 702
 703         int rc;
 704         unsigned long flags;
 705         unsigned int bufsize = ep->buffer_size;
 706
 707         mutex_lock(&ep->ep_mutex);
 708
 709         if (ep->shutting_down || xdev->error)
 710                 goto done;
 711
 712         while (1) {
 713                 spin_lock_irqsave(&ep->buffers_lock, flags);
 714
 715                 if (list_empty(&ep->buffers)) {
 716                         spin_unlock_irqrestore(&ep->buffers_lock, flags);
 717                         goto done;
 718                 }
 719
 720                 xb = list_first_entry(&ep->buffers, struct xillybuffer, entry);
 721                 list_del(&xb->entry);
 722                 ep->outstanding_urbs++;
 723
 724                 spin_unlock_irqrestore(&ep->buffers_lock, flags);
 725
 726                 urb = usb_alloc_urb(0, GFP_KERNEL);
 727                 if (!urb) {
 728                         report_io_error(xdev, -ENOMEM);
 729                         goto relist;
 730                 }
 731
 732                 usb_fill_bulk_urb(urb, xdev->udev,
 733                                   usb_rcvbulkpipe(xdev->udev, ep->ep_num),
 734                                   xb->buf, bufsize, bulk_in_completer, xb);
 735
 736                 usb_anchor_urb(urb, &ep->anchor);
 737
 738                 rc = usb_submit_urb(urb, GFP_KERNEL);
 739
 740                 if (rc) {
 741                         report_io_error(xdev, (rc == -ENOMEM) ? -ENOMEM :
 742                                         -EIO);
 743                         goto unanchor;
 744                 }
 745
 746                 usb_free_urb(urb); /* This just decrements reference count */
 747         }
 748
 749 unanchor:
 750         usb_unanchor_urb(urb);
 751         usb_free_urb(urb);
 752
 753 relist:
 754         spin_lock_irqsave(&ep->buffers_lock, flags);
 755         list_add_tail(&xb->entry, &ep->buffers);
 756         ep->outstanding_urbs--;
 757         spin_unlock_irqrestore(&ep->buffers_lock, flags);
 758
 759 done:
 760         mutex_unlock(&ep->ep_mutex);
 761 }
 762
 763 static void try_queue_bulk_out(struct xillyusb_endpoint *ep)
 764 {
 765         struct xillyfifo *fifo = &ep->fifo;
 766         struct xillyusb_dev *xdev = ep->xdev;
 767         struct xillybuffer *xb;
 768         struct urb *urb;
 769
 770         int rc;
 771         unsigned int fill;
 772         unsigned long flags;
 773         bool do_wake = false;
 774
 775         mutex_lock(&ep->ep_mutex);
 776
 777         if (ep->shutting_down || xdev->error)
 778                 goto done;
 779
 780         fill = READ_ONCE(fifo->fill) & ep->fill_mask;
 781
 782         while (1) {
 783                 int count;
 784                 unsigned int max_read;
 785
 786                 spin_lock_irqsave(&ep->buffers_lock, flags);
 787
 788                 /*
 789                  * Race conditions might have the FIFO filled while the
 790                  * endpoint is marked as drained here. That doesn't matter,
 791                  * because the sole purpose of @drained is to ensure that
 792                  * certain data has been sent on the USB channel before
 793                  * shutting it down. Hence knowing that the FIFO appears
 794                  * to be empty with no outstanding URBs at some moment
 795                  * is good enough.
 796                  */
 797
 798                 if (!fill) {
 799                         ep->drained = !ep->outstanding_urbs;
 800                         if (ep->drained && ep->wake_on_drain)
 801                                 do_wake = true;
 802
 803                         spin_unlock_irqrestore(&ep->buffers_lock, flags);
 804                         goto done;
 805                 }
 806
 807                 ep->drained = false;
 808
 809                 if ((fill < ep->buffer_size && ep->outstanding_urbs) ||
 810                     list_empty(&ep->buffers)) {
 811                         spin_unlock_irqrestore(&ep->buffers_lock, flags);
 812                         goto done;
 813                 }
 814
 815                 xb = list_first_entry(&ep->buffers, struct xillybuffer, entry);
 816                 list_del(&xb->entry);
 817                 ep->outstanding_urbs++;
 818
 819                 spin_unlock_irqrestore(&ep->buffers_lock, flags);
 820
 821                 max_read = min(fill, ep->buffer_size);
 822
 823                 count = fifo_read(&ep->fifo, xb->buf, max_read, xilly_memcpy);
 824
 825                 /*
 826                  * xilly_memcpy always returns 0 => fifo_read can't fail =>
 827                  * count > 0
 828                  */
 829
 830                 urb = usb_alloc_urb(0, GFP_KERNEL);
 831                 if (!urb) {
 832                         report_io_error(xdev, -ENOMEM);
 833                         goto relist;
 834                 }
 835
 836                 usb_fill_bulk_urb(urb, xdev->udev,
 837                                   usb_sndbulkpipe(xdev->udev, ep->ep_num),
 838                                   xb->buf, count, bulk_out_completer, xb);
 839
 840                 usb_anchor_urb(urb, &ep->anchor);
 841
 842                 rc = usb_submit_urb(urb, GFP_KERNEL);
 843
 844                 if (rc) {
 845                         report_io_error(xdev, (rc == -ENOMEM) ? -ENOMEM :
 846                                         -EIO);
 847                         goto unanchor;
 848                 }
 849
 850                 usb_free_urb(urb); /* This just decrements reference count */
 851
 852                 fill -= count;
 853                 do_wake = true;
 854         }
 855
 856 unanchor:
 857         usb_unanchor_urb(urb);
 858         usb_free_urb(urb);
 859
 860 relist:
 861         spin_lock_irqsave(&ep->buffers_lock, flags);
 862         list_add_tail(&xb->entry, &ep->buffers);
 863         ep->outstanding_urbs--;
 864         spin_unlock_irqrestore(&ep->buffers_lock, flags);
 865
 866 done:
 867         mutex_unlock(&ep->ep_mutex);
 868
 869         if (do_wake)
 870                 wake_up_interruptible(&fifo->waitq);
 871 }
 872
 873 static void bulk_out_work(struct work_struct *work)
 874 {
 875         struct xillyusb_endpoint *ep = container_of(work,
 876                                                     struct xillyusb_endpoint,
 877                                                     workitem);
 878         try_queue_bulk_out(ep);
 879 }
 880
 881 static int process_in_opcode(struct xillyusb_dev *xdev,
 882                              int opcode,
 883                              int chan_num)
 884 {
 885         struct xillyusb_channel *chan;
 886         struct device *dev = xdev->dev;
 887         int chan_idx = chan_num >> 1;
 888
 889         if (chan_idx >= xdev->num_channels) {
 890                 dev_err(dev, "Received illegal channel ID %d from FPGA\n",
 891                         chan_num);
 892                 return -EIO;
 893         }
 894
 895         chan = &xdev->channels[chan_idx];
 896
 897         switch (opcode) {
 898         case OPCODE_EOF:
 899                 if (!chan->read_data_ok) {
 900                         dev_err(dev, "Received unexpected EOF for channel %d\n",
 901                                 chan_num);
 902                         return -EIO;
 903                 }
 904
 905                 /*
 906                  * A write memory barrier ensures that the FIFO's fill level
 907                  * is visible before read_data_ok turns zero, so the data in
 908                  * the FIFO isn't missed by the consumer.
 909                  */
 910                 smp_wmb();
 911                 WRITE_ONCE(chan->read_data_ok, 0);
 912                 wake_up_interruptible(&chan->in_fifo->waitq);
 913                 break;
 914
 915         case OPCODE_REACHED_CHECKPOINT:
 916                 chan->flushing = 0;
 917                 wake_up_interruptible(&chan->flushq);
 918                 break;
 919
 920         case OPCODE_CANCELED_CHECKPOINT:
 921                 chan->canceled = 1;
 922                 wake_up_interruptible(&chan->flushq);
 923                 break;
 924
 925         default:
 926                 dev_err(dev, "Received illegal opcode %d from FPGA\n",
 927                         opcode);
 928                 return -EIO;
 929         }
 930
 931         return 0;
 932 }
 933
 934 static int process_bulk_in(struct xillybuffer *xb)
 935 {
 936         struct xillyusb_endpoint *ep = xb->ep;
 937         struct xillyusb_dev *xdev = ep->xdev;
 938         struct device *dev = xdev->dev;
 939         int dws = xb->len >> 2;
 940         __le32 *p = xb->buf;
 941         u32 ctrlword;
 942         struct xillyusb_channel *chan;
 943         struct xillyfifo *fifo;
 944         int chan_num = 0, opcode;
 945         int chan_idx;
 946         int bytes, count, dwconsume;
 947         int in_bytes_left = 0;
 948         int rc;
 949
 950         if ((dws << 2) != xb->len) {
 951                 dev_err(dev, "Received BULK IN transfer with %d bytes, not a multiple of 4\n",
 952                         xb->len);
 953                 return -EIO;
 954         }
 955
 956         if (xdev->in_bytes_left) {
 957                 bytes = min(xdev->in_bytes_left, dws << 2);
 958                 in_bytes_left = xdev->in_bytes_left - bytes;
 959                 chan_num = xdev->leftover_chan_num;
 960                 goto resume_leftovers;
 961         }
 962
 963         while (dws) {
 964                 ctrlword = le32_to_cpu(*p++);
 965                 dws--;
 966
 967                 chan_num = ctrlword & 0xfff;
 968                 count = (ctrlword >> 12) & 0x3ff;
 969                 opcode = (ctrlword >> 24) & 0xf;
 970
 971                 if (opcode != OPCODE_DATA) {
 972                         unsigned int in_counter = xdev->in_counter++ & 0x3ff;
 973
 974                         if (count != in_counter) {
 975                                 dev_err(dev, "Expected opcode counter %d, got %d\n",
 976                                         in_counter, count);
 977                                 return -EIO;
 978                         }
 979
 980                         rc = process_in_opcode(xdev, opcode, chan_num);
 981
 982                         if (rc)
 983                                 return rc;
 984
 985                         continue;
 986                 }
 987
 988                 bytes = min(count + 1, dws << 2);
 989                 in_bytes_left = count + 1 - bytes;
 990
 991 resume_leftovers:
 992                 chan_idx = chan_num >> 1;
 993
 994                 if (!(chan_num & 1) || chan_idx >= xdev->num_channels ||
 995                     !xdev->channels[chan_idx].read_data_ok) {
 996                         dev_err(dev, "Received illegal channel ID %d from FPGA\n",
 997                                 chan_num);
 998                         return -EIO;
 999                 }
1000                 chan = &xdev->channels[chan_idx];
1001
1002                 fifo = chan->in_fifo;
1003
1004                 if (unlikely(!fifo))
1005                         return -EIO; /* We got really unexpected data */
1006
1007                 if (bytes != fifo_write(fifo, p, bytes, xilly_memcpy)) {
1008                         dev_err(dev, "Misbehaving FPGA overflowed an upstream FIFO!\n");
1009                         return -EIO;
1010                 }
1011
1012                 wake_up_interruptible(&fifo->waitq);
1013
1014                 dwconsume = (bytes + 3) >> 2;
1015                 dws -= dwconsume;
1016                 p += dwconsume;
1017         }
1018
1019         xdev->in_bytes_left = in_bytes_left;
1020         xdev->leftover_chan_num = chan_num;
1021         return 0;
1022 }
1023
1024 static void bulk_in_work(struct work_struct *work)
1025 {
1026         struct xillyusb_endpoint *ep =
1027                 container_of(work, struct xillyusb_endpoint, workitem);
1028         struct xillyusb_dev *xdev = ep->xdev;
1029         unsigned long flags;
1030         struct xillybuffer *xb;
1031         bool consumed = false;
1032         int rc = 0;
1033
1034         mutex_lock(&xdev->process_in_mutex);
1035
1036         spin_lock_irqsave(&ep->buffers_lock, flags);
1037
1038         while (1) {
1039                 if (rc || list_empty(&ep->filled_buffers)) {
1040                         spin_unlock_irqrestore(&ep->buffers_lock, flags);
1041                         mutex_unlock(&xdev->process_in_mutex);
1042
1043                         if (rc)
1044                                 report_io_error(xdev, rc);
1045                         else if (consumed)
1046                                 try_queue_bulk_in(ep);
1047
1048                         return;
1049                 }
1050
1051                 xb = list_first_entry(&ep->filled_buffers, struct xillybuffer,
1052                                       entry);
1053                 list_del(&xb->entry);
1054
1055                 spin_unlock_irqrestore(&ep->buffers_lock, flags);
1056
1057                 consumed = true;
1058
1059                 if (!xdev->error)
1060                         rc = process_bulk_in(xb);
1061
1062                 spin_lock_irqsave(&ep->buffers_lock, flags);
1063                 list_add_tail(&xb->entry, &ep->buffers);
1064                 ep->outstanding_urbs--;
1065         }
1066 }
1067
1068 static int xillyusb_send_opcode(struct xillyusb_dev *xdev,
1069                                 int chan_num, char opcode, u32 data)
1070 {
1071         struct xillyusb_endpoint *ep = xdev->msg_ep;
1072         struct xillyfifo *fifo = &ep->fifo;
1073         __le32 msg[2];
1074
1075         int rc = 0;
1076
1077         msg[0] = cpu_to_le32((chan_num & 0xfff) |
1078                              ((opcode & 0xf) << 24));
1079         msg[1] = cpu_to_le32(data);
1080
1081         mutex_lock(&xdev->msg_mutex);
1082
1083         /*
1084          * The wait queue is woken with the interruptible variant, so the
1085          * wait function matches, however returning because of an interrupt
1086          * will mess things up considerably, in particular when the caller is
1087          * the release method. And the xdev->error part prevents being stuck
1088          * forever in the event of a bizarre hardware bug: Pull the USB plug.
1089          */
1090
1091         while (wait_event_interruptible(fifo->waitq,
1092                                         fifo->fill <= (fifo->size - 8) ||
1093                                         xdev->error))
1094                 ; /* Empty loop */
1095
1096         if (xdev->error) {
1097                 rc = xdev->error;
1098                 goto unlock_done;
1099         }
1100
1101         fifo_write(fifo, (void *)msg, 8, xilly_memcpy);
1102
1103         try_queue_bulk_out(ep);
1104
1105 unlock_done:
1106         mutex_unlock(&xdev->msg_mutex);
1107
1108         return rc;
1109 }
1110
1111 /*
1112  * Note that flush_downstream() merely waits for the data to arrive to
1113  * the application logic at the FPGA -- unlike PCIe Xillybus' counterpart,
1114  * it does nothing to make it happen (and neither is it necessary).
1115  *
1116  * This function is not reentrant for the same @chan, but this is covered
1117  * by the fact that for any given @chan, it's called either by the open,
1118  * write, llseek and flush fops methods, which can't run in parallel (and the
1119  * write + flush and llseek method handlers are protected with out_mutex).
1120  *
1121  * chan->flushed is there to avoid multiple flushes at the same position,
1122  * in particular as a result of programs that close the file descriptor
1123  * e.g. after a dup2() for redirection.
1124  */
1125
1126 static int flush_downstream(struct xillyusb_channel *chan,
1127                             long timeout,
1128                             bool interruptible)
1129 {
1130         struct xillyusb_dev *xdev = chan->xdev;
1131         int chan_num = chan->chan_idx << 1;
1132         long deadline, left_to_sleep;
1133         int rc;
1134
1135         if (chan->flushed)
1136                 return 0;
1137
1138         deadline = jiffies + 1 + timeout;
1139
1140         if (chan->flushing) {
1141                 long cancel_deadline = jiffies + 1 + XILLY_RESPONSE_TIMEOUT;
1142
1143                 chan->canceled = 0;
1144                 rc = xillyusb_send_opcode(xdev, chan_num,
1145                                           OPCODE_CANCEL_CHECKPOINT, 0);
1146
1147                 if (rc)
1148                         return rc; /* Only real error, never -EINTR */
1149
1150                 /* Ignoring interrupts. Cancellation must be handled */
1151                 while (!chan->canceled) {
1152                         left_to_sleep = cancel_deadline - ((long)jiffies);
1153
1154                         if (left_to_sleep <= 0) {
1155                                 report_io_error(xdev, -EIO);
1156                                 return -EIO;
1157                         }
1158
1159                         rc = wait_event_interruptible_timeout(chan->flushq,
1160                                                               chan->canceled ||
1161                                                               xdev->error,
1162                                                               left_to_sleep);
1163
1164                         if (xdev->error)
1165                                 return xdev->error;
1166                 }
1167         }
1168
1169         chan->flushing = 1;
1170
1171         /*
1172          * The checkpoint is given in terms of data elements, not bytes. As
1173          * a result, if less than an element's worth of data is stored in the
1174          * FIFO, it's not flushed, including the flush before closing, which
1175          * means that such data is lost. This is consistent with PCIe Xillybus.
1176          */
1177
1178         rc = xillyusb_send_opcode(xdev, chan_num,
1179                                   OPCODE_SET_CHECKPOINT,
1180                                   chan->out_bytes >>
1181                                   chan->out_log2_element_size);
1182
1183         if (rc)
1184                 return rc; /* Only real error, never -EINTR */
1185
1186         if (!timeout) {
1187                 while (chan->flushing) {
1188                         rc = wait_event_interruptible(chan->flushq,
1189                                                       !chan->flushing ||
1190                                                       xdev->error);
1191                         if (xdev->error)
1192                                 return xdev->error;
1193
1194                         if (interruptible && rc)
1195                                 return -EINTR;
1196                 }
1197
1198                 goto done;
1199         }
1200
1201         while (chan->flushing) {
1202                 left_to_sleep = deadline - ((long)jiffies);
1203
1204                 if (left_to_sleep <= 0)
1205                         return -ETIMEDOUT;
1206
1207                 rc = wait_event_interruptible_timeout(chan->flushq,
1208                                                       !chan->flushing ||
1209                                                       xdev->error,
1210                                                       left_to_sleep);
1211
1212                 if (xdev->error)
1213                         return xdev->error;
1214
1215                 if (interruptible && rc < 0)
1216                         return -EINTR;
1217         }
1218
1219 done:
1220         chan->flushed = 1;
1221         return 0;
1222 }
1223
1224 /* request_read_anything(): Ask the FPGA for any little amount of data */
1225 static int request_read_anything(struct xillyusb_channel *chan,
1226                                  char opcode)
1227 {
1228         struct xillyusb_dev *xdev = chan->xdev;
1229         unsigned int sh = chan->in_log2_element_size;
1230         int chan_num = (chan->chan_idx << 1) | 1;
1231         u32 mercy = chan->in_consumed_bytes + (2 << sh) - 1;
1232
1233         return xillyusb_send_opcode(xdev, chan_num, opcode, mercy >> sh);
1234 }
1235
1236 static int xillyusb_open(struct inode *inode, struct file *filp)
1237 {
1238         struct xillyusb_dev *xdev;
1239         struct xillyusb_channel *chan;
1240         struct xillyfifo *in_fifo = NULL;
1241         struct xillyusb_endpoint *out_ep = NULL;
1242         int rc;
1243         int index;
1244
1245         mutex_lock(&kref_mutex);
1246
1247         rc = xillybus_find_inode(inode, (void **)&xdev, &index);
1248         if (rc) {
1249                 mutex_unlock(&kref_mutex);
1250                 return rc;
1251         }
1252
1253         kref_get(&xdev->kref);
1254         mutex_unlock(&kref_mutex);
1255
1256         chan = &xdev->channels[index];
1257         filp->private_data = chan;
1258
1259         mutex_lock(&chan->lock);
1260
1261         rc = -ENODEV;
1262
1263         if (xdev->error)
1264                 goto unmutex_fail;
1265
1266         if (((filp->f_mode & FMODE_READ) && !chan->readable) ||
1267             ((filp->f_mode & FMODE_WRITE) && !chan->writable))
1268                 goto unmutex_fail;
1269
1270         if ((filp->f_flags & O_NONBLOCK) && (filp->f_mode & FMODE_READ) &&
1271             chan->in_synchronous) {
1272                 dev_err(xdev->dev,
1273                         "open() failed: O_NONBLOCK not allowed for read on this device\n");
1274                 goto unmutex_fail;
1275         }
1276
1277         if ((filp->f_flags & O_NONBLOCK) && (filp->f_mode & FMODE_WRITE) &&
1278             chan->out_synchronous) {
1279                 dev_err(xdev->dev,
1280                         "open() failed: O_NONBLOCK not allowed for write on this device\n");
1281                 goto unmutex_fail;
1282         }
1283
1284         rc = -EBUSY;
1285
1286         if (((filp->f_mode & FMODE_READ) && chan->open_for_read) ||
1287             ((filp->f_mode & FMODE_WRITE) && chan->open_for_write))
1288                 goto unmutex_fail;
1289
1290         if (filp->f_mode & FMODE_READ)
1291                 chan->open_for_read = 1;
1292
1293         if (filp->f_mode & FMODE_WRITE)
1294                 chan->open_for_write = 1;
1295
1296         mutex_unlock(&chan->lock);
1297
1298         if (filp->f_mode & FMODE_WRITE) {
1299                 out_ep = endpoint_alloc(xdev,
1300                                         (chan->chan_idx + 2) | USB_DIR_OUT,
1301                                         bulk_out_work, BUF_SIZE_ORDER, BUFNUM);
1302
1303                 if (!out_ep) {
1304                         rc = -ENOMEM;
1305                         goto unopen;
1306                 }
1307
1308                 rc = fifo_init(&out_ep->fifo, chan->out_log2_fifo_size);
1309
1310                 if (rc)
1311                         goto late_unopen;
1312
1313                 out_ep->fill_mask = -(1 << chan->out_log2_element_size);
1314                 chan->out_bytes = 0;
1315                 chan->flushed = 0;
1316
1317                 /*
1318                  * Sending a flush request to a previously closed stream
1319                  * effectively opens it, and also waits until the command is
1320                  * confirmed by the FPGA. The latter is necessary because the
1321                  * data is sent through a separate BULK OUT endpoint, and the
1322                  * xHCI controller is free to reorder transmissions.
1323                  *
1324                  * This can't go wrong unless there's a serious hardware error
1325                  * (or the computer is stuck for 500 ms?)
1326                  */
1327                 rc = flush_downstream(chan, XILLY_RESPONSE_TIMEOUT, false);
1328
1329                 if (rc == -ETIMEDOUT) {
1330                         rc = -EIO;
1331                         report_io_error(xdev, rc);
1332                 }
1333
1334                 if (rc)
1335                         goto late_unopen;
1336         }
1337
1338         if (filp->f_mode & FMODE_READ) {
1339                 in_fifo = kzalloc(sizeof(*in_fifo), GFP_KERNEL);
1340
1341                 if (!in_fifo) {
1342                         rc = -ENOMEM;
1343                         goto late_unopen;
1344                 }
1345
1346                 rc = fifo_init(in_fifo, chan->in_log2_fifo_size);
1347
1348                 if (rc) {
1349                         kfree(in_fifo);
1350                         goto late_unopen;
1351                 }
1352         }
1353
1354         mutex_lock(&chan->lock);
1355         if (in_fifo) {
1356                 chan->in_fifo = in_fifo;
1357                 chan->read_data_ok = 1;
1358         }
1359         if (out_ep)
1360                 chan->out_ep = out_ep;
1361         mutex_unlock(&chan->lock);
1362
1363         if (in_fifo) {
1364                 u32 in_checkpoint = 0;
1365
1366                 if (!chan->in_synchronous)
1367                         in_checkpoint = in_fifo->size >>
1368                                 chan->in_log2_element_size;
1369
1370                 chan->in_consumed_bytes = 0;
1371                 chan->poll_used = 0;
1372                 chan->in_current_checkpoint = in_checkpoint;
1373                 rc = xillyusb_send_opcode(xdev, (chan->chan_idx << 1) | 1,
1374                                           OPCODE_SET_CHECKPOINT,
1375                                           in_checkpoint);
1376
1377                 if (rc) /* Failure guarantees that opcode wasn't sent */
1378                         goto unfifo;
1379
1380                 /*
1381                  * In non-blocking mode, request the FPGA to send any data it
1382                  * has right away. Otherwise, the first read() will always
1383                  * return -EAGAIN, which is OK strictly speaking, but ugly.
1384                  * Checking and unrolling if this fails isn't worth the
1385                  * effort -- the error is propagated to the first read()
1386                  * anyhow.
1387                  */
1388                 if (filp->f_flags & O_NONBLOCK)
1389                         request_read_anything(chan, OPCODE_SET_PUSH);
1390         }
1391
1392         return 0;
1393
1394 unfifo:
1395         chan->read_data_ok = 0;
1396         safely_assign_in_fifo(chan, NULL);
1397         fifo_mem_release(in_fifo);
1398         kfree(in_fifo);
1399
1400         if (out_ep) {
1401                 mutex_lock(&chan->lock);
1402                 chan->out_ep = NULL;
1403                 mutex_unlock(&chan->lock);
1404         }
1405
1406 late_unopen:
1407         if (out_ep)
1408                 endpoint_dealloc(out_ep);
1409
1410 unopen:
1411         mutex_lock(&chan->lock);
1412
1413         if (filp->f_mode & FMODE_READ)
1414                 chan->open_for_read = 0;
1415
1416         if (filp->f_mode & FMODE_WRITE)
1417                 chan->open_for_write = 0;
1418
1419         mutex_unlock(&chan->lock);
1420
1421         kref_put(&xdev->kref, cleanup_dev);
1422
1423         return rc;
1424
1425 unmutex_fail:
1426         kref_put(&xdev->kref, cleanup_dev);
1427         mutex_unlock(&chan->lock);
1428         return rc;
1429 }
1430
1431 static ssize_t xillyusb_read(struct file *filp, char __user *userbuf,
1432                              size_t count, loff_t *f_pos)
1433 {
1434         struct xillyusb_channel *chan = filp->private_data;
1435         struct xillyusb_dev *xdev = chan->xdev;
1436         struct xillyfifo *fifo = chan->in_fifo;
1437         int chan_num = (chan->chan_idx << 1) | 1;
1438
1439         long deadline, left_to_sleep;
1440         int bytes_done = 0;
1441         bool sent_set_push = false;
1442         int rc;
1443
1444         deadline = jiffies + 1 + XILLY_RX_TIMEOUT;
1445
1446         rc = mutex_lock_interruptible(&chan->in_mutex);
1447
1448         if (rc)
1449                 return rc;
1450
1451         while (1) {
1452                 u32 fifo_checkpoint_bytes, complete_checkpoint_bytes;
1453                 u32 complete_checkpoint, fifo_checkpoint;
1454                 u32 checkpoint;
1455                 s32 diff, leap;
1456                 unsigned int sh = chan->in_log2_element_size;
1457                 bool checkpoint_for_complete;
1458
1459                 rc = fifo_read(fifo, (__force void *)userbuf + bytes_done,
1460                                count - bytes_done, xilly_copy_to_user);
1461
1462                 if (rc < 0)
1463                         break;
1464
1465                 bytes_done += rc;
1466                 chan->in_consumed_bytes += rc;
1467
1468                 left_to_sleep = deadline - ((long)jiffies);
1469
1470                 /*
1471                  * Some 32-bit arithmetic that may wrap. Note that
1472                  * complete_checkpoint is rounded up to the closest element
1473                  * boundary, because the read() can't be completed otherwise.
1474                  * fifo_checkpoint_bytes is rounded down, because it protects
1475                  * in_fifo from overflowing.
1476                  */
1477
1478                 fifo_checkpoint_bytes = chan->in_consumed_bytes + fifo->size;
1479                 complete_checkpoint_bytes =
1480                         chan->in_consumed_bytes + count - bytes_done;
1481
1482                 fifo_checkpoint = fifo_checkpoint_bytes >> sh;
1483                 complete_checkpoint =
1484                         (complete_checkpoint_bytes + (1 << sh) - 1) >> sh;
1485
1486                 diff = (fifo_checkpoint - complete_checkpoint) << sh;
1487
1488                 if (chan->in_synchronous && diff >= 0) {
1489                         checkpoint = complete_checkpoint;
1490                         checkpoint_for_complete = true;
1491                 } else {
1492                         checkpoint = fifo_checkpoint;
1493                         checkpoint_for_complete = false;
1494                 }
1495
1496                 leap = (checkpoint - chan->in_current_checkpoint) << sh;
1497
1498                 /*
1499                  * To prevent flooding of OPCODE_SET_CHECKPOINT commands as
1500                  * data is consumed, it's issued only if it moves the
1501                  * checkpoint by at least an 8th of the FIFO's size, or if
1502                  * it's necessary to complete the number of bytes requested by
1503                  * the read() call.
1504                  *
1505                  * chan->read_data_ok is checked to spare an unnecessary
1506                  * submission after receiving EOF, however it's harmless if
1507                  * such slips away.
1508                  */
1509
1510                 if (chan->read_data_ok &&
1511                     (leap > (fifo->size >> 3) ||
1512                      (checkpoint_for_complete && leap > 0))) {
1513                         chan->in_current_checkpoint = checkpoint;
1514                         rc = xillyusb_send_opcode(xdev, chan_num,
1515                                                   OPCODE_SET_CHECKPOINT,
1516                                                   checkpoint);
1517
1518                         if (rc)
1519                                 break;
1520                 }
1521
1522                 if (bytes_done == count ||
1523                     (left_to_sleep <= 0 && bytes_done))
1524                         break;
1525
1526                 /*
1527                  * Reaching here means that the FIFO was empty when
1528                  * fifo_read() returned, but not necessarily right now. Error
1529                  * and EOF are checked and reported only now, so that no data
1530                  * that managed its way to the FIFO is lost.
1531                  */
1532
1533                 if (!READ_ONCE(chan->read_data_ok)) { /* FPGA has sent EOF */
1534                         /* Has data slipped into the FIFO since fifo_read()? */
1535                         smp_rmb();
1536                         if (READ_ONCE(fifo->fill))
1537                                 continue;
1538
1539                         rc = 0;
1540                         break;
1541                 }
1542
1543                 if (xdev->error) {
1544                         rc = xdev->error;
1545                         break;
1546                 }
1547
1548                 if (filp->f_flags & O_NONBLOCK) {
1549                         rc = -EAGAIN;
1550                         break;
1551                 }
1552
1553                 if (!sent_set_push) {
1554                         rc = xillyusb_send_opcode(xdev, chan_num,
1555                                                   OPCODE_SET_PUSH,
1556                                                   complete_checkpoint);
1557
1558                         if (rc)
1559                                 break;
1560
1561                         sent_set_push = true;
1562                 }
1563
1564                 if (left_to_sleep > 0) {
1565                         /*
1566                          * Note that when xdev->error is set (e.g. when the
1567                          * device is unplugged), read_data_ok turns zero and
1568                          * fifo->waitq is awaken.
1569                          * Therefore no special attention to xdev->error.
1570                          */
1571
1572                         rc = wait_event_interruptible_timeout
1573                                 (fifo->waitq,
1574                                  fifo->fill || !chan->read_data_ok,
1575                                  left_to_sleep);
1576                 } else { /* bytes_done == 0 */
1577                         /* Tell FPGA to send anything it has */
1578                         rc = request_read_anything(chan, OPCODE_UPDATE_PUSH);
1579
1580                         if (rc)
1581                                 break;
1582
1583                         rc = wait_event_interruptible
1584                                 (fifo->waitq,
1585                                  fifo->fill || !chan->read_data_ok);
1586                 }
1587
1588                 if (rc < 0) {
1589                         rc = -EINTR;
1590                         break;
1591                 }
1592         }
1593
1594         if (((filp->f_flags & O_NONBLOCK) || chan->poll_used) &&
1595             !READ_ONCE(fifo->fill))
1596                 request_read_anything(chan, OPCODE_SET_PUSH);
1597
1598         mutex_unlock(&chan->in_mutex);
1599
1600         if (bytes_done)
1601                 return bytes_done;
1602
1603         return rc;
1604 }
1605
1606 static int xillyusb_flush(struct file *filp, fl_owner_t id)
1607 {
1608         struct xillyusb_channel *chan = filp->private_data;
1609         int rc;
1610
1611         if (!(filp->f_mode & FMODE_WRITE))
1612                 return 0;
1613
1614         rc = mutex_lock_interruptible(&chan->out_mutex);
1615
1616         if (rc)
1617                 return rc;
1618
1619         /*
1620          * One second's timeout on flushing. Interrupts are ignored, because if
1621          * the user pressed CTRL-C, that interrupt will still be in flight by
1622          * the time we reach here, and the opportunity to flush is lost.
1623          */
1624         rc = flush_downstream(chan, HZ, false);
1625
1626         mutex_unlock(&chan->out_mutex);
1627
1628         if (rc == -ETIMEDOUT) {
1629                 /* The things you do to use dev_warn() and not pr_warn() */
1630                 struct xillyusb_dev *xdev = chan->xdev;
1631
1632                 mutex_lock(&chan->lock);
1633                 if (!xdev->error)
1634                         dev_warn(xdev->dev,
1635                                  "Timed out while flushing. Output data may be lost.\n");
1636                 mutex_unlock(&chan->lock);
1637         }
1638
1639         return rc;
1640 }
1641
1642 static ssize_t xillyusb_write(struct file *filp, const char __user *userbuf,
1643                               size_t count, loff_t *f_pos)
1644 {
1645         struct xillyusb_channel *chan = filp->private_data;
1646         struct xillyusb_dev *xdev = chan->xdev;
1647         struct xillyfifo *fifo = &chan->out_ep->fifo;
1648         int rc;
1649
1650         rc = mutex_lock_interruptible(&chan->out_mutex);
1651
1652         if (rc)
1653                 return rc;
1654
1655         while (1) {
1656                 if (xdev->error) {
1657                         rc = xdev->error;
1658                         break;
1659                 }
1660
1661                 if (count == 0)
1662                         break;
1663
1664                 rc = fifo_write(fifo, (__force void *)userbuf, count,
1665                                 xilly_copy_from_user);
1666
1667                 if (rc != 0)
1668                         break;
1669
1670                 if (filp->f_flags & O_NONBLOCK) {
1671                         rc = -EAGAIN;
1672                         break;
1673                 }
1674
1675                 if (wait_event_interruptible
1676                     (fifo->waitq,
1677                      fifo->fill != fifo->size || xdev->error)) {
1678                         rc = -EINTR;
1679                         break;
1680                 }
1681         }
1682
1683         if (rc < 0)
1684                 goto done;
1685
1686         chan->out_bytes += rc;
1687
1688         if (rc) {
1689                 try_queue_bulk_out(chan->out_ep);
1690                 chan->flushed = 0;
1691         }
1692
1693         if (chan->out_synchronous) {
1694                 int flush_rc = flush_downstream(chan, 0, true);
1695
1696                 if (flush_rc && !rc)
1697                         rc = flush_rc;
1698         }
1699
1700 done:
1701         mutex_unlock(&chan->out_mutex);
1702
1703         return rc;
1704 }
1705
1706 static int xillyusb_release(struct inode *inode, struct file *filp)
1707 {
1708         struct xillyusb_channel *chan = filp->private_data;
1709         struct xillyusb_dev *xdev = chan->xdev;
1710         int rc_read = 0, rc_write = 0;
1711
1712         if (filp->f_mode & FMODE_READ) {
1713                 struct xillyfifo *in_fifo = chan->in_fifo;
1714
1715                 rc_read = xillyusb_send_opcode(xdev, (chan->chan_idx << 1) | 1,
1716                                                OPCODE_CLOSE, 0);
1717                 /*
1718                  * If rc_read is nonzero, xdev->error indicates a global
1719                  * device error. The error is reported later, so that
1720                  * resources are freed.
1721                  *
1722                  * Looping on wait_event_interruptible() kinda breaks the idea
1723                  * of being interruptible, and this should have been
1724                  * wait_event(). Only it's being waken with
1725                  * wake_up_interruptible() for the sake of other uses. If
1726                  * there's a global device error, chan->read_data_ok is
1727                  * deasserted and the wait queue is awaken, so this is covered.
1728                  */
1729
1730                 while (wait_event_interruptible(in_fifo->waitq,
1731                                                 !chan->read_data_ok))
1732                         ; /* Empty loop */
1733
1734                 safely_assign_in_fifo(chan, NULL);
1735                 fifo_mem_release(in_fifo);
1736                 kfree(in_fifo);
1737
1738                 mutex_lock(&chan->lock);
1739                 chan->open_for_read = 0;
1740                 mutex_unlock(&chan->lock);
1741         }
1742
1743         if (filp->f_mode & FMODE_WRITE) {
1744                 struct xillyusb_endpoint *ep = chan->out_ep;
1745                 /*
1746                  * chan->flushing isn't zeroed. If the pre-release flush timed
1747                  * out, a cancel request will be sent before the next
1748                  * OPCODE_SET_CHECKPOINT (i.e. when the file is opened again).
1749                  * This is despite that the FPGA forgets about the checkpoint
1750                  * request as the file closes. Still, in an exceptional race
1751                  * condition, the FPGA could send an OPCODE_REACHED_CHECKPOINT
1752                  * just before closing that would reach the host after the
1753                  * file has re-opened.
1754                  */
1755
1756                 mutex_lock(&chan->lock);
1757                 chan->out_ep = NULL;
1758                 mutex_unlock(&chan->lock);
1759
1760                 endpoint_quiesce(ep);
1761                 endpoint_dealloc(ep);
1762
1763                 /* See comments on rc_read above */
1764                 rc_write = xillyusb_send_opcode(xdev, chan->chan_idx << 1,
1765                                                 OPCODE_CLOSE, 0);
1766
1767                 mutex_lock(&chan->lock);
1768                 chan->open_for_write = 0;
1769                 mutex_unlock(&chan->lock);
1770         }
1771
1772         kref_put(&xdev->kref, cleanup_dev);
1773
1774         return rc_read ? rc_read : rc_write;
1775 }
1776
1777 /*
1778  * Xillybus' API allows device nodes to be seekable, giving the user
1779  * application access to a RAM array on the FPGA (or logic emulating it).
1780  */
1781
1782 static loff_t xillyusb_llseek(struct file *filp, loff_t offset, int whence)
1783 {
1784         struct xillyusb_channel *chan = filp->private_data;
1785         struct xillyusb_dev *xdev = chan->xdev;
1786         loff_t pos = filp->f_pos;
1787         int rc = 0;
1788         unsigned int log2_element_size = chan->readable ?
1789                 chan->in_log2_element_size : chan->out_log2_element_size;
1790
1791         /*
1792          * Take both mutexes not allowing interrupts, since it seems like
1793          * common applications don't expect an -EINTR here. Besides, multiple
1794          * access to a single file descriptor on seekable devices is a mess
1795          * anyhow.
1796          */
1797
1798         mutex_lock(&chan->out_mutex);
1799         mutex_lock(&chan->in_mutex);
1800
1801         switch (whence) {
1802         case SEEK_SET:
1803                 pos = offset;
1804                 break;
1805         case SEEK_CUR:
1806                 pos += offset;
1807                 break;
1808         case SEEK_END:
1809                 pos = offset; /* Going to the end => to the beginning */
1810                 break;
1811         default:
1812                 rc = -EINVAL;
1813                 goto end;
1814         }
1815
1816         /* In any case, we must finish on an element boundary */
1817         if (pos & ((1 << log2_element_size) - 1)) {
1818                 rc = -EINVAL;
1819                 goto end;
1820         }
1821
1822         rc = xillyusb_send_opcode(xdev, chan->chan_idx << 1,
1823                                   OPCODE_SET_ADDR,
1824                                   pos >> log2_element_size);
1825
1826         if (rc)
1827                 goto end;
1828
1829         if (chan->writable) {
1830                 chan->flushed = 0;
1831                 rc = flush_downstream(chan, HZ, false);
1832         }
1833
1834 end:
1835         mutex_unlock(&chan->out_mutex);
1836         mutex_unlock(&chan->in_mutex);
1837
1838         if (rc) /* Return error after releasing mutexes */
1839                 return rc;
1840
1841         filp->f_pos = pos;
1842
1843         return pos;
1844 }
1845
1846 static __poll_t xillyusb_poll(struct file *filp, poll_table *wait)
1847 {
1848         struct xillyusb_channel *chan = filp->private_data;
1849         __poll_t mask = 0;
1850
1851         if (chan->in_fifo)
1852                 poll_wait(filp, &chan->in_fifo->waitq, wait);
1853
1854         if (chan->out_ep)
1855                 poll_wait(filp, &chan->out_ep->fifo.waitq, wait);
1856
1857         /*
1858          * If this is the first time poll() is called, and the file is
1859          * readable, set the relevant flag. Also tell the FPGA to send all it
1860          * has, to kickstart the mechanism that ensures there's always some
1861          * data in in_fifo unless the stream is dry end-to-end. Note that the
1862          * first poll() may not return a EPOLLIN, even if there's data on the
1863          * FPGA. Rather, the data will arrive soon, and trigger the relevant
1864          * wait queue.
1865          */
1866
1867         if (!chan->poll_used && chan->in_fifo) {
1868                 chan->poll_used = 1;
1869                 request_read_anything(chan, OPCODE_SET_PUSH);
1870         }
1871
1872         /*
1873          * poll() won't play ball regarding read() channels which
1874          * are synchronous. Allowing that will create situations where data has
1875          * been delivered at the FPGA, and users expecting select() to wake up,
1876          * which it may not. So make it never work.
1877          */
1878
1879         if (chan->in_fifo && !chan->in_synchronous &&
1880             (READ_ONCE(chan->in_fifo->fill) || !chan->read_data_ok))
1881                 mask |= EPOLLIN | EPOLLRDNORM;
1882
1883         if (chan->out_ep &&
1884             (READ_ONCE(chan->out_ep->fifo.fill) != chan->out_ep->fifo.size))
1885                 mask |= EPOLLOUT | EPOLLWRNORM;
1886
1887         if (chan->xdev->error)
1888                 mask |= EPOLLERR;
1889
1890         return mask;
1891 }
1892
1893 static const struct file_operations xillyusb_fops = {
1894         .owner      = THIS_MODULE,
1895         .read       = xillyusb_read,
1896         .write      = xillyusb_write,
1897         .open       = xillyusb_open,
1898         .flush      = xillyusb_flush,
1899         .release    = xillyusb_release,
1900         .llseek     = xillyusb_llseek,
1901         .poll       = xillyusb_poll,
1902 };
1903
1904 static int xillyusb_setup_base_eps(struct xillyusb_dev *xdev)
1905 {
1906         struct usb_device *udev = xdev->udev;
1907
1908         /* Verify that device has the two fundamental bulk in/out endpoints */
1909         if (usb_pipe_type_check(udev, usb_sndbulkpipe(udev, MSG_EP_NUM)) ||
1910             usb_pipe_type_check(udev, usb_rcvbulkpipe(udev, IN_EP_NUM)))
1911                 return -ENODEV;
1912
1913         xdev->msg_ep = endpoint_alloc(xdev, MSG_EP_NUM | USB_DIR_OUT,
1914                                       bulk_out_work, 1, 2);
1915         if (!xdev->msg_ep)
1916                 return -ENOMEM;
1917
1918         if (fifo_init(&xdev->msg_ep->fifo, 13)) /* 8 kiB */
1919                 goto dealloc;
1920
1921         xdev->msg_ep->fill_mask = -8; /* 8 bytes granularity */
1922
1923         xdev->in_ep = endpoint_alloc(xdev, IN_EP_NUM | USB_DIR_IN,
1924                                      bulk_in_work, BUF_SIZE_ORDER, BUFNUM);
1925         if (!xdev->in_ep)
1926                 goto dealloc;
1927
1928         try_queue_bulk_in(xdev->in_ep);
1929
1930         return 0;
1931
1932 dealloc:
1933         endpoint_dealloc(xdev->msg_ep); /* Also frees FIFO mem if allocated */
1934         xdev->msg_ep = NULL;
1935         return -ENOMEM;
1936 }
1937
1938 static int setup_channels(struct xillyusb_dev *xdev,
1939                           __le16 *chandesc,
1940                           int num_channels)
1941 {
1942         struct usb_device *udev = xdev->udev;
1943         struct xillyusb_channel *chan, *new_channels;
1944         int i;
1945
1946         chan = kcalloc(num_channels, sizeof(*chan), GFP_KERNEL);
1947         if (!chan)
1948                 return -ENOMEM;
1949
1950         new_channels = chan;
1951
1952         for (i = 0; i < num_channels; i++, chan++) {
1953                 unsigned int in_desc = le16_to_cpu(*chandesc++);
1954                 unsigned int out_desc = le16_to_cpu(*chandesc++);
1955
1956                 chan->xdev = xdev;
1957                 mutex_init(&chan->in_mutex);
1958                 mutex_init(&chan->out_mutex);
1959                 mutex_init(&chan->lock);
1960                 init_waitqueue_head(&chan->flushq);
1961
1962                 chan->chan_idx = i;
1963
1964                 if (in_desc & 0x80) { /* Entry is valid */
1965                         chan->readable = 1;
1966                         chan->in_synchronous = !!(in_desc & 0x40);
1967                         chan->in_seekable = !!(in_desc & 0x20);
1968                         chan->in_log2_element_size = in_desc & 0x0f;
1969                         chan->in_log2_fifo_size = ((in_desc >> 8) & 0x1f) + 16;
1970                 }
1971
1972                 /*
1973                  * A downstream channel should never exist above index 13,
1974                  * as it would request a nonexistent BULK endpoint > 15.
1975                  * In the peculiar case that it does, it's ignored silently.
1976                  */
1977
1978                 if ((out_desc & 0x80) && i < 14) { /* Entry is valid */
1979                         if (usb_pipe_type_check(udev,
1980                                                 usb_sndbulkpipe(udev, i + 2))) {
1981                                 dev_err(xdev->dev,
1982                                         "Missing BULK OUT endpoint %d\n",
1983                                         i + 2);
1984                                 kfree(new_channels);
1985                                 return -ENODEV;
1986                         }
1987
1988                         chan->writable = 1;
1989                         chan->out_synchronous = !!(out_desc & 0x40);
1990                         chan->out_seekable = !!(out_desc & 0x20);
1991                         chan->out_log2_element_size = out_desc & 0x0f;
1992                         chan->out_log2_fifo_size =
1993                                 ((out_desc >> 8) & 0x1f) + 16;
1994                 }
1995         }
1996
1997         xdev->channels = new_channels;
1998         return 0;
1999 }
2000
2001 static int xillyusb_discovery(struct usb_interface *interface)
2002 {
2003         int rc;
2004         struct xillyusb_dev *xdev = usb_get_intfdata(interface);
2005         __le16 bogus_chandesc[2];
2006         struct xillyfifo idt_fifo;
2007         struct xillyusb_channel *chan;
2008         unsigned int idt_len, names_offset;
2009         unsigned char *idt;
2010         int num_channels;
2011
2012         rc = xillyusb_send_opcode(xdev, ~0, OPCODE_QUIESCE, 0);
2013
2014         if (rc) {
2015                 dev_err(&interface->dev, "Failed to send quiesce request. Aborting.\n");
2016                 return rc;
2017         }
2018
2019         /* Phase I: Set up one fake upstream channel and obtain IDT */
2020
2021         /* Set up a fake IDT with one async IN stream */
2022         bogus_chandesc[0] = cpu_to_le16(0x80);
2023         bogus_chandesc[1] = cpu_to_le16(0);
2024
2025         rc = setup_channels(xdev, bogus_chandesc, 1);
2026
2027         if (rc)
2028                 return rc;
2029
2030         rc = fifo_init(&idt_fifo, LOG2_IDT_FIFO_SIZE);
2031
2032         if (rc)
2033                 return rc;
2034
2035         chan = xdev->channels;
2036
2037         chan->in_fifo = &idt_fifo;
2038         chan->read_data_ok = 1;
2039
2040         xdev->num_channels = 1;
2041
2042         rc = xillyusb_send_opcode(xdev, ~0, OPCODE_REQ_IDT, 0);
2043
2044         if (rc) {
2045                 dev_err(&interface->dev, "Failed to send IDT request. Aborting.\n");
2046                 goto unfifo;
2047         }
2048
2049         rc = wait_event_interruptible_timeout(idt_fifo.waitq,
2050                                               !chan->read_data_ok,
2051                                               XILLY_RESPONSE_TIMEOUT);
2052
2053         if (xdev->error) {
2054                 rc = xdev->error;
2055                 goto unfifo;
2056         }
2057
2058         if (rc < 0) {
2059                 rc = -EINTR; /* Interrupt on probe method? Interesting. */
2060                 goto unfifo;
2061         }
2062
2063         if (chan->read_data_ok) {
2064                 rc = -ETIMEDOUT;
2065                 dev_err(&interface->dev, "No response from FPGA. Aborting.\n");
2066                 goto unfifo;
2067         }
2068
2069         idt_len = READ_ONCE(idt_fifo.fill);
2070         idt = kmalloc(idt_len, GFP_KERNEL);
2071
2072         if (!idt) {
2073                 rc = -ENOMEM;
2074                 goto unfifo;
2075         }
2076
2077         fifo_read(&idt_fifo, idt, idt_len, xilly_memcpy);
2078
2079         if (crc32_le(~0, idt, idt_len) != 0) {
2080                 dev_err(&interface->dev, "IDT failed CRC check. Aborting.\n");
2081                 rc = -ENODEV;
2082                 goto unidt;
2083         }
2084
2085         if (*idt > 0x90) {
2086                 dev_err(&interface->dev, "No support for IDT version 0x%02x. Maybe the xillyusb driver needs an upgrade. Aborting.\n",
2087                         (int)*idt);
2088                 rc = -ENODEV;
2089                 goto unidt;
2090         }
2091
2092         /* Phase II: Set up the streams as defined in IDT */
2093
2094         num_channels = le16_to_cpu(*((__le16 *)(idt + 1)));
2095         names_offset = 3 + num_channels * 4;
2096         idt_len -= 4; /* Exclude CRC */
2097
2098         if (idt_len < names_offset) {
2099                 dev_err(&interface->dev, "IDT too short. This is exceptionally weird, because its CRC is OK\n");
2100                 rc = -ENODEV;
2101                 goto unidt;
2102         }
2103
2104         rc = setup_channels(xdev, (void *)idt + 3, num_channels);
2105
2106         if (rc)
2107                 goto unidt;
2108
2109         /*
2110          * Except for wildly misbehaving hardware, or if it was disconnected
2111          * just after responding with the IDT, there is no reason for any
2112          * work item to be running now. To be sure that xdev->channels
2113          * is updated on anything that might run in parallel, flush the
2114          * device's workqueue and the wakeup work item. This rarely
2115          * does anything.
2116          */
2117         flush_workqueue(xdev->workq);
2118         flush_work(&xdev->wakeup_workitem);
2119
2120         xdev->num_channels = num_channels;
2121
2122         fifo_mem_release(&idt_fifo);
2123         kfree(chan);
2124
2125         rc = xillybus_init_chrdev(&interface->dev, &xillyusb_fops,
2126                                   THIS_MODULE, xdev,
2127                                   idt + names_offset,
2128                                   idt_len - names_offset,
2129                                   num_channels,
2130                                   xillyname, true);
2131
2132         kfree(idt);
2133
2134         return rc;
2135
2136 unidt:
2137         kfree(idt);
2138
2139 unfifo:
2140         safely_assign_in_fifo(chan, NULL);
2141         fifo_mem_release(&idt_fifo);
2142
2143         return rc;
2144 }
2145
2146 static int xillyusb_probe(struct usb_interface *interface,
2147                           const struct usb_device_id *id)
2148 {
2149         struct xillyusb_dev *xdev;
2150         int rc;
2151
2152         xdev = kzalloc(sizeof(*xdev), GFP_KERNEL);
2153         if (!xdev)
2154                 return -ENOMEM;
2155
2156         kref_init(&xdev->kref);
2157         mutex_init(&xdev->process_in_mutex);
2158         mutex_init(&xdev->msg_mutex);
2159
2160         xdev->udev = usb_get_dev(interface_to_usbdev(interface));
2161         xdev->dev = &interface->dev;
2162         xdev->error = 0;
2163         spin_lock_init(&xdev->error_lock);
2164         xdev->in_counter = 0;
2165         xdev->in_bytes_left = 0;
2166         xdev->workq = alloc_workqueue(xillyname, WQ_HIGHPRI, 0);
2167
2168         if (!xdev->workq) {
2169                 dev_err(&interface->dev, "Failed to allocate work queue\n");
2170                 rc = -ENOMEM;
2171                 goto fail;
2172         }
2173
2174         INIT_WORK(&xdev->wakeup_workitem, wakeup_all);
2175
2176         usb_set_intfdata(interface, xdev);
2177
2178         rc = xillyusb_setup_base_eps(xdev);
2179         if (rc)
2180                 goto fail;
2181
2182         rc = xillyusb_discovery(interface);
2183         if (rc)
2184                 goto latefail;
2185
2186         return 0;
2187
2188 latefail:
2189         endpoint_quiesce(xdev->in_ep);
2190         endpoint_quiesce(xdev->msg_ep);
2191
2192 fail:
2193         usb_set_intfdata(interface, NULL);
2194         kref_put(&xdev->kref, cleanup_dev);
2195         return rc;
2196 }
2197
2198 static void xillyusb_disconnect(struct usb_interface *interface)
2199 {
2200         struct xillyusb_dev *xdev = usb_get_intfdata(interface);
2201         struct xillyusb_endpoint *msg_ep = xdev->msg_ep;
2202         struct xillyfifo *fifo = &msg_ep->fifo;
2203         int rc;
2204         int i;
2205
2206         xillybus_cleanup_chrdev(xdev, &interface->dev);
2207
2208         /*
2209          * Try to send OPCODE_QUIESCE, which will fail silently if the device
2210          * was disconnected, but makes sense on module unload.
2211          */
2212
2213         msg_ep->wake_on_drain = true;
2214         xillyusb_send_opcode(xdev, ~0, OPCODE_QUIESCE, 0);
2215
2216         /*
2217          * If the device has been disconnected, sending the opcode causes
2218          * a global device error with xdev->error, if such error didn't
2219          * occur earlier. Hence timing out means that the USB link is fine,
2220          * but somehow the message wasn't sent. Should never happen.
2221          */
2222
2223         rc = wait_event_interruptible_timeout(fifo->waitq,
2224                                               msg_ep->drained || xdev->error,
2225                                               XILLY_RESPONSE_TIMEOUT);
2226
2227         if (!rc)
2228                 dev_err(&interface->dev,
2229                         "Weird timeout condition on sending quiesce request.\n");
2230
2231         report_io_error(xdev, -ENODEV); /* Discourage further activity */
2232
2233         /*
2234          * This device driver is declared with soft_unbind set, or else
2235          * sending OPCODE_QUIESCE above would always fail. The price is
2236          * that the USB framework didn't kill outstanding URBs, so it has
2237          * to be done explicitly before returning from this call.
2238          */
2239
2240         for (i = 0; i < xdev->num_channels; i++) {
2241                 struct xillyusb_channel *chan = &xdev->channels[i];
2242
2243                 /*
2244                  * Lock taken to prevent chan->out_ep from changing. It also
2245                  * ensures xillyusb_open() and xillyusb_flush() don't access
2246                  * xdev->dev after being nullified below.
2247                  */
2248                 mutex_lock(&chan->lock);
2249                 if (chan->out_ep)
2250                         endpoint_quiesce(chan->out_ep);
2251                 mutex_unlock(&chan->lock);
2252         }
2253
2254         endpoint_quiesce(xdev->in_ep);
2255         endpoint_quiesce(xdev->msg_ep);
2256
2257         usb_set_intfdata(interface, NULL);
2258
2259         xdev->dev = NULL;
2260
2261         mutex_lock(&kref_mutex);
2262         kref_put(&xdev->kref, cleanup_dev);
2263         mutex_unlock(&kref_mutex);
2264 }
2265
2266 static struct usb_driver xillyusb_driver = {
2267         .name = xillyname,
2268         .id_table = xillyusb_table,
2269         .probe = xillyusb_probe,
2270         .disconnect = xillyusb_disconnect,
2271         .soft_unbind = 1,
2272 };
2273
2274 static int __init xillyusb_init(void)
2275 {
2276         int rc = 0;
2277
2278         wakeup_wq = alloc_workqueue(xillyname, 0, 0);
2279         if (!wakeup_wq)
2280                 return -ENOMEM;
2281
2282         if (LOG2_INITIAL_FIFO_BUF_SIZE > PAGE_SHIFT)
2283                 fifo_buf_order = LOG2_INITIAL_FIFO_BUF_SIZE - PAGE_SHIFT;
2284         else
2285                 fifo_buf_order = 0;
2286
2287         rc = usb_register(&xillyusb_driver);
2288
2289         if (rc)
2290                 destroy_workqueue(wakeup_wq);
2291
2292         return rc;
2293 }
2294
2295 static void __exit xillyusb_exit(void)
2296 {
2297         usb_deregister(&xillyusb_driver);
2298
2299         destroy_workqueue(wakeup_wq);
2300 }
2301
2302 module_init(xillyusb_init);
2303 module_exit(xillyusb_exit);