fs/fuse/dev.c

   1 /*
   2   FUSE: Filesystem in Userspace
   3   Copyright (C) 2001-2008  Miklos Szeredi <[email protected]>
   4
   5   This program can be distributed under the terms of the GNU GPL.
   6   See the file COPYING.
   7 */
   8
   9 #include "fuse_i.h"
  10
  11 #include <linux/init.h>
  12 #include <linux/module.h>
  13 #include <linux/poll.h>
  14 #include <linux/uio.h>
  15 #include <linux/miscdevice.h>
  16 #include <linux/pagemap.h>
  17 #include <linux/file.h>
  18 #include <linux/slab.h>
  19 #include <linux/pipe_fs_i.h>
  20 #include <linux/swap.h>
  21 #include <linux/splice.h>
  22 #include <linux/aio.h>
  23
  24 MODULE_ALIAS_MISCDEV(FUSE_MINOR);
  25 MODULE_ALIAS("devname:fuse");
  26
  27 static struct kmem_cache *fuse_req_cachep;
  28
  29 static struct fuse_conn *fuse_get_conn(struct file *file)
  30 {
  31         /*
  32          * Lockless access is OK, because file->private data is set
  33          * once during mount and is valid until the file is released.
  34          */
  35         return file->private_data;
  36 }
  37
  38 static void fuse_request_init(struct fuse_req *req, struct page **pages,
  39                               struct fuse_page_desc *page_descs,
  40                               unsigned npages)
  41 {
  42         memset(req, 0, sizeof(*req));
  43         memset(pages, 0, sizeof(*pages) * npages);
  44         memset(page_descs, 0, sizeof(*page_descs) * npages);
  45         INIT_LIST_HEAD(&req->list);
  46         INIT_LIST_HEAD(&req->intr_entry);
  47         init_waitqueue_head(&req->waitq);
  48         atomic_set(&req->count, 1);
  49         req->pages = pages;
  50         req->page_descs = page_descs;
  51         req->max_pages = npages;
  52 }
  53
  54 static struct fuse_req *__fuse_request_alloc(unsigned npages, gfp_t flags)
  55 {
  56         struct fuse_req *req = kmem_cache_alloc(fuse_req_cachep, flags);
  57         if (req) {
  58                 struct page **pages;
  59                 struct fuse_page_desc *page_descs;
  60
  61                 if (npages <= FUSE_REQ_INLINE_PAGES) {
  62                         pages = req->inline_pages;
  63                         page_descs = req->inline_page_descs;
  64                 } else {
  65                         pages = kmalloc(sizeof(struct page *) * npages, flags);
  66                         page_descs = kmalloc(sizeof(struct fuse_page_desc) *
  67                                              npages, flags);
  68                 }
  69
  70                 if (!pages || !page_descs) {
  71                         kfree(pages);
  72                         kfree(page_descs);
  73                         kmem_cache_free(fuse_req_cachep, req);
  74                         return NULL;
  75                 }
  76
  77                 fuse_request_init(req, pages, page_descs, npages);
  78         }
  79         return req;
  80 }
  81
  82 struct fuse_req *fuse_request_alloc(unsigned npages)
  83 {
  84         return __fuse_request_alloc(npages, GFP_KERNEL);
  85 }
  86 EXPORT_SYMBOL_GPL(fuse_request_alloc);
  87
  88 struct fuse_req *fuse_request_alloc_nofs(unsigned npages)
  89 {
  90         return __fuse_request_alloc(npages, GFP_NOFS);
  91 }
  92
  93 void fuse_request_free(struct fuse_req *req)
  94 {
  95         if (req->pages != req->inline_pages) {
  96                 kfree(req->pages);
  97                 kfree(req->page_descs);
  98         }
  99         kmem_cache_free(fuse_req_cachep, req);
 100 }
 101
 102 static void block_sigs(sigset_t *oldset)
 103 {
 104         sigset_t mask;
 105
 106         siginitsetinv(&mask, sigmask(SIGKILL));
 107         sigprocmask(SIG_BLOCK, &mask, oldset);
 108 }
 109
 110 static void restore_sigs(sigset_t *oldset)
 111 {
 112         sigprocmask(SIG_SETMASK, oldset, NULL);
 113 }
 114
 115 void __fuse_get_request(struct fuse_req *req)
 116 {
 117         atomic_inc(&req->count);
 118 }
 119
 120 /* Must be called with > 1 refcount */
 121 static void __fuse_put_request(struct fuse_req *req)
 122 {
 123         BUG_ON(atomic_read(&req->count) < 2);
 124         atomic_dec(&req->count);
 125 }
 126
 127 static void fuse_req_init_context(struct fuse_req *req)
 128 {
 129         req->in.h.uid = from_kuid_munged(&init_user_ns, current_fsuid());
 130         req->in.h.gid = from_kgid_munged(&init_user_ns, current_fsgid());
 131         req->in.h.pid = current->pid;
 132 }
 133
 134 static bool fuse_block_alloc(struct fuse_conn *fc, bool for_background)
 135 {
 136         return !fc->initialized || (for_background && fc->blocked);
 137 }
 138
 139 static struct fuse_req *__fuse_get_req(struct fuse_conn *fc, unsigned npages,
 140                                        bool for_background)
 141 {
 142         struct fuse_req *req;
 143         int err;
 144         atomic_inc(&fc->num_waiting);
 145
 146         if (fuse_block_alloc(fc, for_background)) {
 147                 sigset_t oldset;
 148                 int intr;
 149
 150                 block_sigs(&oldset);
 151                 intr = wait_event_interruptible_exclusive(fc->blocked_waitq,
 152                                 !fuse_block_alloc(fc, for_background));
 153                 restore_sigs(&oldset);
 154                 err = -EINTR;
 155                 if (intr)
 156                         goto out;
 157         }
 158
 159         err = -ENOTCONN;
 160         if (!fc->connected)
 161                 goto out;
 162
 163         req = fuse_request_alloc(npages);
 164         err = -ENOMEM;
 165         if (!req) {
 166                 if (for_background)
 167                         wake_up(&fc->blocked_waitq);
 168                 goto out;
 169         }
 170
 171         fuse_req_init_context(req);
 172         req->waiting = 1;
 173         req->background = for_background;
 174         return req;
 175
 176  out:
 177         atomic_dec(&fc->num_waiting);
 178         return ERR_PTR(err);
 179 }
 180
 181 struct fuse_req *fuse_get_req(struct fuse_conn *fc, unsigned npages)
 182 {
 183         return __fuse_get_req(fc, npages, false);
 184 }
 185 EXPORT_SYMBOL_GPL(fuse_get_req);
 186
 187 struct fuse_req *fuse_get_req_for_background(struct fuse_conn *fc,
 188                                              unsigned npages)
 189 {
 190         return __fuse_get_req(fc, npages, true);
 191 }
 192 EXPORT_SYMBOL_GPL(fuse_get_req_for_background);
 193
 194 /*
 195  * Return request in fuse_file->reserved_req.  However that may
 196  * currently be in use.  If that is the case, wait for it to become
 197  * available.
 198  */
 199 static struct fuse_req *get_reserved_req(struct fuse_conn *fc,
 200                                          struct file *file)
 201 {
 202         struct fuse_req *req = NULL;
 203         struct fuse_file *ff = file->private_data;
 204
 205         do {
 206                 wait_event(fc->reserved_req_waitq, ff->reserved_req);
 207                 spin_lock(&fc->lock);
 208                 if (ff->reserved_req) {
 209                         req = ff->reserved_req;
 210                         ff->reserved_req = NULL;
 211                         req->stolen_file = get_file(file);
 212                 }
 213                 spin_unlock(&fc->lock);
 214         } while (!req);
 215
 216         return req;
 217 }
 218
 219 /*
 220  * Put stolen request back into fuse_file->reserved_req
 221  */
 222 static void put_reserved_req(struct fuse_conn *fc, struct fuse_req *req)
 223 {
 224         struct file *file = req->stolen_file;
 225         struct fuse_file *ff = file->private_data;
 226
 227         spin_lock(&fc->lock);
 228         fuse_request_init(req, req->pages, req->page_descs, req->max_pages);
 229         BUG_ON(ff->reserved_req);
 230         ff->reserved_req = req;
 231         wake_up_all(&fc->reserved_req_waitq);
 232         spin_unlock(&fc->lock);
 233         fput(file);
 234 }
 235
 236 /*
 237  * Gets a requests for a file operation, always succeeds
 238  *
 239  * This is used for sending the FLUSH request, which must get to
 240  * userspace, due to POSIX locks which may need to be unlocked.
 241  *
 242  * If allocation fails due to OOM, use the reserved request in
 243  * fuse_file.
 244  *
 245  * This is very unlikely to deadlock accidentally, since the
 246  * filesystem should not have it's own file open.  If deadlock is
 247  * intentional, it can still be broken by "aborting" the filesystem.
 248  */
 249 struct fuse_req *fuse_get_req_nofail_nopages(struct fuse_conn *fc,
 250                                              struct file *file)
 251 {
 252         struct fuse_req *req;
 253
 254         atomic_inc(&fc->num_waiting);
 255         wait_event(fc->blocked_waitq, fc->initialized);
 256         req = fuse_request_alloc(0);
 257         if (!req)
 258                 req = get_reserved_req(fc, file);
 259
 260         fuse_req_init_context(req);
 261         req->waiting = 1;
 262         req->background = 0;
 263         return req;
 264 }
 265
 266 void fuse_put_request(struct fuse_conn *fc, struct fuse_req *req)
 267 {
 268         if (atomic_dec_and_test(&req->count)) {
 269                 if (unlikely(req->background)) {
 270                         /*
 271                          * We get here in the unlikely case that a background
 272                          * request was allocated but not sent
 273                          */
 274                         spin_lock(&fc->lock);
 275                         if (!fc->blocked)
 276                                 wake_up(&fc->blocked_waitq);
 277                         spin_unlock(&fc->lock);
 278                 }
 279
 280                 if (req->waiting)
 281                         atomic_dec(&fc->num_waiting);
 282
 283                 if (req->stolen_file)
 284                         put_reserved_req(fc, req);
 285                 else
 286                         fuse_request_free(req);
 287         }
 288 }
 289 EXPORT_SYMBOL_GPL(fuse_put_request);
 290
 291 static unsigned len_args(unsigned numargs, struct fuse_arg *args)
 292 {
 293         unsigned nbytes = 0;
 294         unsigned i;
 295
 296         for (i = 0; i < numargs; i++)
 297                 nbytes += args[i].size;
 298
 299         return nbytes;
 300 }
 301
 302 static u64 fuse_get_unique(struct fuse_conn *fc)
 303 {
 304         fc->reqctr++;
 305         /* zero is special */
 306         if (fc->reqctr == 0)
 307                 fc->reqctr = 1;
 308
 309         return fc->reqctr;
 310 }
 311
 312 static void queue_request(struct fuse_conn *fc, struct fuse_req *req)
 313 {
 314         req->in.h.len = sizeof(struct fuse_in_header) +
 315                 len_args(req->in.numargs, (struct fuse_arg *) req->in.args);
 316         list_add_tail(&req->list, &fc->pending);
 317         req->state = FUSE_REQ_PENDING;
 318         if (!req->waiting) {
 319                 req->waiting = 1;
 320                 atomic_inc(&fc->num_waiting);
 321         }
 322         wake_up(&fc->waitq);
 323         kill_fasync(&fc->fasync, SIGIO, POLL_IN);
 324 }
 325
 326 void fuse_queue_forget(struct fuse_conn *fc, struct fuse_forget_link *forget,
 327                        u64 nodeid, u64 nlookup)
 328 {
 329         forget->forget_one.nodeid = nodeid;
 330         forget->forget_one.nlookup = nlookup;
 331
 332         spin_lock(&fc->lock);
 333         if (fc->connected) {
 334                 fc->forget_list_tail->next = forget;
 335                 fc->forget_list_tail = forget;
 336                 wake_up(&fc->waitq);
 337                 kill_fasync(&fc->fasync, SIGIO, POLL_IN);
 338         } else {
 339                 kfree(forget);
 340         }
 341         spin_unlock(&fc->lock);
 342 }
 343
 344 static void flush_bg_queue(struct fuse_conn *fc)
 345 {
 346         while (fc->active_background < fc->max_background &&
 347                !list_empty(&fc->bg_queue)) {
 348                 struct fuse_req *req;
 349
 350                 req = list_entry(fc->bg_queue.next, struct fuse_req, list);
 351                 list_del(&req->list);
 352                 fc->active_background++;
 353                 req->in.h.unique = fuse_get_unique(fc);
 354                 queue_request(fc, req);
 355         }
 356 }
 357
 358 /*
 359  * This function is called when a request is finished.  Either a reply
 360  * has arrived or it was aborted (and not yet sent) or some error
 361  * occurred during communication with userspace, or the device file
 362  * was closed.  The requester thread is woken up (if still waiting),
 363  * the 'end' callback is called if given, else the reference to the
 364  * request is released
 365  *
 366  * Called with fc->lock, unlocks it
 367  */
 368 static void request_end(struct fuse_conn *fc, struct fuse_req *req)
 369 __releases(fc->lock)
 370 {
 371         void (*end) (struct fuse_conn *, struct fuse_req *) = req->end;
 372         req->end = NULL;
 373         list_del(&req->list);
 374         list_del(&req->intr_entry);
 375         req->state = FUSE_REQ_FINISHED;
 376         if (req->background) {
 377                 req->background = 0;
 378
 379                 if (fc->num_background == fc->max_background)
 380                         fc->blocked = 0;
 381
 382                 /* Wake up next waiter, if any */
 383                 if (!fc->blocked && waitqueue_active(&fc->blocked_waitq))
 384                         wake_up(&fc->blocked_waitq);
 385
 386                 if (fc->num_background == fc->congestion_threshold &&
 387                     fc->connected && fc->bdi_initialized) {
 388                         clear_bdi_congested(&fc->bdi, BLK_RW_SYNC);
 389                         clear_bdi_congested(&fc->bdi, BLK_RW_ASYNC);
 390                 }
 391                 fc->num_background--;
 392                 fc->active_background--;
 393                 flush_bg_queue(fc);
 394         }
 395         spin_unlock(&fc->lock);
 396         wake_up(&req->waitq);
 397         if (end)
 398                 end(fc, req);
 399         fuse_put_request(fc, req);
 400 }
 401
 402 static void wait_answer_interruptible(struct fuse_conn *fc,
 403                                       struct fuse_req *req)
 404 __releases(fc->lock)
 405 __acquires(fc->lock)
 406 {
 407         if (signal_pending(current))
 408                 return;
 409
 410         spin_unlock(&fc->lock);
 411         wait_event_interruptible(req->waitq, req->state == FUSE_REQ_FINISHED);
 412         spin_lock(&fc->lock);
 413 }
 414
 415 static void queue_interrupt(struct fuse_conn *fc, struct fuse_req *req)
 416 {
 417         list_add_tail(&req->intr_entry, &fc->interrupts);
 418         wake_up(&fc->waitq);
 419         kill_fasync(&fc->fasync, SIGIO, POLL_IN);
 420 }
 421
 422 static void request_wait_answer(struct fuse_conn *fc, struct fuse_req *req)
 423 __releases(fc->lock)
 424 __acquires(fc->lock)
 425 {
 426         if (!fc->no_interrupt) {
 427                 /* Any signal may interrupt this */
 428                 wait_answer_interruptible(fc, req);
 429
 430                 if (req->aborted)
 431                         goto aborted;
 432                 if (req->state == FUSE_REQ_FINISHED)
 433                         return;
 434
 435                 req->interrupted = 1;
 436                 if (req->state == FUSE_REQ_SENT)
 437                         queue_interrupt(fc, req);
 438         }
 439
 440         if (!req->force) {
 441                 sigset_t oldset;
 442
 443                 /* Only fatal signals may interrupt this */
 444                 block_sigs(&oldset);
 445                 wait_answer_interruptible(fc, req);
 446                 restore_sigs(&oldset);
 447
 448                 if (req->aborted)
 449                         goto aborted;
 450                 if (req->state == FUSE_REQ_FINISHED)
 451                         return;
 452
 453                 /* Request is not yet in userspace, bail out */
 454                 if (req->state == FUSE_REQ_PENDING) {
 455                         list_del(&req->list);
 456                         __fuse_put_request(req);
 457                         req->out.h.error = -EINTR;
 458                         return;
 459                 }
 460         }
 461
 462         /*
 463          * Either request is already in userspace, or it was forced.
 464          * Wait it out.
 465          */
 466         spin_unlock(&fc->lock);
 467         wait_event(req->waitq, req->state == FUSE_REQ_FINISHED);
 468         spin_lock(&fc->lock);
 469
 470         if (!req->aborted)
 471                 return;
 472
 473  aborted:
 474         BUG_ON(req->state != FUSE_REQ_FINISHED);
 475         if (req->locked) {
 476                 /* This is uninterruptible sleep, because data is
 477                    being copied to/from the buffers of req.  During
 478                    locked state, there mustn't be any filesystem
 479                    operation (e.g. page fault), since that could lead
 480                    to deadlock */
 481                 spin_unlock(&fc->lock);
 482                 wait_event(req->waitq, !req->locked);
 483                 spin_lock(&fc->lock);
 484         }
 485 }
 486
 487 static void __fuse_request_send(struct fuse_conn *fc, struct fuse_req *req)
 488 {
 489         BUG_ON(req->background);
 490         spin_lock(&fc->lock);
 491         if (!fc->connected)
 492                 req->out.h.error = -ENOTCONN;
 493         else if (fc->conn_error)
 494                 req->out.h.error = -ECONNREFUSED;
 495         else {
 496                 req->in.h.unique = fuse_get_unique(fc);
 497                 queue_request(fc, req);
 498                 /* acquire extra reference, since request is still needed
 499                    after request_end() */
 500                 __fuse_get_request(req);
 501
 502                 request_wait_answer(fc, req);
 503         }
 504         spin_unlock(&fc->lock);
 505 }
 506
 507 void fuse_request_send(struct fuse_conn *fc, struct fuse_req *req)
 508 {
 509         req->isreply = 1;
 510         __fuse_request_send(fc, req);
 511 }
 512 EXPORT_SYMBOL_GPL(fuse_request_send);
 513
 514 ssize_t fuse_simple_request(struct fuse_conn *fc, struct fuse_args *args)
 515 {
 516         struct fuse_req *req;
 517         ssize_t ret;
 518
 519         req = fuse_get_req(fc, 0);
 520         if (IS_ERR(req))
 521                 return PTR_ERR(req);
 522
 523         req->in.h.opcode = args->in.h.opcode;
 524         req->in.h.nodeid = args->in.h.nodeid;
 525         req->in.numargs = args->in.numargs;
 526         memcpy(req->in.args, args->in.args,
 527                args->in.numargs * sizeof(struct fuse_in_arg));
 528         req->out.argvar = args->out.argvar;
 529         req->out.numargs = args->out.numargs;
 530         memcpy(req->out.args, args->out.args,
 531                args->out.numargs * sizeof(struct fuse_arg));
 532         fuse_request_send(fc, req);
 533         ret = req->out.h.error;
 534         if (!ret && args->out.argvar) {
 535                 BUG_ON(args->out.numargs != 1);
 536                 ret = req->out.args[0].size;
 537         }
 538         fuse_put_request(fc, req);
 539
 540         return ret;
 541 }
 542
 543 static void fuse_request_send_nowait_locked(struct fuse_conn *fc,
 544                                             struct fuse_req *req)
 545 {
 546         BUG_ON(!req->background);
 547         fc->num_background++;
 548         if (fc->num_background == fc->max_background)
 549                 fc->blocked = 1;
 550         if (fc->num_background == fc->congestion_threshold &&
 551             fc->bdi_initialized) {
 552                 set_bdi_congested(&fc->bdi, BLK_RW_SYNC);
 553                 set_bdi_congested(&fc->bdi, BLK_RW_ASYNC);
 554         }
 555         list_add_tail(&req->list, &fc->bg_queue);
 556         flush_bg_queue(fc);
 557 }
 558
 559 static void fuse_request_send_nowait(struct fuse_conn *fc, struct fuse_req *req)
 560 {
 561         spin_lock(&fc->lock);
 562         if (fc->connected) {
 563                 fuse_request_send_nowait_locked(fc, req);
 564                 spin_unlock(&fc->lock);
 565         } else {
 566                 req->out.h.error = -ENOTCONN;
 567                 request_end(fc, req);
 568         }
 569 }
 570
 571 void fuse_request_send_background(struct fuse_conn *fc, struct fuse_req *req)
 572 {
 573         req->isreply = 1;
 574         fuse_request_send_nowait(fc, req);
 575 }
 576 EXPORT_SYMBOL_GPL(fuse_request_send_background);
 577
 578 static int fuse_request_send_notify_reply(struct fuse_conn *fc,
 579                                           struct fuse_req *req, u64 unique)
 580 {
 581         int err = -ENODEV;
 582
 583         req->isreply = 0;
 584         req->in.h.unique = unique;
 585         spin_lock(&fc->lock);
 586         if (fc->connected) {
 587                 queue_request(fc, req);
 588                 err = 0;
 589         }
 590         spin_unlock(&fc->lock);
 591
 592         return err;
 593 }
 594
 595 /*
 596  * Called under fc->lock
 597  *
 598  * fc->connected must have been checked previously
 599  */
 600 void fuse_request_send_background_locked(struct fuse_conn *fc,
 601                                          struct fuse_req *req)
 602 {
 603         req->isreply = 1;
 604         fuse_request_send_nowait_locked(fc, req);
 605 }
 606
 607 void fuse_force_forget(struct file *file, u64 nodeid)
 608 {
 609         struct inode *inode = file_inode(file);
 610         struct fuse_conn *fc = get_fuse_conn(inode);
 611         struct fuse_req *req;
 612         struct fuse_forget_in inarg;
 613
 614         memset(&inarg, 0, sizeof(inarg));
 615         inarg.nlookup = 1;
 616         req = fuse_get_req_nofail_nopages(fc, file);
 617         req->in.h.opcode = FUSE_FORGET;
 618         req->in.h.nodeid = nodeid;
 619         req->in.numargs = 1;
 620         req->in.args[0].size = sizeof(inarg);
 621         req->in.args[0].value = &inarg;
 622         req->isreply = 0;
 623         __fuse_request_send(fc, req);
 624         /* ignore errors */
 625         fuse_put_request(fc, req);
 626 }
 627
 628 /*
 629  * Lock the request.  Up to the next unlock_request() there mustn't be
 630  * anything that could cause a page-fault.  If the request was already
 631  * aborted bail out.
 632  */
 633 static int lock_request(struct fuse_conn *fc, struct fuse_req *req)
 634 {
 635         int err = 0;
 636         if (req) {
 637                 spin_lock(&fc->lock);
 638                 if (req->aborted)
 639                         err = -ENOENT;
 640                 else
 641                         req->locked = 1;
 642                 spin_unlock(&fc->lock);
 643         }
 644         return err;
 645 }
 646
 647 /*
 648  * Unlock request.  If it was aborted during being locked, the
 649  * requester thread is currently waiting for it to be unlocked, so
 650  * wake it up.
 651  */
 652 static void unlock_request(struct fuse_conn *fc, struct fuse_req *req)
 653 {
 654         if (req) {
 655                 spin_lock(&fc->lock);
 656                 req->locked = 0;
 657                 if (req->aborted)
 658                         wake_up(&req->waitq);
 659                 spin_unlock(&fc->lock);
 660         }
 661 }
 662
 663 struct fuse_copy_state {
 664         struct fuse_conn *fc;
 665         int write;
 666         struct fuse_req *req;
 667         const struct iovec *iov;
 668         struct pipe_buffer *pipebufs;
 669         struct pipe_buffer *currbuf;
 670         struct pipe_inode_info *pipe;
 671         unsigned long nr_segs;
 672         unsigned long seglen;
 673         unsigned long addr;
 674         struct page *pg;
 675         unsigned len;
 676         unsigned offset;
 677         unsigned move_pages:1;
 678 };
 679
 680 static void fuse_copy_init(struct fuse_copy_state *cs, struct fuse_conn *fc,
 681                            int write,
 682                            const struct iovec *iov, unsigned long nr_segs)
 683 {
 684         memset(cs, 0, sizeof(*cs));
 685         cs->fc = fc;
 686         cs->write = write;
 687         cs->iov = iov;
 688         cs->nr_segs = nr_segs;
 689 }
 690
 691 /* Unmap and put previous page of userspace buffer */
 692 static void fuse_copy_finish(struct fuse_copy_state *cs)
 693 {
 694         if (cs->currbuf) {
 695                 struct pipe_buffer *buf = cs->currbuf;
 696
 697                 if (cs->write)
 698                         buf->len = PAGE_SIZE - cs->len;
 699                 cs->currbuf = NULL;
 700         } else if (cs->pg) {
 701                 if (cs->write) {
 702                         flush_dcache_page(cs->pg);
 703                         set_page_dirty_lock(cs->pg);
 704                 }
 705                 put_page(cs->pg);
 706         }
 707         cs->pg = NULL;
 708 }
 709
 710 /*
 711  * Get another pagefull of userspace buffer, and map it to kernel
 712  * address space, and lock request
 713  */
 714 static int fuse_copy_fill(struct fuse_copy_state *cs)
 715 {
 716         struct page *page;
 717         int err;
 718
 719         unlock_request(cs->fc, cs->req);
 720         fuse_copy_finish(cs);
 721         if (cs->pipebufs) {
 722                 struct pipe_buffer *buf = cs->pipebufs;
 723
 724                 if (!cs->write) {
 725                         err = buf->ops->confirm(cs->pipe, buf);
 726                         if (err)
 727                                 return err;
 728
 729                         BUG_ON(!cs->nr_segs);
 730                         cs->currbuf = buf;
 731                         cs->pg = buf->page;
 732                         cs->offset = buf->offset;
 733                         cs->len = buf->len;
 734                         cs->pipebufs++;
 735                         cs->nr_segs--;
 736                 } else {
 737                         if (cs->nr_segs == cs->pipe->buffers)
 738                                 return -EIO;
 739
 740                         page = alloc_page(GFP_HIGHUSER);
 741                         if (!page)
 742                                 return -ENOMEM;
 743
 744                         buf->page = page;
 745                         buf->offset = 0;
 746                         buf->len = 0;
 747
 748                         cs->currbuf = buf;
 749                         cs->pg = page;
 750                         cs->offset = 0;
 751                         cs->len = PAGE_SIZE;
 752                         cs->pipebufs++;
 753                         cs->nr_segs++;
 754                 }
 755         } else {
 756                 if (!cs->seglen) {
 757                         BUG_ON(!cs->nr_segs);
 758                         cs->seglen = cs->iov[0].iov_len;
 759                         cs->addr = (unsigned long) cs->iov[0].iov_base;
 760                         cs->iov++;
 761                         cs->nr_segs--;
 762                 }
 763                 err = get_user_pages_fast(cs->addr, 1, cs->write, &page);
 764                 if (err < 0)
 765                         return err;
 766                 BUG_ON(err != 1);
 767                 cs->pg = page;
 768                 cs->offset = cs->addr % PAGE_SIZE;
 769                 cs->len = min(PAGE_SIZE - cs->offset, cs->seglen);
 770                 cs->seglen -= cs->len;
 771                 cs->addr += cs->len;
 772         }
 773
 774         return lock_request(cs->fc, cs->req);
 775 }
 776
 777 /* Do as much copy to/from userspace buffer as we can */
 778 static int fuse_copy_do(struct fuse_copy_state *cs, void **val, unsigned *size)
 779 {
 780         unsigned ncpy = min(*size, cs->len);
 781         if (val) {
 782                 void *pgaddr = kmap_atomic(cs->pg);
 783                 void *buf = pgaddr + cs->offset;
 784
 785                 if (cs->write)
 786                         memcpy(buf, *val, ncpy);
 787                 else
 788                         memcpy(*val, buf, ncpy);
 789
 790                 kunmap_atomic(pgaddr);
 791                 *val += ncpy;
 792         }
 793         *size -= ncpy;
 794         cs->len -= ncpy;
 795         cs->offset += ncpy;
 796         return ncpy;
 797 }
 798
 799 static int fuse_check_page(struct page *page)
 800 {
 801         if (page_mapcount(page) ||
 802             page->mapping != NULL ||
 803             page_count(page) != 1 ||
 804             (page->flags & PAGE_FLAGS_CHECK_AT_PREP &
 805              ~(1 << PG_locked |
 806                1 << PG_referenced |
 807                1 << PG_uptodate |
 808                1 << PG_lru |
 809                1 << PG_active |
 810                1 << PG_reclaim))) {
 811                 printk(KERN_WARNING "fuse: trying to steal weird page\n");
 812                 printk(KERN_WARNING "  page=%p index=%li flags=%08lx, count=%i, mapcount=%i, mapping=%p\n", page, page->index, page->flags, page_count(page), page_mapcount(page), page->mapping);
 813                 return 1;
 814         }
 815         return 0;
 816 }
 817
 818 static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)
 819 {
 820         int err;
 821         struct page *oldpage = *pagep;
 822         struct page *newpage;
 823         struct pipe_buffer *buf = cs->pipebufs;
 824
 825         unlock_request(cs->fc, cs->req);
 826         fuse_copy_finish(cs);
 827
 828         err = buf->ops->confirm(cs->pipe, buf);
 829         if (err)
 830                 return err;
 831
 832         BUG_ON(!cs->nr_segs);
 833         cs->currbuf = buf;
 834         cs->len = buf->len;
 835         cs->pipebufs++;
 836         cs->nr_segs--;
 837
 838         if (cs->len != PAGE_SIZE)
 839                 goto out_fallback;
 840
 841         if (buf->ops->steal(cs->pipe, buf) != 0)
 842                 goto out_fallback;
 843
 844         newpage = buf->page;
 845
 846         if (WARN_ON(!PageUptodate(newpage)))
 847                 return -EIO;
 848
 849         ClearPageMappedToDisk(newpage);
 850
 851         if (fuse_check_page(newpage) != 0)
 852                 goto out_fallback_unlock;
 853
 854         /*
 855          * This is a new and locked page, it shouldn't be mapped or
 856          * have any special flags on it
 857          */
 858         if (WARN_ON(page_mapped(oldpage)))
 859                 goto out_fallback_unlock;
 860         if (WARN_ON(page_has_private(oldpage)))
 861                 goto out_fallback_unlock;
 862         if (WARN_ON(PageDirty(oldpage) || PageWriteback(oldpage)))
 863                 goto out_fallback_unlock;
 864         if (WARN_ON(PageMlocked(oldpage)))
 865                 goto out_fallback_unlock;
 866
 867         err = replace_page_cache_page(oldpage, newpage, GFP_KERNEL);
 868         if (err) {
 869                 unlock_page(newpage);
 870                 return err;
 871         }
 872
 873         page_cache_get(newpage);
 874
 875         if (!(buf->flags & PIPE_BUF_FLAG_LRU))
 876                 lru_cache_add_file(newpage);
 877
 878         err = 0;
 879         spin_lock(&cs->fc->lock);
 880         if (cs->req->aborted)
 881                 err = -ENOENT;
 882         else
 883                 *pagep = newpage;
 884         spin_unlock(&cs->fc->lock);
 885
 886         if (err) {
 887                 unlock_page(newpage);
 888                 page_cache_release(newpage);
 889                 return err;
 890         }
 891
 892         unlock_page(oldpage);
 893         page_cache_release(oldpage);
 894         cs->len = 0;
 895
 896         return 0;
 897
 898 out_fallback_unlock:
 899         unlock_page(newpage);
 900 out_fallback:
 901         cs->pg = buf->page;
 902         cs->offset = buf->offset;
 903
 904         err = lock_request(cs->fc, cs->req);
 905         if (err)
 906                 return err;
 907
 908         return 1;
 909 }
 910
 911 static int fuse_ref_page(struct fuse_copy_state *cs, struct page *page,
 912                          unsigned offset, unsigned count)
 913 {
 914         struct pipe_buffer *buf;
 915
 916         if (cs->nr_segs == cs->pipe->buffers)
 917                 return -EIO;
 918
 919         unlock_request(cs->fc, cs->req);
 920         fuse_copy_finish(cs);
 921
 922         buf = cs->pipebufs;
 923         page_cache_get(page);
 924         buf->page = page;
 925         buf->offset = offset;
 926         buf->len = count;
 927
 928         cs->pipebufs++;
 929         cs->nr_segs++;
 930         cs->len = 0;
 931
 932         return 0;
 933 }
 934
 935 /*
 936  * Copy a page in the request to/from the userspace buffer.  Must be
 937  * done atomically
 938  */
 939 static int fuse_copy_page(struct fuse_copy_state *cs, struct page **pagep,
 940                           unsigned offset, unsigned count, int zeroing)
 941 {
 942         int err;
 943         struct page *page = *pagep;
 944
 945         if (page && zeroing && count < PAGE_SIZE)
 946                 clear_highpage(page);
 947
 948         while (count) {
 949                 if (cs->write && cs->pipebufs && page) {
 950                         return fuse_ref_page(cs, page, offset, count);
 951                 } else if (!cs->len) {
 952                         if (cs->move_pages && page &&
 953                             offset == 0 && count == PAGE_SIZE) {
 954                                 err = fuse_try_move_page(cs, pagep);
 955                                 if (err <= 0)
 956                                         return err;
 957                         } else {
 958                                 err = fuse_copy_fill(cs);
 959                                 if (err)
 960                                         return err;
 961                         }
 962                 }
 963                 if (page) {
 964                         void *mapaddr = kmap_atomic(page);
 965                         void *buf = mapaddr + offset;
 966                         offset += fuse_copy_do(cs, &buf, &count);
 967                         kunmap_atomic(mapaddr);
 968                 } else
 969                         offset += fuse_copy_do(cs, NULL, &count);
 970         }
 971         if (page && !cs->write)
 972                 flush_dcache_page(page);
 973         return 0;
 974 }
 975
 976 /* Copy pages in the request to/from userspace buffer */
 977 static int fuse_copy_pages(struct fuse_copy_state *cs, unsigned nbytes,
 978                            int zeroing)
 979 {
 980         unsigned i;
 981         struct fuse_req *req = cs->req;
 982
 983         for (i = 0; i < req->num_pages && (nbytes || zeroing); i++) {
 984                 int err;
 985                 unsigned offset = req->page_descs[i].offset;
 986                 unsigned count = min(nbytes, req->page_descs[i].length);
 987
 988                 err = fuse_copy_page(cs, &req->pages[i], offset, count,
 989                                      zeroing);
 990                 if (err)
 991                         return err;
 992
 993                 nbytes -= count;
 994         }
 995         return 0;
 996 }
 997
 998 /* Copy a single argument in the request to/from userspace buffer */
 999 static int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size)
1000 {
1001         while (size) {
1002                 if (!cs->len) {
1003                         int err = fuse_copy_fill(cs);
1004                         if (err)
1005                                 return err;
1006                 }
1007                 fuse_copy_do(cs, &val, &size);
1008         }
1009         return 0;
1010 }
1011
1012 /* Copy request arguments to/from userspace buffer */
1013 static int fuse_copy_args(struct fuse_copy_state *cs, unsigned numargs,
1014                           unsigned argpages, struct fuse_arg *args,
1015                           int zeroing)
1016 {
1017         int err = 0;
1018         unsigned i;
1019
1020         for (i = 0; !err && i < numargs; i++)  {
1021                 struct fuse_arg *arg = &args[i];
1022                 if (i == numargs - 1 && argpages)
1023                         err = fuse_copy_pages(cs, arg->size, zeroing);
1024                 else
1025                         err = fuse_copy_one(cs, arg->value, arg->size);
1026         }
1027         return err;
1028 }
1029
1030 static int forget_pending(struct fuse_conn *fc)
1031 {
1032         return fc->forget_list_head.next != NULL;
1033 }
1034
1035 static int request_pending(struct fuse_conn *fc)
1036 {
1037         return !list_empty(&fc->pending) || !list_empty(&fc->interrupts) ||
1038                 forget_pending(fc);
1039 }
1040
1041 /* Wait until a request is available on the pending list */
1042 static void request_wait(struct fuse_conn *fc)
1043 __releases(fc->lock)
1044 __acquires(fc->lock)
1045 {
1046         DECLARE_WAITQUEUE(wait, current);
1047
1048         add_wait_queue_exclusive(&fc->waitq, &wait);
1049         while (fc->connected && !request_pending(fc)) {
1050                 set_current_state(TASK_INTERRUPTIBLE);
1051                 if (signal_pending(current))
1052                         break;
1053
1054                 spin_unlock(&fc->lock);
1055                 schedule();
1056                 spin_lock(&fc->lock);
1057         }
1058         set_current_state(TASK_RUNNING);
1059         remove_wait_queue(&fc->waitq, &wait);
1060 }
1061
1062 /*
1063  * Transfer an interrupt request to userspace
1064  *
1065  * Unlike other requests this is assembled on demand, without a need
1066  * to allocate a separate fuse_req structure.
1067  *
1068  * Called with fc->lock held, releases it
1069  */
1070 static int fuse_read_interrupt(struct fuse_conn *fc, struct fuse_copy_state *cs,
1071                                size_t nbytes, struct fuse_req *req)
1072 __releases(fc->lock)
1073 {
1074         struct fuse_in_header ih;
1075         struct fuse_interrupt_in arg;
1076         unsigned reqsize = sizeof(ih) + sizeof(arg);
1077         int err;
1078
1079         list_del_init(&req->intr_entry);
1080         req->intr_unique = fuse_get_unique(fc);
1081         memset(&ih, 0, sizeof(ih));
1082         memset(&arg, 0, sizeof(arg));
1083         ih.len = reqsize;
1084         ih.opcode = FUSE_INTERRUPT;
1085         ih.unique = req->intr_unique;
1086         arg.unique = req->in.h.unique;
1087
1088         spin_unlock(&fc->lock);
1089         if (nbytes < reqsize)
1090                 return -EINVAL;
1091
1092         err = fuse_copy_one(cs, &ih, sizeof(ih));
1093         if (!err)
1094                 err = fuse_copy_one(cs, &arg, sizeof(arg));
1095         fuse_copy_finish(cs);
1096
1097         return err ? err : reqsize;
1098 }
1099
1100 static struct fuse_forget_link *dequeue_forget(struct fuse_conn *fc,
1101                                                unsigned max,
1102                                                unsigned *countp)
1103 {
1104         struct fuse_forget_link *head = fc->forget_list_head.next;
1105         struct fuse_forget_link **newhead = &head;
1106         unsigned count;
1107
1108         for (count = 0; *newhead != NULL && count < max; count++)
1109                 newhead = &(*newhead)->next;
1110
1111         fc->forget_list_head.next = *newhead;
1112         *newhead = NULL;
1113         if (fc->forget_list_head.next == NULL)
1114                 fc->forget_list_tail = &fc->forget_list_head;
1115
1116         if (countp != NULL)
1117                 *countp = count;
1118
1119         return head;
1120 }
1121
1122 static int fuse_read_single_forget(struct fuse_conn *fc,
1123                                    struct fuse_copy_state *cs,
1124                                    size_t nbytes)
1125 __releases(fc->lock)
1126 {
1127         int err;
1128         struct fuse_forget_link *forget = dequeue_forget(fc, 1, NULL);
1129         struct fuse_forget_in arg = {
1130                 .nlookup = forget->forget_one.nlookup,
1131         };
1132         struct fuse_in_header ih = {
1133                 .opcode = FUSE_FORGET,
1134                 .nodeid = forget->forget_one.nodeid,
1135                 .unique = fuse_get_unique(fc),
1136                 .len = sizeof(ih) + sizeof(arg),
1137         };
1138
1139         spin_unlock(&fc->lock);
1140         kfree(forget);
1141         if (nbytes < ih.len)
1142                 return -EINVAL;
1143
1144         err = fuse_copy_one(cs, &ih, sizeof(ih));
1145         if (!err)
1146                 err = fuse_copy_one(cs, &arg, sizeof(arg));
1147         fuse_copy_finish(cs);
1148
1149         if (err)
1150                 return err;
1151
1152         return ih.len;
1153 }
1154
1155 static int fuse_read_batch_forget(struct fuse_conn *fc,
1156                                    struct fuse_copy_state *cs, size_t nbytes)
1157 __releases(fc->lock)
1158 {
1159         int err;
1160         unsigned max_forgets;
1161         unsigned count;
1162         struct fuse_forget_link *head;
1163         struct fuse_batch_forget_in arg = { .count = 0 };
1164         struct fuse_in_header ih = {
1165                 .opcode = FUSE_BATCH_FORGET,
1166                 .unique = fuse_get_unique(fc),
1167                 .len = sizeof(ih) + sizeof(arg),
1168         };
1169
1170         if (nbytes < ih.len) {
1171                 spin_unlock(&fc->lock);
1172                 return -EINVAL;
1173         }
1174
1175         max_forgets = (nbytes - ih.len) / sizeof(struct fuse_forget_one);
1176         head = dequeue_forget(fc, max_forgets, &count);
1177         spin_unlock(&fc->lock);
1178
1179         arg.count = count;
1180         ih.len += count * sizeof(struct fuse_forget_one);
1181         err = fuse_copy_one(cs, &ih, sizeof(ih));
1182         if (!err)
1183                 err = fuse_copy_one(cs, &arg, sizeof(arg));
1184
1185         while (head) {
1186                 struct fuse_forget_link *forget = head;
1187
1188                 if (!err) {
1189                         err = fuse_copy_one(cs, &forget->forget_one,
1190                                             sizeof(forget->forget_one));
1191                 }
1192                 head = forget->next;
1193                 kfree(forget);
1194         }
1195
1196         fuse_copy_finish(cs);
1197
1198         if (err)
1199                 return err;
1200
1201         return ih.len;
1202 }
1203
1204 static int fuse_read_forget(struct fuse_conn *fc, struct fuse_copy_state *cs,
1205                             size_t nbytes)
1206 __releases(fc->lock)
1207 {
1208         if (fc->minor < 16 || fc->forget_list_head.next->next == NULL)
1209                 return fuse_read_single_forget(fc, cs, nbytes);
1210         else
1211                 return fuse_read_batch_forget(fc, cs, nbytes);
1212 }
1213
1214 /*
1215  * Read a single request into the userspace filesystem's buffer.  This
1216  * function waits until a request is available, then removes it from
1217  * the pending list and copies request data to userspace buffer.  If
1218  * no reply is needed (FORGET) or request has been aborted or there
1219  * was an error during the copying then it's finished by calling
1220  * request_end().  Otherwise add it to the processing list, and set
1221  * the 'sent' flag.
1222  */
1223 static ssize_t fuse_dev_do_read(struct fuse_conn *fc, struct file *file,
1224                                 struct fuse_copy_state *cs, size_t nbytes)
1225 {
1226         int err;
1227         struct fuse_req *req;
1228         struct fuse_in *in;
1229         unsigned reqsize;
1230
1231  restart:
1232         spin_lock(&fc->lock);
1233         err = -EAGAIN;
1234         if ((file->f_flags & O_NONBLOCK) && fc->connected &&
1235             !request_pending(fc))
1236                 goto err_unlock;
1237
1238         request_wait(fc);
1239         err = -ENODEV;
1240         if (!fc->connected)
1241                 goto err_unlock;
1242         err = -ERESTARTSYS;
1243         if (!request_pending(fc))
1244                 goto err_unlock;
1245
1246         if (!list_empty(&fc->interrupts)) {
1247                 req = list_entry(fc->interrupts.next, struct fuse_req,
1248                                  intr_entry);
1249                 return fuse_read_interrupt(fc, cs, nbytes, req);
1250         }
1251
1252         if (forget_pending(fc)) {
1253                 if (list_empty(&fc->pending) || fc->forget_batch-- > 0)
1254                         return fuse_read_forget(fc, cs, nbytes);
1255
1256                 if (fc->forget_batch <= -8)
1257                         fc->forget_batch = 16;
1258         }
1259
1260         req = list_entry(fc->pending.next, struct fuse_req, list);
1261         req->state = FUSE_REQ_READING;
1262         list_move(&req->list, &fc->io);
1263
1264         in = &req->in;
1265         reqsize = in->h.len;
1266         /* If request is too large, reply with an error and restart the read */
1267         if (nbytes < reqsize) {
1268                 req->out.h.error = -EIO;
1269                 /* SETXATTR is special, since it may contain too large data */
1270                 if (in->h.opcode == FUSE_SETXATTR)
1271                         req->out.h.error = -E2BIG;
1272                 request_end(fc, req);
1273                 goto restart;
1274         }
1275         spin_unlock(&fc->lock);
1276         cs->req = req;
1277         err = fuse_copy_one(cs, &in->h, sizeof(in->h));
1278         if (!err)
1279                 err = fuse_copy_args(cs, in->numargs, in->argpages,
1280                                      (struct fuse_arg *) in->args, 0);
1281         fuse_copy_finish(cs);
1282         spin_lock(&fc->lock);
1283         req->locked = 0;
1284         if (req->aborted) {
1285                 request_end(fc, req);
1286                 return -ENODEV;
1287         }
1288         if (err) {
1289                 req->out.h.error = -EIO;
1290                 request_end(fc, req);
1291                 return err;
1292         }
1293         if (!req->isreply)
1294                 request_end(fc, req);
1295         else {
1296                 req->state = FUSE_REQ_SENT;
1297                 list_move_tail(&req->list, &fc->processing);
1298                 if (req->interrupted)
1299                         queue_interrupt(fc, req);
1300                 spin_unlock(&fc->lock);
1301         }
1302         return reqsize;
1303
1304  err_unlock:
1305         spin_unlock(&fc->lock);
1306         return err;
1307 }
1308
1309 static ssize_t fuse_dev_read(struct kiocb *iocb, const struct iovec *iov,
1310                               unsigned long nr_segs, loff_t pos)
1311 {
1312         struct fuse_copy_state cs;
1313         struct file *file = iocb->ki_filp;
1314         struct fuse_conn *fc = fuse_get_conn(file);
1315         if (!fc)
1316                 return -EPERM;
1317
1318         fuse_copy_init(&cs, fc, 1, iov, nr_segs);
1319
1320         return fuse_dev_do_read(fc, file, &cs, iov_length(iov, nr_segs));
1321 }
1322
1323 static ssize_t fuse_dev_splice_read(struct file *in, loff_t *ppos,
1324                                     struct pipe_inode_info *pipe,
1325                                     size_t len, unsigned int flags)
1326 {
1327         int ret;
1328         int page_nr = 0;
1329         int do_wakeup = 0;
1330         struct pipe_buffer *bufs;
1331         struct fuse_copy_state cs;
1332         struct fuse_conn *fc = fuse_get_conn(in);
1333         if (!fc)
1334                 return -EPERM;
1335
1336         bufs = kmalloc(pipe->buffers * sizeof(struct pipe_buffer), GFP_KERNEL);
1337         if (!bufs)
1338                 return -ENOMEM;
1339
1340         fuse_copy_init(&cs, fc, 1, NULL, 0);
1341         cs.pipebufs = bufs;
1342         cs.pipe = pipe;
1343         ret = fuse_dev_do_read(fc, in, &cs, len);
1344         if (ret < 0)
1345                 goto out;
1346
1347         ret = 0;
1348         pipe_lock(pipe);
1349
1350         if (!pipe->readers) {
1351                 send_sig(SIGPIPE, current, 0);
1352                 if (!ret)
1353                         ret = -EPIPE;
1354                 goto out_unlock;
1355         }
1356
1357         if (pipe->nrbufs + cs.nr_segs > pipe->buffers) {
1358                 ret = -EIO;
1359                 goto out_unlock;
1360         }
1361
1362         while (page_nr < cs.nr_segs) {
1363                 int newbuf = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);
1364                 struct pipe_buffer *buf = pipe->bufs + newbuf;
1365
1366                 buf->page = bufs[page_nr].page;
1367                 buf->offset = bufs[page_nr].offset;
1368                 buf->len = bufs[page_nr].len;
1369                 /*
1370                  * Need to be careful about this.  Having buf->ops in module
1371                  * code can Oops if the buffer persists after module unload.
1372                  */
1373                 buf->ops = &nosteal_pipe_buf_ops;
1374
1375                 pipe->nrbufs++;
1376                 page_nr++;
1377                 ret += buf->len;
1378
1379                 if (pipe->files)
1380                         do_wakeup = 1;
1381         }
1382
1383 out_unlock:
1384         pipe_unlock(pipe);
1385
1386         if (do_wakeup) {
1387                 smp_mb();
1388                 if (waitqueue_active(&pipe->wait))
1389                         wake_up_interruptible(&pipe->wait);
1390                 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
1391         }
1392
1393 out:
1394         for (; page_nr < cs.nr_segs; page_nr++)
1395                 page_cache_release(bufs[page_nr].page);
1396
1397         kfree(bufs);
1398         return ret;
1399 }
1400
1401 static int fuse_notify_poll(struct fuse_conn *fc, unsigned int size,
1402                             struct fuse_copy_state *cs)
1403 {
1404         struct fuse_notify_poll_wakeup_out outarg;
1405         int err = -EINVAL;
1406
1407         if (size != sizeof(outarg))
1408                 goto err;
1409
1410         err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1411         if (err)
1412                 goto err;
1413
1414         fuse_copy_finish(cs);
1415         return fuse_notify_poll_wakeup(fc, &outarg);
1416
1417 err:
1418         fuse_copy_finish(cs);
1419         return err;
1420 }
1421
1422 static int fuse_notify_inval_inode(struct fuse_conn *fc, unsigned int size,
1423                                    struct fuse_copy_state *cs)
1424 {
1425         struct fuse_notify_inval_inode_out outarg;
1426         int err = -EINVAL;
1427
1428         if (size != sizeof(outarg))
1429                 goto err;
1430
1431         err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1432         if (err)
1433                 goto err;
1434         fuse_copy_finish(cs);
1435
1436         down_read(&fc->killsb);
1437         err = -ENOENT;
1438         if (fc->sb) {
1439                 err = fuse_reverse_inval_inode(fc->sb, outarg.ino,
1440                                                outarg.off, outarg.len);
1441         }
1442         up_read(&fc->killsb);
1443         return err;
1444
1445 err:
1446         fuse_copy_finish(cs);
1447         return err;
1448 }
1449
1450 static int fuse_notify_inval_entry(struct fuse_conn *fc, unsigned int size,
1451                                    struct fuse_copy_state *cs)
1452 {
1453         struct fuse_notify_inval_entry_out outarg;
1454         int err = -ENOMEM;
1455         char *buf;
1456         struct qstr name;
1457
1458         buf = kzalloc(FUSE_NAME_MAX + 1, GFP_KERNEL);
1459         if (!buf)
1460                 goto err;
1461
1462         err = -EINVAL;
1463         if (size < sizeof(outarg))
1464                 goto err;
1465
1466         err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1467         if (err)
1468                 goto err;
1469
1470         err = -ENAMETOOLONG;
1471         if (outarg.namelen > FUSE_NAME_MAX)
1472                 goto err;
1473
1474         err = -EINVAL;
1475         if (size != sizeof(outarg) + outarg.namelen + 1)
1476                 goto err;
1477
1478         name.name = buf;
1479         name.len = outarg.namelen;
1480         err = fuse_copy_one(cs, buf, outarg.namelen + 1);
1481         if (err)
1482                 goto err;
1483         fuse_copy_finish(cs);
1484         buf[outarg.namelen] = 0;
1485         name.hash = full_name_hash(name.name, name.len);
1486
1487         down_read(&fc->killsb);
1488         err = -ENOENT;
1489         if (fc->sb)
1490                 err = fuse_reverse_inval_entry(fc->sb, outarg.parent, 0, &name);
1491         up_read(&fc->killsb);
1492         kfree(buf);
1493         return err;
1494
1495 err:
1496         kfree(buf);
1497         fuse_copy_finish(cs);
1498         return err;
1499 }
1500
1501 static int fuse_notify_delete(struct fuse_conn *fc, unsigned int size,
1502                               struct fuse_copy_state *cs)
1503 {
1504         struct fuse_notify_delete_out outarg;
1505         int err = -ENOMEM;
1506         char *buf;
1507         struct qstr name;
1508
1509         buf = kzalloc(FUSE_NAME_MAX + 1, GFP_KERNEL);
1510         if (!buf)
1511                 goto err;
1512
1513         err = -EINVAL;
1514         if (size < sizeof(outarg))
1515                 goto err;
1516
1517         err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1518         if (err)
1519                 goto err;
1520
1521         err = -ENAMETOOLONG;
1522         if (outarg.namelen > FUSE_NAME_MAX)
1523                 goto err;
1524
1525         err = -EINVAL;
1526         if (size != sizeof(outarg) + outarg.namelen + 1)
1527                 goto err;
1528
1529         name.name = buf;
1530         name.len = outarg.namelen;
1531         err = fuse_copy_one(cs, buf, outarg.namelen + 1);
1532         if (err)
1533                 goto err;
1534         fuse_copy_finish(cs);
1535         buf[outarg.namelen] = 0;
1536         name.hash = full_name_hash(name.name, name.len);
1537
1538         down_read(&fc->killsb);
1539         err = -ENOENT;
1540         if (fc->sb)
1541                 err = fuse_reverse_inval_entry(fc->sb, outarg.parent,
1542                                                outarg.child, &name);
1543         up_read(&fc->killsb);
1544         kfree(buf);
1545         return err;
1546
1547 err:
1548         kfree(buf);
1549         fuse_copy_finish(cs);
1550         return err;
1551 }
1552
1553 static int fuse_notify_store(struct fuse_conn *fc, unsigned int size,
1554                              struct fuse_copy_state *cs)
1555 {
1556         struct fuse_notify_store_out outarg;
1557         struct inode *inode;
1558         struct address_space *mapping;
1559         u64 nodeid;
1560         int err;
1561         pgoff_t index;
1562         unsigned int offset;
1563         unsigned int num;
1564         loff_t file_size;
1565         loff_t end;
1566
1567         err = -EINVAL;
1568         if (size < sizeof(outarg))
1569                 goto out_finish;
1570
1571         err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1572         if (err)
1573                 goto out_finish;
1574
1575         err = -EINVAL;
1576         if (size - sizeof(outarg) != outarg.size)
1577                 goto out_finish;
1578
1579         nodeid = outarg.nodeid;
1580
1581         down_read(&fc->killsb);
1582
1583         err = -ENOENT;
1584         if (!fc->sb)
1585                 goto out_up_killsb;
1586
1587         inode = ilookup5(fc->sb, nodeid, fuse_inode_eq, &nodeid);
1588         if (!inode)
1589                 goto out_up_killsb;
1590
1591         mapping = inode->i_mapping;
1592         index = outarg.offset >> PAGE_CACHE_SHIFT;
1593         offset = outarg.offset & ~PAGE_CACHE_MASK;
1594         file_size = i_size_read(inode);
1595         end = outarg.offset + outarg.size;
1596         if (end > file_size) {
1597                 file_size = end;
1598                 fuse_write_update_size(inode, file_size);
1599         }
1600
1601         num = outarg.size;
1602         while (num) {
1603                 struct page *page;
1604                 unsigned int this_num;
1605
1606                 err = -ENOMEM;
1607                 page = find_or_create_page(mapping, index,
1608                                            mapping_gfp_mask(mapping));
1609                 if (!page)
1610                         goto out_iput;
1611
1612                 this_num = min_t(unsigned, num, PAGE_CACHE_SIZE - offset);
1613                 err = fuse_copy_page(cs, &page, offset, this_num, 0);
1614                 if (!err && offset == 0 &&
1615                     (this_num == PAGE_CACHE_SIZE || file_size == end))
1616                         SetPageUptodate(page);
1617                 unlock_page(page);
1618                 page_cache_release(page);
1619
1620                 if (err)
1621                         goto out_iput;
1622
1623                 num -= this_num;
1624                 offset = 0;
1625                 index++;
1626         }
1627
1628         err = 0;
1629
1630 out_iput:
1631         iput(inode);
1632 out_up_killsb:
1633         up_read(&fc->killsb);
1634 out_finish:
1635         fuse_copy_finish(cs);
1636         return err;
1637 }
1638
1639 static void fuse_retrieve_end(struct fuse_conn *fc, struct fuse_req *req)
1640 {
1641         release_pages(req->pages, req->num_pages, false);
1642 }
1643
1644 static int fuse_retrieve(struct fuse_conn *fc, struct inode *inode,
1645                          struct fuse_notify_retrieve_out *outarg)
1646 {
1647         int err;
1648         struct address_space *mapping = inode->i_mapping;
1649         struct fuse_req *req;
1650         pgoff_t index;
1651         loff_t file_size;
1652         unsigned int num;
1653         unsigned int offset;
1654         size_t total_len = 0;
1655         int num_pages;
1656
1657         offset = outarg->offset & ~PAGE_CACHE_MASK;
1658         file_size = i_size_read(inode);
1659
1660         num = outarg->size;
1661         if (outarg->offset > file_size)
1662                 num = 0;
1663         else if (outarg->offset + num > file_size)
1664                 num = file_size - outarg->offset;
1665
1666         num_pages = (num + offset + PAGE_SIZE - 1) >> PAGE_SHIFT;
1667         num_pages = min(num_pages, FUSE_MAX_PAGES_PER_REQ);
1668
1669         req = fuse_get_req(fc, num_pages);
1670         if (IS_ERR(req))
1671                 return PTR_ERR(req);
1672
1673         req->in.h.opcode = FUSE_NOTIFY_REPLY;
1674         req->in.h.nodeid = outarg->nodeid;
1675         req->in.numargs = 2;
1676         req->in.argpages = 1;
1677         req->page_descs[0].offset = offset;
1678         req->end = fuse_retrieve_end;
1679
1680         index = outarg->offset >> PAGE_CACHE_SHIFT;
1681
1682         while (num && req->num_pages < num_pages) {
1683                 struct page *page;
1684                 unsigned int this_num;
1685
1686                 page = find_get_page(mapping, index);
1687                 if (!page)
1688                         break;
1689
1690                 this_num = min_t(unsigned, num, PAGE_CACHE_SIZE - offset);
1691                 req->pages[req->num_pages] = page;
1692                 req->page_descs[req->num_pages].length = this_num;
1693                 req->num_pages++;
1694
1695                 offset = 0;
1696                 num -= this_num;
1697                 total_len += this_num;
1698                 index++;
1699         }
1700         req->misc.retrieve_in.offset = outarg->offset;
1701         req->misc.retrieve_in.size = total_len;
1702         req->in.args[0].size = sizeof(req->misc.retrieve_in);
1703         req->in.args[0].value = &req->misc.retrieve_in;
1704         req->in.args[1].size = total_len;
1705
1706         err = fuse_request_send_notify_reply(fc, req, outarg->notify_unique);
1707         if (err)
1708                 fuse_retrieve_end(fc, req);
1709
1710         return err;
1711 }
1712
1713 static int fuse_notify_retrieve(struct fuse_conn *fc, unsigned int size,
1714                                 struct fuse_copy_state *cs)
1715 {
1716         struct fuse_notify_retrieve_out outarg;
1717         struct inode *inode;
1718         int err;
1719
1720         err = -EINVAL;
1721         if (size != sizeof(outarg))
1722                 goto copy_finish;
1723
1724         err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1725         if (err)
1726                 goto copy_finish;
1727
1728         fuse_copy_finish(cs);
1729
1730         down_read(&fc->killsb);
1731         err = -ENOENT;
1732         if (fc->sb) {
1733                 u64 nodeid = outarg.nodeid;
1734
1735                 inode = ilookup5(fc->sb, nodeid, fuse_inode_eq, &nodeid);
1736                 if (inode) {
1737                         err = fuse_retrieve(fc, inode, &outarg);
1738                         iput(inode);
1739                 }
1740         }
1741         up_read(&fc->killsb);
1742
1743         return err;
1744
1745 copy_finish:
1746         fuse_copy_finish(cs);
1747         return err;
1748 }
1749
1750 static int fuse_notify(struct fuse_conn *fc, enum fuse_notify_code code,
1751                        unsigned int size, struct fuse_copy_state *cs)
1752 {
1753         switch (code) {
1754         case FUSE_NOTIFY_POLL:
1755                 return fuse_notify_poll(fc, size, cs);
1756
1757         case FUSE_NOTIFY_INVAL_INODE:
1758                 return fuse_notify_inval_inode(fc, size, cs);
1759
1760         case FUSE_NOTIFY_INVAL_ENTRY:
1761                 return fuse_notify_inval_entry(fc, size, cs);
1762
1763         case FUSE_NOTIFY_STORE:
1764                 return fuse_notify_store(fc, size, cs);
1765
1766         case FUSE_NOTIFY_RETRIEVE:
1767                 return fuse_notify_retrieve(fc, size, cs);
1768
1769         case FUSE_NOTIFY_DELETE:
1770                 return fuse_notify_delete(fc, size, cs);
1771
1772         default:
1773                 fuse_copy_finish(cs);
1774                 return -EINVAL;
1775         }
1776 }
1777
1778 /* Look up request on processing list by unique ID */
1779 static struct fuse_req *request_find(struct fuse_conn *fc, u64 unique)
1780 {
1781         struct fuse_req *req;
1782
1783         list_for_each_entry(req, &fc->processing, list) {
1784                 if (req->in.h.unique == unique || req->intr_unique == unique)
1785                         return req;
1786         }
1787         return NULL;
1788 }
1789
1790 static int copy_out_args(struct fuse_copy_state *cs, struct fuse_out *out,
1791                          unsigned nbytes)
1792 {
1793         unsigned reqsize = sizeof(struct fuse_out_header);
1794
1795         if (out->h.error)
1796                 return nbytes != reqsize ? -EINVAL : 0;
1797
1798         reqsize += len_args(out->numargs, out->args);
1799
1800         if (reqsize < nbytes || (reqsize > nbytes && !out->argvar))
1801                 return -EINVAL;
1802         else if (reqsize > nbytes) {
1803                 struct fuse_arg *lastarg = &out->args[out->numargs-1];
1804                 unsigned diffsize = reqsize - nbytes;
1805                 if (diffsize > lastarg->size)
1806                         return -EINVAL;
1807                 lastarg->size -= diffsize;
1808         }
1809         return fuse_copy_args(cs, out->numargs, out->argpages, out->args,
1810                               out->page_zeroing);
1811 }
1812
1813 /*
1814  * Write a single reply to a request.  First the header is copied from
1815  * the write buffer.  The request is then searched on the processing
1816  * list by the unique ID found in the header.  If found, then remove
1817  * it from the list and copy the rest of the buffer to the request.
1818  * The request is finished by calling request_end()
1819  */
1820 static ssize_t fuse_dev_do_write(struct fuse_conn *fc,
1821                                  struct fuse_copy_state *cs, size_t nbytes)
1822 {
1823         int err;
1824         struct fuse_req *req;
1825         struct fuse_out_header oh;
1826
1827         if (nbytes < sizeof(struct fuse_out_header))
1828                 return -EINVAL;
1829
1830         err = fuse_copy_one(cs, &oh, sizeof(oh));
1831         if (err)
1832                 goto err_finish;
1833
1834         err = -EINVAL;
1835         if (oh.len != nbytes)
1836                 goto err_finish;
1837
1838         /*
1839          * Zero oh.unique indicates unsolicited notification message
1840          * and error contains notification code.
1841          */
1842         if (!oh.unique) {
1843                 err = fuse_notify(fc, oh.error, nbytes - sizeof(oh), cs);
1844                 return err ? err : nbytes;
1845         }
1846
1847         err = -EINVAL;
1848         if (oh.error <= -1000 || oh.error > 0)
1849                 goto err_finish;
1850
1851         spin_lock(&fc->lock);
1852         err = -ENOENT;
1853         if (!fc->connected)
1854                 goto err_unlock;
1855
1856         req = request_find(fc, oh.unique);
1857         if (!req)
1858                 goto err_unlock;
1859
1860         if (req->aborted) {
1861                 spin_unlock(&fc->lock);
1862                 fuse_copy_finish(cs);
1863                 spin_lock(&fc->lock);
1864                 request_end(fc, req);
1865                 return -ENOENT;
1866         }
1867         /* Is it an interrupt reply? */
1868         if (req->intr_unique == oh.unique) {
1869                 err = -EINVAL;
1870                 if (nbytes != sizeof(struct fuse_out_header))
1871                         goto err_unlock;
1872
1873                 if (oh.error == -ENOSYS)
1874                         fc->no_interrupt = 1;
1875                 else if (oh.error == -EAGAIN)
1876                         queue_interrupt(fc, req);
1877
1878                 spin_unlock(&fc->lock);
1879                 fuse_copy_finish(cs);
1880                 return nbytes;
1881         }
1882
1883         req->state = FUSE_REQ_WRITING;
1884         list_move(&req->list, &fc->io);
1885         req->out.h = oh;
1886         req->locked = 1;
1887         cs->req = req;
1888         if (!req->out.page_replace)
1889                 cs->move_pages = 0;
1890         spin_unlock(&fc->lock);
1891
1892         err = copy_out_args(cs, &req->out, nbytes);
1893         fuse_copy_finish(cs);
1894
1895         spin_lock(&fc->lock);
1896         req->locked = 0;
1897         if (!err) {
1898                 if (req->aborted)
1899                         err = -ENOENT;
1900         } else if (!req->aborted)
1901                 req->out.h.error = -EIO;
1902         request_end(fc, req);
1903
1904         return err ? err : nbytes;
1905
1906  err_unlock:
1907         spin_unlock(&fc->lock);
1908  err_finish:
1909         fuse_copy_finish(cs);
1910         return err;
1911 }
1912
1913 static ssize_t fuse_dev_write(struct kiocb *iocb, const struct iovec *iov,
1914                               unsigned long nr_segs, loff_t pos)
1915 {
1916         struct fuse_copy_state cs;
1917         struct fuse_conn *fc = fuse_get_conn(iocb->ki_filp);
1918         if (!fc)
1919                 return -EPERM;
1920
1921         fuse_copy_init(&cs, fc, 0, iov, nr_segs);
1922
1923         return fuse_dev_do_write(fc, &cs, iov_length(iov, nr_segs));
1924 }
1925
1926 static ssize_t fuse_dev_splice_write(struct pipe_inode_info *pipe,
1927                                      struct file *out, loff_t *ppos,
1928                                      size_t len, unsigned int flags)
1929 {
1930         unsigned nbuf;
1931         unsigned idx;
1932         struct pipe_buffer *bufs;
1933         struct fuse_copy_state cs;
1934         struct fuse_conn *fc;
1935         size_t rem;
1936         ssize_t ret;
1937
1938         fc = fuse_get_conn(out);
1939         if (!fc)
1940                 return -EPERM;
1941
1942         bufs = kmalloc(pipe->buffers * sizeof(struct pipe_buffer), GFP_KERNEL);
1943         if (!bufs)
1944                 return -ENOMEM;
1945
1946         pipe_lock(pipe);
1947         nbuf = 0;
1948         rem = 0;
1949         for (idx = 0; idx < pipe->nrbufs && rem < len; idx++)
1950                 rem += pipe->bufs[(pipe->curbuf + idx) & (pipe->buffers - 1)].len;
1951
1952         ret = -EINVAL;
1953         if (rem < len) {
1954                 pipe_unlock(pipe);
1955                 goto out;
1956         }
1957
1958         rem = len;
1959         while (rem) {
1960                 struct pipe_buffer *ibuf;
1961                 struct pipe_buffer *obuf;
1962
1963                 BUG_ON(nbuf >= pipe->buffers);
1964                 BUG_ON(!pipe->nrbufs);
1965                 ibuf = &pipe->bufs[pipe->curbuf];
1966                 obuf = &bufs[nbuf];
1967
1968                 if (rem >= ibuf->len) {
1969                         *obuf = *ibuf;
1970                         ibuf->ops = NULL;
1971                         pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1);
1972                         pipe->nrbufs--;
1973                 } else {
1974                         ibuf->ops->get(pipe, ibuf);
1975                         *obuf = *ibuf;
1976                         obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
1977                         obuf->len = rem;
1978                         ibuf->offset += obuf->len;
1979                         ibuf->len -= obuf->len;
1980                 }
1981                 nbuf++;
1982                 rem -= obuf->len;
1983         }
1984         pipe_unlock(pipe);
1985
1986         fuse_copy_init(&cs, fc, 0, NULL, nbuf);
1987         cs.pipebufs = bufs;
1988         cs.pipe = pipe;
1989
1990         if (flags & SPLICE_F_MOVE)
1991                 cs.move_pages = 1;
1992
1993         ret = fuse_dev_do_write(fc, &cs, len);
1994
1995         for (idx = 0; idx < nbuf; idx++) {
1996                 struct pipe_buffer *buf = &bufs[idx];
1997                 buf->ops->release(pipe, buf);
1998         }
1999 out:
2000         kfree(bufs);
2001         return ret;
2002 }
2003
2004 static unsigned fuse_dev_poll(struct file *file, poll_table *wait)
2005 {
2006         unsigned mask = POLLOUT | POLLWRNORM;
2007         struct fuse_conn *fc = fuse_get_conn(file);
2008         if (!fc)
2009                 return POLLERR;
2010
2011         poll_wait(file, &fc->waitq, wait);
2012
2013         spin_lock(&fc->lock);
2014         if (!fc->connected)
2015                 mask = POLLERR;
2016         else if (request_pending(fc))
2017                 mask |= POLLIN | POLLRDNORM;
2018         spin_unlock(&fc->lock);
2019
2020         return mask;
2021 }
2022
2023 /*
2024  * Abort all requests on the given list (pending or processing)
2025  *
2026  * This function releases and reacquires fc->lock
2027  */
2028 static void end_requests(struct fuse_conn *fc, struct list_head *head)
2029 __releases(fc->lock)
2030 __acquires(fc->lock)
2031 {
2032         while (!list_empty(head)) {
2033                 struct fuse_req *req;
2034                 req = list_entry(head->next, struct fuse_req, list);
2035                 req->out.h.error = -ECONNABORTED;
2036                 request_end(fc, req);
2037                 spin_lock(&fc->lock);
2038         }
2039 }
2040
2041 /*
2042  * Abort requests under I/O
2043  *
2044  * The requests are set to aborted and finished, and the request
2045  * waiter is woken up.  This will make request_wait_answer() wait
2046  * until the request is unlocked and then return.
2047  *
2048  * If the request is asynchronous, then the end function needs to be
2049  * called after waiting for the request to be unlocked (if it was
2050  * locked).
2051  */
2052 static void end_io_requests(struct fuse_conn *fc)
2053 __releases(fc->lock)
2054 __acquires(fc->lock)
2055 {
2056         while (!list_empty(&fc->io)) {
2057                 struct fuse_req *req =
2058                         list_entry(fc->io.next, struct fuse_req, list);
2059                 void (*end) (struct fuse_conn *, struct fuse_req *) = req->end;
2060
2061                 req->aborted = 1;
2062                 req->out.h.error = -ECONNABORTED;
2063                 req->state = FUSE_REQ_FINISHED;
2064                 list_del_init(&req->list);
2065                 wake_up(&req->waitq);
2066                 if (end) {
2067                         req->end = NULL;
2068                         __fuse_get_request(req);
2069                         spin_unlock(&fc->lock);
2070                         wait_event(req->waitq, !req->locked);
2071                         end(fc, req);
2072                         fuse_put_request(fc, req);
2073                         spin_lock(&fc->lock);
2074                 }
2075         }
2076 }
2077
2078 static void end_queued_requests(struct fuse_conn *fc)
2079 __releases(fc->lock)
2080 __acquires(fc->lock)
2081 {
2082         fc->max_background = UINT_MAX;
2083         flush_bg_queue(fc);
2084         end_requests(fc, &fc->pending);
2085         end_requests(fc, &fc->processing);
2086         while (forget_pending(fc))
2087                 kfree(dequeue_forget(fc, 1, NULL));
2088 }
2089
2090 static void end_polls(struct fuse_conn *fc)
2091 {
2092         struct rb_node *p;
2093
2094         p = rb_first(&fc->polled_files);
2095
2096         while (p) {
2097                 struct fuse_file *ff;
2098                 ff = rb_entry(p, struct fuse_file, polled_node);
2099                 wake_up_interruptible_all(&ff->poll_wait);
2100
2101                 p = rb_next(p);
2102         }
2103 }
2104
2105 /*
2106  * Abort all requests.
2107  *
2108  * Emergency exit in case of a malicious or accidental deadlock, or
2109  * just a hung filesystem.
2110  *
2111  * The same effect is usually achievable through killing the
2112  * filesystem daemon and all users of the filesystem.  The exception
2113  * is the combination of an asynchronous request and the tricky
2114  * deadlock (see Documentation/filesystems/fuse.txt).
2115  *
2116  * During the aborting, progression of requests from the pending and
2117  * processing lists onto the io list, and progression of new requests
2118  * onto the pending list is prevented by req->connected being false.
2119  *
2120  * Progression of requests under I/O to the processing list is
2121  * prevented by the req->aborted flag being true for these requests.
2122  * For this reason requests on the io list must be aborted first.
2123  */
2124 void fuse_abort_conn(struct fuse_conn *fc)
2125 {
2126         spin_lock(&fc->lock);
2127         if (fc->connected) {
2128                 fc->connected = 0;
2129                 fc->blocked = 0;
2130                 fc->initialized = 1;
2131                 end_io_requests(fc);
2132                 end_queued_requests(fc);
2133                 end_polls(fc);
2134                 wake_up_all(&fc->waitq);
2135                 wake_up_all(&fc->blocked_waitq);
2136                 kill_fasync(&fc->fasync, SIGIO, POLL_IN);
2137         }
2138         spin_unlock(&fc->lock);
2139 }
2140 EXPORT_SYMBOL_GPL(fuse_abort_conn);
2141
2142 int fuse_dev_release(struct inode *inode, struct file *file)
2143 {
2144         struct fuse_conn *fc = fuse_get_conn(file);
2145         if (fc) {
2146                 spin_lock(&fc->lock);
2147                 fc->connected = 0;
2148                 fc->blocked = 0;
2149                 fc->initialized = 1;
2150                 end_queued_requests(fc);
2151                 end_polls(fc);
2152                 wake_up_all(&fc->blocked_waitq);
2153                 spin_unlock(&fc->lock);
2154                 fuse_conn_put(fc);
2155         }
2156
2157         return 0;
2158 }
2159 EXPORT_SYMBOL_GPL(fuse_dev_release);
2160
2161 static int fuse_dev_fasync(int fd, struct file *file, int on)
2162 {
2163         struct fuse_conn *fc = fuse_get_conn(file);
2164         if (!fc)
2165                 return -EPERM;
2166
2167         /* No locking - fasync_helper does its own locking */
2168         return fasync_helper(fd, file, on, &fc->fasync);
2169 }
2170
2171 const struct file_operations fuse_dev_operations = {
2172         .owner          = THIS_MODULE,
2173         .llseek         = no_llseek,
2174         .read           = do_sync_read,
2175         .aio_read       = fuse_dev_read,
2176         .splice_read    = fuse_dev_splice_read,
2177         .write          = do_sync_write,
2178         .aio_write      = fuse_dev_write,
2179         .splice_write   = fuse_dev_splice_write,
2180         .poll           = fuse_dev_poll,
2181         .release        = fuse_dev_release,
2182         .fasync         = fuse_dev_fasync,
2183 };
2184 EXPORT_SYMBOL_GPL(fuse_dev_operations);
2185
2186 static struct miscdevice fuse_miscdevice = {
2187         .minor = FUSE_MINOR,
2188         .name  = "fuse",
2189         .fops = &fuse_dev_operations,
2190 };
2191
2192 int __init fuse_dev_init(void)
2193 {
2194         int err = -ENOMEM;
2195         fuse_req_cachep = kmem_cache_create("fuse_request",
2196                                             sizeof(struct fuse_req),
2197                                             0, 0, NULL);
2198         if (!fuse_req_cachep)
2199                 goto out;
2200
2201         err = misc_register(&fuse_miscdevice);
2202         if (err)
2203                 goto out_cache_clean;
2204
2205         return 0;
2206
2207  out_cache_clean:
2208         kmem_cache_destroy(fuse_req_cachep);
2209  out:
2210         return err;
2211 }
2212
2213 void fuse_dev_cleanup(void)
2214 {
2215         misc_deregister(&fuse_miscdevice);
2216         kmem_cache_destroy(fuse_req_cachep);
2217 }