fs/fuse/dev.c

   1 /*
   2   FUSE: Filesystem in Userspace
   3   Copyright (C) 2001-2008  Miklos Szeredi <[email protected]>
   4
   5   This program can be distributed under the terms of the GNU GPL.
   6   See the file COPYING.
   7 */
   8
   9 #include "fuse_i.h"
  10
  11 #include <linux/init.h>
  12 #include <linux/module.h>
  13 #include <linux/poll.h>
  14 #include <linux/sched/signal.h>
  15 #include <linux/uio.h>
  16 #include <linux/miscdevice.h>
  17 #include <linux/pagemap.h>
  18 #include <linux/file.h>
  19 #include <linux/slab.h>
  20 #include <linux/pipe_fs_i.h>
  21 #include <linux/swap.h>
  22 #include <linux/splice.h>
  23 #include <linux/sched.h>
  24
  25 #define CREATE_TRACE_POINTS
  26 #include "fuse_trace.h"
  27
  28 MODULE_ALIAS_MISCDEV(FUSE_MINOR);
  29 MODULE_ALIAS("devname:fuse");
  30
  31 /* Ordinary requests have even IDs, while interrupts IDs are odd */
  32 #define FUSE_INT_REQ_BIT (1ULL << 0)
  33 #define FUSE_REQ_ID_STEP (1ULL << 1)
  34
  35 static struct kmem_cache *fuse_req_cachep;
  36
  37 static void end_requests(struct list_head *head);
  38
  39 static struct fuse_dev *fuse_get_dev(struct file *file)
  40 {
  41         /*
  42          * Lockless access is OK, because file->private data is set
  43          * once during mount and is valid until the file is released.
  44          */
  45         return READ_ONCE(file->private_data);
  46 }
  47
  48 static void fuse_request_init(struct fuse_mount *fm, struct fuse_req *req)
  49 {
  50         INIT_LIST_HEAD(&req->list);
  51         INIT_LIST_HEAD(&req->intr_entry);
  52         init_waitqueue_head(&req->waitq);
  53         refcount_set(&req->count, 1);
  54         __set_bit(FR_PENDING, &req->flags);
  55         req->fm = fm;
  56 }
  57
  58 static struct fuse_req *fuse_request_alloc(struct fuse_mount *fm, gfp_t flags)
  59 {
  60         struct fuse_req *req = kmem_cache_zalloc(fuse_req_cachep, flags);
  61         if (req)
  62                 fuse_request_init(fm, req);
  63
  64         return req;
  65 }
  66
  67 static void fuse_request_free(struct fuse_req *req)
  68 {
  69         kmem_cache_free(fuse_req_cachep, req);
  70 }
  71
  72 static void __fuse_get_request(struct fuse_req *req)
  73 {
  74         refcount_inc(&req->count);
  75 }
  76
  77 /* Must be called with > 1 refcount */
  78 static void __fuse_put_request(struct fuse_req *req)
  79 {
  80         refcount_dec(&req->count);
  81 }
  82
  83 void fuse_set_initialized(struct fuse_conn *fc)
  84 {
  85         /* Make sure stores before this are seen on another CPU */
  86         smp_wmb();
  87         fc->initialized = 1;
  88 }
  89
  90 static bool fuse_block_alloc(struct fuse_conn *fc, bool for_background)
  91 {
  92         return !fc->initialized || (for_background && fc->blocked);
  93 }
  94
  95 static void fuse_drop_waiting(struct fuse_conn *fc)
  96 {
  97         /*
  98          * lockess check of fc->connected is okay, because atomic_dec_and_test()
  99          * provides a memory barrier matched with the one in fuse_wait_aborted()
 100          * to ensure no wake-up is missed.
 101          */
 102         if (atomic_dec_and_test(&fc->num_waiting) &&
 103             !READ_ONCE(fc->connected)) {
 104                 /* wake up aborters */
 105                 wake_up_all(&fc->blocked_waitq);
 106         }
 107 }
 108
 109 static void fuse_put_request(struct fuse_req *req);
 110
 111 static struct fuse_req *fuse_get_req(struct mnt_idmap *idmap,
 112                                      struct fuse_mount *fm,
 113                                      bool for_background)
 114 {
 115         struct fuse_conn *fc = fm->fc;
 116         struct fuse_req *req;
 117         bool no_idmap = !fm->sb || (fm->sb->s_iflags & SB_I_NOIDMAP);
 118         kuid_t fsuid;
 119         kgid_t fsgid;
 120         int err;
 121
 122         atomic_inc(&fc->num_waiting);
 123
 124         if (fuse_block_alloc(fc, for_background)) {
 125                 err = -EINTR;
 126                 if (wait_event_killable_exclusive(fc->blocked_waitq,
 127                                 !fuse_block_alloc(fc, for_background)))
 128                         goto out;
 129         }
 130         /* Matches smp_wmb() in fuse_set_initialized() */
 131         smp_rmb();
 132
 133         err = -ENOTCONN;
 134         if (!fc->connected)
 135                 goto out;
 136
 137         err = -ECONNREFUSED;
 138         if (fc->conn_error)
 139                 goto out;
 140
 141         req = fuse_request_alloc(fm, GFP_KERNEL);
 142         err = -ENOMEM;
 143         if (!req) {
 144                 if (for_background)
 145                         wake_up(&fc->blocked_waitq);
 146                 goto out;
 147         }
 148
 149         req->in.h.pid = pid_nr_ns(task_pid(current), fc->pid_ns);
 150
 151         __set_bit(FR_WAITING, &req->flags);
 152         if (for_background)
 153                 __set_bit(FR_BACKGROUND, &req->flags);
 154
 155         /*
 156          * Keep the old behavior when idmappings support was not
 157          * declared by a FUSE server.
 158          *
 159          * For those FUSE servers who support idmapped mounts,
 160          * we send UID/GID only along with "inode creation"
 161          * fuse requests, otherwise idmap == &invalid_mnt_idmap and
 162          * req->in.h.{u,g}id will be equal to FUSE_INVALID_UIDGID.
 163          */
 164         fsuid = no_idmap ? current_fsuid() : mapped_fsuid(idmap, fc->user_ns);
 165         fsgid = no_idmap ? current_fsgid() : mapped_fsgid(idmap, fc->user_ns);
 166         req->in.h.uid = from_kuid(fc->user_ns, fsuid);
 167         req->in.h.gid = from_kgid(fc->user_ns, fsgid);
 168
 169         if (no_idmap && unlikely(req->in.h.uid == ((uid_t)-1) ||
 170                                  req->in.h.gid == ((gid_t)-1))) {
 171                 fuse_put_request(req);
 172                 return ERR_PTR(-EOVERFLOW);
 173         }
 174
 175         return req;
 176
 177  out:
 178         fuse_drop_waiting(fc);
 179         return ERR_PTR(err);
 180 }
 181
 182 static void fuse_put_request(struct fuse_req *req)
 183 {
 184         struct fuse_conn *fc = req->fm->fc;
 185
 186         if (refcount_dec_and_test(&req->count)) {
 187                 if (test_bit(FR_BACKGROUND, &req->flags)) {
 188                         /*
 189                          * We get here in the unlikely case that a background
 190                          * request was allocated but not sent
 191                          */
 192                         spin_lock(&fc->bg_lock);
 193                         if (!fc->blocked)
 194                                 wake_up(&fc->blocked_waitq);
 195                         spin_unlock(&fc->bg_lock);
 196                 }
 197
 198                 if (test_bit(FR_WAITING, &req->flags)) {
 199                         __clear_bit(FR_WAITING, &req->flags);
 200                         fuse_drop_waiting(fc);
 201                 }
 202
 203                 fuse_request_free(req);
 204         }
 205 }
 206
 207 unsigned int fuse_len_args(unsigned int numargs, struct fuse_arg *args)
 208 {
 209         unsigned nbytes = 0;
 210         unsigned i;
 211
 212         for (i = 0; i < numargs; i++)
 213                 nbytes += args[i].size;
 214
 215         return nbytes;
 216 }
 217 EXPORT_SYMBOL_GPL(fuse_len_args);
 218
 219 static u64 fuse_get_unique_locked(struct fuse_iqueue *fiq)
 220 {
 221         fiq->reqctr += FUSE_REQ_ID_STEP;
 222         return fiq->reqctr;
 223 }
 224
 225 u64 fuse_get_unique(struct fuse_iqueue *fiq)
 226 {
 227         u64 ret;
 228
 229         spin_lock(&fiq->lock);
 230         ret = fuse_get_unique_locked(fiq);
 231         spin_unlock(&fiq->lock);
 232
 233         return ret;
 234 }
 235 EXPORT_SYMBOL_GPL(fuse_get_unique);
 236
 237 static unsigned int fuse_req_hash(u64 unique)
 238 {
 239         return hash_long(unique & ~FUSE_INT_REQ_BIT, FUSE_PQ_HASH_BITS);
 240 }
 241
 242 /*
 243  * A new request is available, wake fiq->waitq
 244  */
 245 static void fuse_dev_wake_and_unlock(struct fuse_iqueue *fiq)
 246 __releases(fiq->lock)
 247 {
 248         wake_up(&fiq->waitq);
 249         kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
 250         spin_unlock(&fiq->lock);
 251 }
 252
 253 static void fuse_dev_queue_forget(struct fuse_iqueue *fiq, struct fuse_forget_link *forget)
 254 {
 255         spin_lock(&fiq->lock);
 256         if (fiq->connected) {
 257                 fiq->forget_list_tail->next = forget;
 258                 fiq->forget_list_tail = forget;
 259                 fuse_dev_wake_and_unlock(fiq);
 260         } else {
 261                 kfree(forget);
 262                 spin_unlock(&fiq->lock);
 263         }
 264 }
 265
 266 static void fuse_dev_queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req)
 267 {
 268         spin_lock(&fiq->lock);
 269         if (list_empty(&req->intr_entry)) {
 270                 list_add_tail(&req->intr_entry, &fiq->interrupts);
 271                 /*
 272                  * Pairs with smp_mb() implied by test_and_set_bit()
 273                  * from fuse_request_end().
 274                  */
 275                 smp_mb();
 276                 if (test_bit(FR_FINISHED, &req->flags)) {
 277                         list_del_init(&req->intr_entry);
 278                         spin_unlock(&fiq->lock);
 279                 } else  {
 280                         fuse_dev_wake_and_unlock(fiq);
 281                 }
 282         } else {
 283                 spin_unlock(&fiq->lock);
 284         }
 285 }
 286
 287 static void fuse_dev_queue_req(struct fuse_iqueue *fiq, struct fuse_req *req)
 288 {
 289         spin_lock(&fiq->lock);
 290         if (fiq->connected) {
 291                 if (req->in.h.opcode != FUSE_NOTIFY_REPLY)
 292                         req->in.h.unique = fuse_get_unique_locked(fiq);
 293                 list_add_tail(&req->list, &fiq->pending);
 294                 fuse_dev_wake_and_unlock(fiq);
 295         } else {
 296                 spin_unlock(&fiq->lock);
 297                 req->out.h.error = -ENOTCONN;
 298                 clear_bit(FR_PENDING, &req->flags);
 299                 fuse_request_end(req);
 300         }
 301 }
 302
 303 const struct fuse_iqueue_ops fuse_dev_fiq_ops = {
 304         .send_forget    = fuse_dev_queue_forget,
 305         .send_interrupt = fuse_dev_queue_interrupt,
 306         .send_req       = fuse_dev_queue_req,
 307 };
 308 EXPORT_SYMBOL_GPL(fuse_dev_fiq_ops);
 309
 310 static void fuse_send_one(struct fuse_iqueue *fiq, struct fuse_req *req)
 311 {
 312         req->in.h.len = sizeof(struct fuse_in_header) +
 313                 fuse_len_args(req->args->in_numargs,
 314                               (struct fuse_arg *) req->args->in_args);
 315         trace_fuse_request_send(req);
 316         fiq->ops->send_req(fiq, req);
 317 }
 318
 319 void fuse_queue_forget(struct fuse_conn *fc, struct fuse_forget_link *forget,
 320                        u64 nodeid, u64 nlookup)
 321 {
 322         struct fuse_iqueue *fiq = &fc->iq;
 323
 324         forget->forget_one.nodeid = nodeid;
 325         forget->forget_one.nlookup = nlookup;
 326
 327         fiq->ops->send_forget(fiq, forget);
 328 }
 329
 330 static void flush_bg_queue(struct fuse_conn *fc)
 331 {
 332         struct fuse_iqueue *fiq = &fc->iq;
 333
 334         while (fc->active_background < fc->max_background &&
 335                !list_empty(&fc->bg_queue)) {
 336                 struct fuse_req *req;
 337
 338                 req = list_first_entry(&fc->bg_queue, struct fuse_req, list);
 339                 list_del(&req->list);
 340                 fc->active_background++;
 341                 fuse_send_one(fiq, req);
 342         }
 343 }
 344
 345 /*
 346  * This function is called when a request is finished.  Either a reply
 347  * has arrived or it was aborted (and not yet sent) or some error
 348  * occurred during communication with userspace, or the device file
 349  * was closed.  The requester thread is woken up (if still waiting),
 350  * the 'end' callback is called if given, else the reference to the
 351  * request is released
 352  */
 353 void fuse_request_end(struct fuse_req *req)
 354 {
 355         struct fuse_mount *fm = req->fm;
 356         struct fuse_conn *fc = fm->fc;
 357         struct fuse_iqueue *fiq = &fc->iq;
 358
 359         if (test_and_set_bit(FR_FINISHED, &req->flags))
 360                 goto put_request;
 361
 362         trace_fuse_request_end(req);
 363         /*
 364          * test_and_set_bit() implies smp_mb() between bit
 365          * changing and below FR_INTERRUPTED check. Pairs with
 366          * smp_mb() from queue_interrupt().
 367          */
 368         if (test_bit(FR_INTERRUPTED, &req->flags)) {
 369                 spin_lock(&fiq->lock);
 370                 list_del_init(&req->intr_entry);
 371                 spin_unlock(&fiq->lock);
 372         }
 373         WARN_ON(test_bit(FR_PENDING, &req->flags));
 374         WARN_ON(test_bit(FR_SENT, &req->flags));
 375         if (test_bit(FR_BACKGROUND, &req->flags)) {
 376                 spin_lock(&fc->bg_lock);
 377                 clear_bit(FR_BACKGROUND, &req->flags);
 378                 if (fc->num_background == fc->max_background) {
 379                         fc->blocked = 0;
 380                         wake_up(&fc->blocked_waitq);
 381                 } else if (!fc->blocked) {
 382                         /*
 383                          * Wake up next waiter, if any.  It's okay to use
 384                          * waitqueue_active(), as we've already synced up
 385                          * fc->blocked with waiters with the wake_up() call
 386                          * above.
 387                          */
 388                         if (waitqueue_active(&fc->blocked_waitq))
 389                                 wake_up(&fc->blocked_waitq);
 390                 }
 391
 392                 fc->num_background--;
 393                 fc->active_background--;
 394                 flush_bg_queue(fc);
 395                 spin_unlock(&fc->bg_lock);
 396         } else {
 397                 /* Wake up waiter sleeping in request_wait_answer() */
 398                 wake_up(&req->waitq);
 399         }
 400
 401         if (test_bit(FR_ASYNC, &req->flags))
 402                 req->args->end(fm, req->args, req->out.h.error);
 403 put_request:
 404         fuse_put_request(req);
 405 }
 406 EXPORT_SYMBOL_GPL(fuse_request_end);
 407
 408 static int queue_interrupt(struct fuse_req *req)
 409 {
 410         struct fuse_iqueue *fiq = &req->fm->fc->iq;
 411
 412         /* Check for we've sent request to interrupt this req */
 413         if (unlikely(!test_bit(FR_INTERRUPTED, &req->flags)))
 414                 return -EINVAL;
 415
 416         fiq->ops->send_interrupt(fiq, req);
 417
 418         return 0;
 419 }
 420
 421 static void request_wait_answer(struct fuse_req *req)
 422 {
 423         struct fuse_conn *fc = req->fm->fc;
 424         struct fuse_iqueue *fiq = &fc->iq;
 425         int err;
 426
 427         if (!fc->no_interrupt) {
 428                 /* Any signal may interrupt this */
 429                 err = wait_event_interruptible(req->waitq,
 430                                         test_bit(FR_FINISHED, &req->flags));
 431                 if (!err)
 432                         return;
 433
 434                 set_bit(FR_INTERRUPTED, &req->flags);
 435                 /* matches barrier in fuse_dev_do_read() */
 436                 smp_mb__after_atomic();
 437                 if (test_bit(FR_SENT, &req->flags))
 438                         queue_interrupt(req);
 439         }
 440
 441         if (!test_bit(FR_FORCE, &req->flags)) {
 442                 /* Only fatal signals may interrupt this */
 443                 err = wait_event_killable(req->waitq,
 444                                         test_bit(FR_FINISHED, &req->flags));
 445                 if (!err)
 446                         return;
 447
 448                 spin_lock(&fiq->lock);
 449                 /* Request is not yet in userspace, bail out */
 450                 if (test_bit(FR_PENDING, &req->flags)) {
 451                         list_del(&req->list);
 452                         spin_unlock(&fiq->lock);
 453                         __fuse_put_request(req);
 454                         req->out.h.error = -EINTR;
 455                         return;
 456                 }
 457                 spin_unlock(&fiq->lock);
 458         }
 459
 460         /*
 461          * Either request is already in userspace, or it was forced.
 462          * Wait it out.
 463          */
 464         wait_event(req->waitq, test_bit(FR_FINISHED, &req->flags));
 465 }
 466
 467 static void __fuse_request_send(struct fuse_req *req)
 468 {
 469         struct fuse_iqueue *fiq = &req->fm->fc->iq;
 470
 471         BUG_ON(test_bit(FR_BACKGROUND, &req->flags));
 472
 473         /* acquire extra reference, since request is still needed after
 474            fuse_request_end() */
 475         __fuse_get_request(req);
 476         fuse_send_one(fiq, req);
 477
 478         request_wait_answer(req);
 479         /* Pairs with smp_wmb() in fuse_request_end() */
 480         smp_rmb();
 481 }
 482
 483 static void fuse_adjust_compat(struct fuse_conn *fc, struct fuse_args *args)
 484 {
 485         if (fc->minor < 4 && args->opcode == FUSE_STATFS)
 486                 args->out_args[0].size = FUSE_COMPAT_STATFS_SIZE;
 487
 488         if (fc->minor < 9) {
 489                 switch (args->opcode) {
 490                 case FUSE_LOOKUP:
 491                 case FUSE_CREATE:
 492                 case FUSE_MKNOD:
 493                 case FUSE_MKDIR:
 494                 case FUSE_SYMLINK:
 495                 case FUSE_LINK:
 496                         args->out_args[0].size = FUSE_COMPAT_ENTRY_OUT_SIZE;
 497                         break;
 498                 case FUSE_GETATTR:
 499                 case FUSE_SETATTR:
 500                         args->out_args[0].size = FUSE_COMPAT_ATTR_OUT_SIZE;
 501                         break;
 502                 }
 503         }
 504         if (fc->minor < 12) {
 505                 switch (args->opcode) {
 506                 case FUSE_CREATE:
 507                         args->in_args[0].size = sizeof(struct fuse_open_in);
 508                         break;
 509                 case FUSE_MKNOD:
 510                         args->in_args[0].size = FUSE_COMPAT_MKNOD_IN_SIZE;
 511                         break;
 512                 }
 513         }
 514 }
 515
 516 static void fuse_force_creds(struct fuse_req *req)
 517 {
 518         struct fuse_conn *fc = req->fm->fc;
 519
 520         if (!req->fm->sb || req->fm->sb->s_iflags & SB_I_NOIDMAP) {
 521                 req->in.h.uid = from_kuid_munged(fc->user_ns, current_fsuid());
 522                 req->in.h.gid = from_kgid_munged(fc->user_ns, current_fsgid());
 523         } else {
 524                 req->in.h.uid = FUSE_INVALID_UIDGID;
 525                 req->in.h.gid = FUSE_INVALID_UIDGID;
 526         }
 527
 528         req->in.h.pid = pid_nr_ns(task_pid(current), fc->pid_ns);
 529 }
 530
 531 static void fuse_args_to_req(struct fuse_req *req, struct fuse_args *args)
 532 {
 533         req->in.h.opcode = args->opcode;
 534         req->in.h.nodeid = args->nodeid;
 535         req->args = args;
 536         if (args->is_ext)
 537                 req->in.h.total_extlen = args->in_args[args->ext_idx].size / 8;
 538         if (args->end)
 539                 __set_bit(FR_ASYNC, &req->flags);
 540 }
 541
 542 ssize_t __fuse_simple_request(struct mnt_idmap *idmap,
 543                               struct fuse_mount *fm,
 544                               struct fuse_args *args)
 545 {
 546         struct fuse_conn *fc = fm->fc;
 547         struct fuse_req *req;
 548         ssize_t ret;
 549
 550         if (args->force) {
 551                 atomic_inc(&fc->num_waiting);
 552                 req = fuse_request_alloc(fm, GFP_KERNEL | __GFP_NOFAIL);
 553
 554                 if (!args->nocreds)
 555                         fuse_force_creds(req);
 556
 557                 __set_bit(FR_WAITING, &req->flags);
 558                 __set_bit(FR_FORCE, &req->flags);
 559         } else {
 560                 WARN_ON(args->nocreds);
 561                 req = fuse_get_req(idmap, fm, false);
 562                 if (IS_ERR(req))
 563                         return PTR_ERR(req);
 564         }
 565
 566         /* Needs to be done after fuse_get_req() so that fc->minor is valid */
 567         fuse_adjust_compat(fc, args);
 568         fuse_args_to_req(req, args);
 569
 570         if (!args->noreply)
 571                 __set_bit(FR_ISREPLY, &req->flags);
 572         __fuse_request_send(req);
 573         ret = req->out.h.error;
 574         if (!ret && args->out_argvar) {
 575                 BUG_ON(args->out_numargs == 0);
 576                 ret = args->out_args[args->out_numargs - 1].size;
 577         }
 578         fuse_put_request(req);
 579
 580         return ret;
 581 }
 582
 583 static bool fuse_request_queue_background(struct fuse_req *req)
 584 {
 585         struct fuse_mount *fm = req->fm;
 586         struct fuse_conn *fc = fm->fc;
 587         bool queued = false;
 588
 589         WARN_ON(!test_bit(FR_BACKGROUND, &req->flags));
 590         if (!test_bit(FR_WAITING, &req->flags)) {
 591                 __set_bit(FR_WAITING, &req->flags);
 592                 atomic_inc(&fc->num_waiting);
 593         }
 594         __set_bit(FR_ISREPLY, &req->flags);
 595         spin_lock(&fc->bg_lock);
 596         if (likely(fc->connected)) {
 597                 fc->num_background++;
 598                 if (fc->num_background == fc->max_background)
 599                         fc->blocked = 1;
 600                 list_add_tail(&req->list, &fc->bg_queue);
 601                 flush_bg_queue(fc);
 602                 queued = true;
 603         }
 604         spin_unlock(&fc->bg_lock);
 605
 606         return queued;
 607 }
 608
 609 int fuse_simple_background(struct fuse_mount *fm, struct fuse_args *args,
 610                             gfp_t gfp_flags)
 611 {
 612         struct fuse_req *req;
 613
 614         if (args->force) {
 615                 WARN_ON(!args->nocreds);
 616                 req = fuse_request_alloc(fm, gfp_flags);
 617                 if (!req)
 618                         return -ENOMEM;
 619                 __set_bit(FR_BACKGROUND, &req->flags);
 620         } else {
 621                 WARN_ON(args->nocreds);
 622                 req = fuse_get_req(&invalid_mnt_idmap, fm, true);
 623                 if (IS_ERR(req))
 624                         return PTR_ERR(req);
 625         }
 626
 627         fuse_args_to_req(req, args);
 628
 629         if (!fuse_request_queue_background(req)) {
 630                 fuse_put_request(req);
 631                 return -ENOTCONN;
 632         }
 633
 634         return 0;
 635 }
 636 EXPORT_SYMBOL_GPL(fuse_simple_background);
 637
 638 static int fuse_simple_notify_reply(struct fuse_mount *fm,
 639                                     struct fuse_args *args, u64 unique)
 640 {
 641         struct fuse_req *req;
 642         struct fuse_iqueue *fiq = &fm->fc->iq;
 643
 644         req = fuse_get_req(&invalid_mnt_idmap, fm, false);
 645         if (IS_ERR(req))
 646                 return PTR_ERR(req);
 647
 648         __clear_bit(FR_ISREPLY, &req->flags);
 649         req->in.h.unique = unique;
 650
 651         fuse_args_to_req(req, args);
 652
 653         fuse_send_one(fiq, req);
 654
 655         return 0;
 656 }
 657
 658 /*
 659  * Lock the request.  Up to the next unlock_request() there mustn't be
 660  * anything that could cause a page-fault.  If the request was already
 661  * aborted bail out.
 662  */
 663 static int lock_request(struct fuse_req *req)
 664 {
 665         int err = 0;
 666         if (req) {
 667                 spin_lock(&req->waitq.lock);
 668                 if (test_bit(FR_ABORTED, &req->flags))
 669                         err = -ENOENT;
 670                 else
 671                         set_bit(FR_LOCKED, &req->flags);
 672                 spin_unlock(&req->waitq.lock);
 673         }
 674         return err;
 675 }
 676
 677 /*
 678  * Unlock request.  If it was aborted while locked, caller is responsible
 679  * for unlocking and ending the request.
 680  */
 681 static int unlock_request(struct fuse_req *req)
 682 {
 683         int err = 0;
 684         if (req) {
 685                 spin_lock(&req->waitq.lock);
 686                 if (test_bit(FR_ABORTED, &req->flags))
 687                         err = -ENOENT;
 688                 else
 689                         clear_bit(FR_LOCKED, &req->flags);
 690                 spin_unlock(&req->waitq.lock);
 691         }
 692         return err;
 693 }
 694
 695 struct fuse_copy_state {
 696         int write;
 697         struct fuse_req *req;
 698         struct iov_iter *iter;
 699         struct pipe_buffer *pipebufs;
 700         struct pipe_buffer *currbuf;
 701         struct pipe_inode_info *pipe;
 702         unsigned long nr_segs;
 703         struct page *pg;
 704         unsigned len;
 705         unsigned offset;
 706         unsigned move_pages:1;
 707 };
 708
 709 static void fuse_copy_init(struct fuse_copy_state *cs, int write,
 710                            struct iov_iter *iter)
 711 {
 712         memset(cs, 0, sizeof(*cs));
 713         cs->write = write;
 714         cs->iter = iter;
 715 }
 716
 717 /* Unmap and put previous page of userspace buffer */
 718 static void fuse_copy_finish(struct fuse_copy_state *cs)
 719 {
 720         if (cs->currbuf) {
 721                 struct pipe_buffer *buf = cs->currbuf;
 722
 723                 if (cs->write)
 724                         buf->len = PAGE_SIZE - cs->len;
 725                 cs->currbuf = NULL;
 726         } else if (cs->pg) {
 727                 if (cs->write) {
 728                         flush_dcache_page(cs->pg);
 729                         set_page_dirty_lock(cs->pg);
 730                 }
 731                 put_page(cs->pg);
 732         }
 733         cs->pg = NULL;
 734 }
 735
 736 /*
 737  * Get another pagefull of userspace buffer, and map it to kernel
 738  * address space, and lock request
 739  */
 740 static int fuse_copy_fill(struct fuse_copy_state *cs)
 741 {
 742         struct page *page;
 743         int err;
 744
 745         err = unlock_request(cs->req);
 746         if (err)
 747                 return err;
 748
 749         fuse_copy_finish(cs);
 750         if (cs->pipebufs) {
 751                 struct pipe_buffer *buf = cs->pipebufs;
 752
 753                 if (!cs->write) {
 754                         err = pipe_buf_confirm(cs->pipe, buf);
 755                         if (err)
 756                                 return err;
 757
 758                         BUG_ON(!cs->nr_segs);
 759                         cs->currbuf = buf;
 760                         cs->pg = buf->page;
 761                         cs->offset = buf->offset;
 762                         cs->len = buf->len;
 763                         cs->pipebufs++;
 764                         cs->nr_segs--;
 765                 } else {
 766                         if (cs->nr_segs >= cs->pipe->max_usage)
 767                                 return -EIO;
 768
 769                         page = alloc_page(GFP_HIGHUSER);
 770                         if (!page)
 771                                 return -ENOMEM;
 772
 773                         buf->page = page;
 774                         buf->offset = 0;
 775                         buf->len = 0;
 776
 777                         cs->currbuf = buf;
 778                         cs->pg = page;
 779                         cs->offset = 0;
 780                         cs->len = PAGE_SIZE;
 781                         cs->pipebufs++;
 782                         cs->nr_segs++;
 783                 }
 784         } else {
 785                 size_t off;
 786                 err = iov_iter_get_pages2(cs->iter, &page, PAGE_SIZE, 1, &off);
 787                 if (err < 0)
 788                         return err;
 789                 BUG_ON(!err);
 790                 cs->len = err;
 791                 cs->offset = off;
 792                 cs->pg = page;
 793         }
 794
 795         return lock_request(cs->req);
 796 }
 797
 798 /* Do as much copy to/from userspace buffer as we can */
 799 static int fuse_copy_do(struct fuse_copy_state *cs, void **val, unsigned *size)
 800 {
 801         unsigned ncpy = min(*size, cs->len);
 802         if (val) {
 803                 void *pgaddr = kmap_local_page(cs->pg);
 804                 void *buf = pgaddr + cs->offset;
 805
 806                 if (cs->write)
 807                         memcpy(buf, *val, ncpy);
 808                 else
 809                         memcpy(*val, buf, ncpy);
 810
 811                 kunmap_local(pgaddr);
 812                 *val += ncpy;
 813         }
 814         *size -= ncpy;
 815         cs->len -= ncpy;
 816         cs->offset += ncpy;
 817         return ncpy;
 818 }
 819
 820 static int fuse_check_folio(struct folio *folio)
 821 {
 822         if (folio_mapped(folio) ||
 823             folio->mapping != NULL ||
 824             (folio->flags & PAGE_FLAGS_CHECK_AT_PREP &
 825              ~(1 << PG_locked |
 826                1 << PG_referenced |
 827                1 << PG_lru |
 828                1 << PG_active |
 829                1 << PG_workingset |
 830                1 << PG_reclaim |
 831                1 << PG_waiters |
 832                LRU_GEN_MASK | LRU_REFS_MASK))) {
 833                 dump_page(&folio->page, "fuse: trying to steal weird page");
 834                 return 1;
 835         }
 836         return 0;
 837 }
 838
 839 static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)
 840 {
 841         int err;
 842         struct folio *oldfolio = page_folio(*pagep);
 843         struct folio *newfolio;
 844         struct pipe_buffer *buf = cs->pipebufs;
 845
 846         folio_get(oldfolio);
 847         err = unlock_request(cs->req);
 848         if (err)
 849                 goto out_put_old;
 850
 851         fuse_copy_finish(cs);
 852
 853         err = pipe_buf_confirm(cs->pipe, buf);
 854         if (err)
 855                 goto out_put_old;
 856
 857         BUG_ON(!cs->nr_segs);
 858         cs->currbuf = buf;
 859         cs->len = buf->len;
 860         cs->pipebufs++;
 861         cs->nr_segs--;
 862
 863         if (cs->len != PAGE_SIZE)
 864                 goto out_fallback;
 865
 866         if (!pipe_buf_try_steal(cs->pipe, buf))
 867                 goto out_fallback;
 868
 869         newfolio = page_folio(buf->page);
 870
 871         folio_clear_uptodate(newfolio);
 872         folio_clear_mappedtodisk(newfolio);
 873
 874         if (fuse_check_folio(newfolio) != 0)
 875                 goto out_fallback_unlock;
 876
 877         /*
 878          * This is a new and locked page, it shouldn't be mapped or
 879          * have any special flags on it
 880          */
 881         if (WARN_ON(folio_mapped(oldfolio)))
 882                 goto out_fallback_unlock;
 883         if (WARN_ON(folio_has_private(oldfolio)))
 884                 goto out_fallback_unlock;
 885         if (WARN_ON(folio_test_dirty(oldfolio) ||
 886                                 folio_test_writeback(oldfolio)))
 887                 goto out_fallback_unlock;
 888         if (WARN_ON(folio_test_mlocked(oldfolio)))
 889                 goto out_fallback_unlock;
 890
 891         replace_page_cache_folio(oldfolio, newfolio);
 892
 893         folio_get(newfolio);
 894
 895         if (!(buf->flags & PIPE_BUF_FLAG_LRU))
 896                 folio_add_lru(newfolio);
 897
 898         /*
 899          * Release while we have extra ref on stolen page.  Otherwise
 900          * anon_pipe_buf_release() might think the page can be reused.
 901          */
 902         pipe_buf_release(cs->pipe, buf);
 903
 904         err = 0;
 905         spin_lock(&cs->req->waitq.lock);
 906         if (test_bit(FR_ABORTED, &cs->req->flags))
 907                 err = -ENOENT;
 908         else
 909                 *pagep = &newfolio->page;
 910         spin_unlock(&cs->req->waitq.lock);
 911
 912         if (err) {
 913                 folio_unlock(newfolio);
 914                 folio_put(newfolio);
 915                 goto out_put_old;
 916         }
 917
 918         folio_unlock(oldfolio);
 919         /* Drop ref for ap->pages[] array */
 920         folio_put(oldfolio);
 921         cs->len = 0;
 922
 923         err = 0;
 924 out_put_old:
 925         /* Drop ref obtained in this function */
 926         folio_put(oldfolio);
 927         return err;
 928
 929 out_fallback_unlock:
 930         folio_unlock(newfolio);
 931 out_fallback:
 932         cs->pg = buf->page;
 933         cs->offset = buf->offset;
 934
 935         err = lock_request(cs->req);
 936         if (!err)
 937                 err = 1;
 938
 939         goto out_put_old;
 940 }
 941
 942 static int fuse_ref_page(struct fuse_copy_state *cs, struct page *page,
 943                          unsigned offset, unsigned count)
 944 {
 945         struct pipe_buffer *buf;
 946         int err;
 947
 948         if (cs->nr_segs >= cs->pipe->max_usage)
 949                 return -EIO;
 950
 951         get_page(page);
 952         err = unlock_request(cs->req);
 953         if (err) {
 954                 put_page(page);
 955                 return err;
 956         }
 957
 958         fuse_copy_finish(cs);
 959
 960         buf = cs->pipebufs;
 961         buf->page = page;
 962         buf->offset = offset;
 963         buf->len = count;
 964
 965         cs->pipebufs++;
 966         cs->nr_segs++;
 967         cs->len = 0;
 968
 969         return 0;
 970 }
 971
 972 /*
 973  * Copy a page in the request to/from the userspace buffer.  Must be
 974  * done atomically
 975  */
 976 static int fuse_copy_page(struct fuse_copy_state *cs, struct page **pagep,
 977                           unsigned offset, unsigned count, int zeroing)
 978 {
 979         int err;
 980         struct page *page = *pagep;
 981
 982         if (page && zeroing && count < PAGE_SIZE)
 983                 clear_highpage(page);
 984
 985         while (count) {
 986                 if (cs->write && cs->pipebufs && page) {
 987                         /*
 988                          * Can't control lifetime of pipe buffers, so always
 989                          * copy user pages.
 990                          */
 991                         if (cs->req->args->user_pages) {
 992                                 err = fuse_copy_fill(cs);
 993                                 if (err)
 994                                         return err;
 995                         } else {
 996                                 return fuse_ref_page(cs, page, offset, count);
 997                         }
 998                 } else if (!cs->len) {
 999                         if (cs->move_pages && page &&
1000                             offset == 0 && count == PAGE_SIZE) {
1001                                 err = fuse_try_move_page(cs, pagep);
1002                                 if (err <= 0)
1003                                         return err;
1004                         } else {
1005                                 err = fuse_copy_fill(cs);
1006                                 if (err)
1007                                         return err;
1008                         }
1009                 }
1010                 if (page) {
1011                         void *mapaddr = kmap_local_page(page);
1012                         void *buf = mapaddr + offset;
1013                         offset += fuse_copy_do(cs, &buf, &count);
1014                         kunmap_local(mapaddr);
1015                 } else
1016                         offset += fuse_copy_do(cs, NULL, &count);
1017         }
1018         if (page && !cs->write)
1019                 flush_dcache_page(page);
1020         return 0;
1021 }
1022
1023 /* Copy pages in the request to/from userspace buffer */
1024 static int fuse_copy_pages(struct fuse_copy_state *cs, unsigned nbytes,
1025                            int zeroing)
1026 {
1027         unsigned i;
1028         struct fuse_req *req = cs->req;
1029         struct fuse_args_pages *ap = container_of(req->args, typeof(*ap), args);
1030
1031         for (i = 0; i < ap->num_folios && (nbytes || zeroing); i++) {
1032                 int err;
1033                 unsigned int offset = ap->descs[i].offset;
1034                 unsigned int count = min(nbytes, ap->descs[i].length);
1035                 struct page *orig, *pagep;
1036
1037                 orig = pagep = &ap->folios[i]->page;
1038
1039                 err = fuse_copy_page(cs, &pagep, offset, count, zeroing);
1040                 if (err)
1041                         return err;
1042
1043                 nbytes -= count;
1044
1045                 /*
1046                  *  fuse_copy_page may have moved a page from a pipe instead of
1047                  *  copying into our given page, so update the folios if it was
1048                  *  replaced.
1049                  */
1050                 if (pagep != orig)
1051                         ap->folios[i] = page_folio(pagep);
1052         }
1053         return 0;
1054 }
1055
1056 /* Copy a single argument in the request to/from userspace buffer */
1057 static int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size)
1058 {
1059         while (size) {
1060                 if (!cs->len) {
1061                         int err = fuse_copy_fill(cs);
1062                         if (err)
1063                                 return err;
1064                 }
1065                 fuse_copy_do(cs, &val, &size);
1066         }
1067         return 0;
1068 }
1069
1070 /* Copy request arguments to/from userspace buffer */
1071 static int fuse_copy_args(struct fuse_copy_state *cs, unsigned numargs,
1072                           unsigned argpages, struct fuse_arg *args,
1073                           int zeroing)
1074 {
1075         int err = 0;
1076         unsigned i;
1077
1078         for (i = 0; !err && i < numargs; i++)  {
1079                 struct fuse_arg *arg = &args[i];
1080                 if (i == numargs - 1 && argpages)
1081                         err = fuse_copy_pages(cs, arg->size, zeroing);
1082                 else
1083                         err = fuse_copy_one(cs, arg->value, arg->size);
1084         }
1085         return err;
1086 }
1087
1088 static int forget_pending(struct fuse_iqueue *fiq)
1089 {
1090         return fiq->forget_list_head.next != NULL;
1091 }
1092
1093 static int request_pending(struct fuse_iqueue *fiq)
1094 {
1095         return !list_empty(&fiq->pending) || !list_empty(&fiq->interrupts) ||
1096                 forget_pending(fiq);
1097 }
1098
1099 /*
1100  * Transfer an interrupt request to userspace
1101  *
1102  * Unlike other requests this is assembled on demand, without a need
1103  * to allocate a separate fuse_req structure.
1104  *
1105  * Called with fiq->lock held, releases it
1106  */
1107 static int fuse_read_interrupt(struct fuse_iqueue *fiq,
1108                                struct fuse_copy_state *cs,
1109                                size_t nbytes, struct fuse_req *req)
1110 __releases(fiq->lock)
1111 {
1112         struct fuse_in_header ih;
1113         struct fuse_interrupt_in arg;
1114         unsigned reqsize = sizeof(ih) + sizeof(arg);
1115         int err;
1116
1117         list_del_init(&req->intr_entry);
1118         memset(&ih, 0, sizeof(ih));
1119         memset(&arg, 0, sizeof(arg));
1120         ih.len = reqsize;
1121         ih.opcode = FUSE_INTERRUPT;
1122         ih.unique = (req->in.h.unique | FUSE_INT_REQ_BIT);
1123         arg.unique = req->in.h.unique;
1124
1125         spin_unlock(&fiq->lock);
1126         if (nbytes < reqsize)
1127                 return -EINVAL;
1128
1129         err = fuse_copy_one(cs, &ih, sizeof(ih));
1130         if (!err)
1131                 err = fuse_copy_one(cs, &arg, sizeof(arg));
1132         fuse_copy_finish(cs);
1133
1134         return err ? err : reqsize;
1135 }
1136
1137 static struct fuse_forget_link *fuse_dequeue_forget(struct fuse_iqueue *fiq,
1138                                                     unsigned int max,
1139                                                     unsigned int *countp)
1140 {
1141         struct fuse_forget_link *head = fiq->forget_list_head.next;
1142         struct fuse_forget_link **newhead = &head;
1143         unsigned count;
1144
1145         for (count = 0; *newhead != NULL && count < max; count++)
1146                 newhead = &(*newhead)->next;
1147
1148         fiq->forget_list_head.next = *newhead;
1149         *newhead = NULL;
1150         if (fiq->forget_list_head.next == NULL)
1151                 fiq->forget_list_tail = &fiq->forget_list_head;
1152
1153         if (countp != NULL)
1154                 *countp = count;
1155
1156         return head;
1157 }
1158
1159 static int fuse_read_single_forget(struct fuse_iqueue *fiq,
1160                                    struct fuse_copy_state *cs,
1161                                    size_t nbytes)
1162 __releases(fiq->lock)
1163 {
1164         int err;
1165         struct fuse_forget_link *forget = fuse_dequeue_forget(fiq, 1, NULL);
1166         struct fuse_forget_in arg = {
1167                 .nlookup = forget->forget_one.nlookup,
1168         };
1169         struct fuse_in_header ih = {
1170                 .opcode = FUSE_FORGET,
1171                 .nodeid = forget->forget_one.nodeid,
1172                 .unique = fuse_get_unique_locked(fiq),
1173                 .len = sizeof(ih) + sizeof(arg),
1174         };
1175
1176         spin_unlock(&fiq->lock);
1177         kfree(forget);
1178         if (nbytes < ih.len)
1179                 return -EINVAL;
1180
1181         err = fuse_copy_one(cs, &ih, sizeof(ih));
1182         if (!err)
1183                 err = fuse_copy_one(cs, &arg, sizeof(arg));
1184         fuse_copy_finish(cs);
1185
1186         if (err)
1187                 return err;
1188
1189         return ih.len;
1190 }
1191
1192 static int fuse_read_batch_forget(struct fuse_iqueue *fiq,
1193                                    struct fuse_copy_state *cs, size_t nbytes)
1194 __releases(fiq->lock)
1195 {
1196         int err;
1197         unsigned max_forgets;
1198         unsigned count;
1199         struct fuse_forget_link *head;
1200         struct fuse_batch_forget_in arg = { .count = 0 };
1201         struct fuse_in_header ih = {
1202                 .opcode = FUSE_BATCH_FORGET,
1203                 .unique = fuse_get_unique_locked(fiq),
1204                 .len = sizeof(ih) + sizeof(arg),
1205         };
1206
1207         if (nbytes < ih.len) {
1208                 spin_unlock(&fiq->lock);
1209                 return -EINVAL;
1210         }
1211
1212         max_forgets = (nbytes - ih.len) / sizeof(struct fuse_forget_one);
1213         head = fuse_dequeue_forget(fiq, max_forgets, &count);
1214         spin_unlock(&fiq->lock);
1215
1216         arg.count = count;
1217         ih.len += count * sizeof(struct fuse_forget_one);
1218         err = fuse_copy_one(cs, &ih, sizeof(ih));
1219         if (!err)
1220                 err = fuse_copy_one(cs, &arg, sizeof(arg));
1221
1222         while (head) {
1223                 struct fuse_forget_link *forget = head;
1224
1225                 if (!err) {
1226                         err = fuse_copy_one(cs, &forget->forget_one,
1227                                             sizeof(forget->forget_one));
1228                 }
1229                 head = forget->next;
1230                 kfree(forget);
1231         }
1232
1233         fuse_copy_finish(cs);
1234
1235         if (err)
1236                 return err;
1237
1238         return ih.len;
1239 }
1240
1241 static int fuse_read_forget(struct fuse_conn *fc, struct fuse_iqueue *fiq,
1242                             struct fuse_copy_state *cs,
1243                             size_t nbytes)
1244 __releases(fiq->lock)
1245 {
1246         if (fc->minor < 16 || fiq->forget_list_head.next->next == NULL)
1247                 return fuse_read_single_forget(fiq, cs, nbytes);
1248         else
1249                 return fuse_read_batch_forget(fiq, cs, nbytes);
1250 }
1251
1252 /*
1253  * Read a single request into the userspace filesystem's buffer.  This
1254  * function waits until a request is available, then removes it from
1255  * the pending list and copies request data to userspace buffer.  If
1256  * no reply is needed (FORGET) or request has been aborted or there
1257  * was an error during the copying then it's finished by calling
1258  * fuse_request_end().  Otherwise add it to the processing list, and set
1259  * the 'sent' flag.
1260  */
1261 static ssize_t fuse_dev_do_read(struct fuse_dev *fud, struct file *file,
1262                                 struct fuse_copy_state *cs, size_t nbytes)
1263 {
1264         ssize_t err;
1265         struct fuse_conn *fc = fud->fc;
1266         struct fuse_iqueue *fiq = &fc->iq;
1267         struct fuse_pqueue *fpq = &fud->pq;
1268         struct fuse_req *req;
1269         struct fuse_args *args;
1270         unsigned reqsize;
1271         unsigned int hash;
1272
1273         /*
1274          * Require sane minimum read buffer - that has capacity for fixed part
1275          * of any request header + negotiated max_write room for data.
1276          *
1277          * Historically libfuse reserves 4K for fixed header room, but e.g.
1278          * GlusterFS reserves only 80 bytes
1279          *
1280          *      = `sizeof(fuse_in_header) + sizeof(fuse_write_in)`
1281          *
1282          * which is the absolute minimum any sane filesystem should be using
1283          * for header room.
1284          */
1285         if (nbytes < max_t(size_t, FUSE_MIN_READ_BUFFER,
1286                            sizeof(struct fuse_in_header) +
1287                            sizeof(struct fuse_write_in) +
1288                            fc->max_write))
1289                 return -EINVAL;
1290
1291  restart:
1292         for (;;) {
1293                 spin_lock(&fiq->lock);
1294                 if (!fiq->connected || request_pending(fiq))
1295                         break;
1296                 spin_unlock(&fiq->lock);
1297
1298                 if (file->f_flags & O_NONBLOCK)
1299                         return -EAGAIN;
1300                 err = wait_event_interruptible_exclusive(fiq->waitq,
1301                                 !fiq->connected || request_pending(fiq));
1302                 if (err)
1303                         return err;
1304         }
1305
1306         if (!fiq->connected) {
1307                 err = fc->aborted ? -ECONNABORTED : -ENODEV;
1308                 goto err_unlock;
1309         }
1310
1311         if (!list_empty(&fiq->interrupts)) {
1312                 req = list_entry(fiq->interrupts.next, struct fuse_req,
1313                                  intr_entry);
1314                 return fuse_read_interrupt(fiq, cs, nbytes, req);
1315         }
1316
1317         if (forget_pending(fiq)) {
1318                 if (list_empty(&fiq->pending) || fiq->forget_batch-- > 0)
1319                         return fuse_read_forget(fc, fiq, cs, nbytes);
1320
1321                 if (fiq->forget_batch <= -8)
1322                         fiq->forget_batch = 16;
1323         }
1324
1325         req = list_entry(fiq->pending.next, struct fuse_req, list);
1326         clear_bit(FR_PENDING, &req->flags);
1327         list_del_init(&req->list);
1328         spin_unlock(&fiq->lock);
1329
1330         args = req->args;
1331         reqsize = req->in.h.len;
1332
1333         /* If request is too large, reply with an error and restart the read */
1334         if (nbytes < reqsize) {
1335                 req->out.h.error = -EIO;
1336                 /* SETXATTR is special, since it may contain too large data */
1337                 if (args->opcode == FUSE_SETXATTR)
1338                         req->out.h.error = -E2BIG;
1339                 fuse_request_end(req);
1340                 goto restart;
1341         }
1342         spin_lock(&fpq->lock);
1343         /*
1344          *  Must not put request on fpq->io queue after having been shut down by
1345          *  fuse_abort_conn()
1346          */
1347         if (!fpq->connected) {
1348                 req->out.h.error = err = -ECONNABORTED;
1349                 goto out_end;
1350
1351         }
1352         list_add(&req->list, &fpq->io);
1353         spin_unlock(&fpq->lock);
1354         cs->req = req;
1355         err = fuse_copy_one(cs, &req->in.h, sizeof(req->in.h));
1356         if (!err)
1357                 err = fuse_copy_args(cs, args->in_numargs, args->in_pages,
1358                                      (struct fuse_arg *) args->in_args, 0);
1359         fuse_copy_finish(cs);
1360         spin_lock(&fpq->lock);
1361         clear_bit(FR_LOCKED, &req->flags);
1362         if (!fpq->connected) {
1363                 err = fc->aborted ? -ECONNABORTED : -ENODEV;
1364                 goto out_end;
1365         }
1366         if (err) {
1367                 req->out.h.error = -EIO;
1368                 goto out_end;
1369         }
1370         if (!test_bit(FR_ISREPLY, &req->flags)) {
1371                 err = reqsize;
1372                 goto out_end;
1373         }
1374         hash = fuse_req_hash(req->in.h.unique);
1375         list_move_tail(&req->list, &fpq->processing[hash]);
1376         __fuse_get_request(req);
1377         set_bit(FR_SENT, &req->flags);
1378         spin_unlock(&fpq->lock);
1379         /* matches barrier in request_wait_answer() */
1380         smp_mb__after_atomic();
1381         if (test_bit(FR_INTERRUPTED, &req->flags))
1382                 queue_interrupt(req);
1383         fuse_put_request(req);
1384
1385         return reqsize;
1386
1387 out_end:
1388         if (!test_bit(FR_PRIVATE, &req->flags))
1389                 list_del_init(&req->list);
1390         spin_unlock(&fpq->lock);
1391         fuse_request_end(req);
1392         return err;
1393
1394  err_unlock:
1395         spin_unlock(&fiq->lock);
1396         return err;
1397 }
1398
1399 static int fuse_dev_open(struct inode *inode, struct file *file)
1400 {
1401         /*
1402          * The fuse device's file's private_data is used to hold
1403          * the fuse_conn(ection) when it is mounted, and is used to
1404          * keep track of whether the file has been mounted already.
1405          */
1406         file->private_data = NULL;
1407         return 0;
1408 }
1409
1410 static ssize_t fuse_dev_read(struct kiocb *iocb, struct iov_iter *to)
1411 {
1412         struct fuse_copy_state cs;
1413         struct file *file = iocb->ki_filp;
1414         struct fuse_dev *fud = fuse_get_dev(file);
1415
1416         if (!fud)
1417                 return -EPERM;
1418
1419         if (!user_backed_iter(to))
1420                 return -EINVAL;
1421
1422         fuse_copy_init(&cs, 1, to);
1423
1424         return fuse_dev_do_read(fud, file, &cs, iov_iter_count(to));
1425 }
1426
1427 static ssize_t fuse_dev_splice_read(struct file *in, loff_t *ppos,
1428                                     struct pipe_inode_info *pipe,
1429                                     size_t len, unsigned int flags)
1430 {
1431         int total, ret;
1432         int page_nr = 0;
1433         struct pipe_buffer *bufs;
1434         struct fuse_copy_state cs;
1435         struct fuse_dev *fud = fuse_get_dev(in);
1436
1437         if (!fud)
1438                 return -EPERM;
1439
1440         bufs = kvmalloc_array(pipe->max_usage, sizeof(struct pipe_buffer),
1441                               GFP_KERNEL);
1442         if (!bufs)
1443                 return -ENOMEM;
1444
1445         fuse_copy_init(&cs, 1, NULL);
1446         cs.pipebufs = bufs;
1447         cs.pipe = pipe;
1448         ret = fuse_dev_do_read(fud, in, &cs, len);
1449         if (ret < 0)
1450                 goto out;
1451
1452         if (pipe_occupancy(pipe->head, pipe->tail) + cs.nr_segs > pipe->max_usage) {
1453                 ret = -EIO;
1454                 goto out;
1455         }
1456
1457         for (ret = total = 0; page_nr < cs.nr_segs; total += ret) {
1458                 /*
1459                  * Need to be careful about this.  Having buf->ops in module
1460                  * code can Oops if the buffer persists after module unload.
1461                  */
1462                 bufs[page_nr].ops = &nosteal_pipe_buf_ops;
1463                 bufs[page_nr].flags = 0;
1464                 ret = add_to_pipe(pipe, &bufs[page_nr++]);
1465                 if (unlikely(ret < 0))
1466                         break;
1467         }
1468         if (total)
1469                 ret = total;
1470 out:
1471         for (; page_nr < cs.nr_segs; page_nr++)
1472                 put_page(bufs[page_nr].page);
1473
1474         kvfree(bufs);
1475         return ret;
1476 }
1477
1478 static int fuse_notify_poll(struct fuse_conn *fc, unsigned int size,
1479                             struct fuse_copy_state *cs)
1480 {
1481         struct fuse_notify_poll_wakeup_out outarg;
1482         int err = -EINVAL;
1483
1484         if (size != sizeof(outarg))
1485                 goto err;
1486
1487         err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1488         if (err)
1489                 goto err;
1490
1491         fuse_copy_finish(cs);
1492         return fuse_notify_poll_wakeup(fc, &outarg);
1493
1494 err:
1495         fuse_copy_finish(cs);
1496         return err;
1497 }
1498
1499 static int fuse_notify_inval_inode(struct fuse_conn *fc, unsigned int size,
1500                                    struct fuse_copy_state *cs)
1501 {
1502         struct fuse_notify_inval_inode_out outarg;
1503         int err = -EINVAL;
1504
1505         if (size != sizeof(outarg))
1506                 goto err;
1507
1508         err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1509         if (err)
1510                 goto err;
1511         fuse_copy_finish(cs);
1512
1513         down_read(&fc->killsb);
1514         err = fuse_reverse_inval_inode(fc, outarg.ino,
1515                                        outarg.off, outarg.len);
1516         up_read(&fc->killsb);
1517         return err;
1518
1519 err:
1520         fuse_copy_finish(cs);
1521         return err;
1522 }
1523
1524 static int fuse_notify_inval_entry(struct fuse_conn *fc, unsigned int size,
1525                                    struct fuse_copy_state *cs)
1526 {
1527         struct fuse_notify_inval_entry_out outarg;
1528         int err = -ENOMEM;
1529         char *buf;
1530         struct qstr name;
1531
1532         buf = kzalloc(FUSE_NAME_MAX + 1, GFP_KERNEL);
1533         if (!buf)
1534                 goto err;
1535
1536         err = -EINVAL;
1537         if (size < sizeof(outarg))
1538                 goto err;
1539
1540         err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1541         if (err)
1542                 goto err;
1543
1544         err = -ENAMETOOLONG;
1545         if (outarg.namelen > FUSE_NAME_MAX)
1546                 goto err;
1547
1548         err = -EINVAL;
1549         if (size != sizeof(outarg) + outarg.namelen + 1)
1550                 goto err;
1551
1552         name.name = buf;
1553         name.len = outarg.namelen;
1554         err = fuse_copy_one(cs, buf, outarg.namelen + 1);
1555         if (err)
1556                 goto err;
1557         fuse_copy_finish(cs);
1558         buf[outarg.namelen] = 0;
1559
1560         down_read(&fc->killsb);
1561         err = fuse_reverse_inval_entry(fc, outarg.parent, 0, &name, outarg.flags);
1562         up_read(&fc->killsb);
1563         kfree(buf);
1564         return err;
1565
1566 err:
1567         kfree(buf);
1568         fuse_copy_finish(cs);
1569         return err;
1570 }
1571
1572 static int fuse_notify_delete(struct fuse_conn *fc, unsigned int size,
1573                               struct fuse_copy_state *cs)
1574 {
1575         struct fuse_notify_delete_out outarg;
1576         int err = -ENOMEM;
1577         char *buf;
1578         struct qstr name;
1579
1580         buf = kzalloc(FUSE_NAME_MAX + 1, GFP_KERNEL);
1581         if (!buf)
1582                 goto err;
1583
1584         err = -EINVAL;
1585         if (size < sizeof(outarg))
1586                 goto err;
1587
1588         err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1589         if (err)
1590                 goto err;
1591
1592         err = -ENAMETOOLONG;
1593         if (outarg.namelen > FUSE_NAME_MAX)
1594                 goto err;
1595
1596         err = -EINVAL;
1597         if (size != sizeof(outarg) + outarg.namelen + 1)
1598                 goto err;
1599
1600         name.name = buf;
1601         name.len = outarg.namelen;
1602         err = fuse_copy_one(cs, buf, outarg.namelen + 1);
1603         if (err)
1604                 goto err;
1605         fuse_copy_finish(cs);
1606         buf[outarg.namelen] = 0;
1607
1608         down_read(&fc->killsb);
1609         err = fuse_reverse_inval_entry(fc, outarg.parent, outarg.child, &name, 0);
1610         up_read(&fc->killsb);
1611         kfree(buf);
1612         return err;
1613
1614 err:
1615         kfree(buf);
1616         fuse_copy_finish(cs);
1617         return err;
1618 }
1619
1620 static int fuse_notify_store(struct fuse_conn *fc, unsigned int size,
1621                              struct fuse_copy_state *cs)
1622 {
1623         struct fuse_notify_store_out outarg;
1624         struct inode *inode;
1625         struct address_space *mapping;
1626         u64 nodeid;
1627         int err;
1628         pgoff_t index;
1629         unsigned int offset;
1630         unsigned int num;
1631         loff_t file_size;
1632         loff_t end;
1633
1634         err = -EINVAL;
1635         if (size < sizeof(outarg))
1636                 goto out_finish;
1637
1638         err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1639         if (err)
1640                 goto out_finish;
1641
1642         err = -EINVAL;
1643         if (size - sizeof(outarg) != outarg.size)
1644                 goto out_finish;
1645
1646         nodeid = outarg.nodeid;
1647
1648         down_read(&fc->killsb);
1649
1650         err = -ENOENT;
1651         inode = fuse_ilookup(fc, nodeid,  NULL);
1652         if (!inode)
1653                 goto out_up_killsb;
1654
1655         mapping = inode->i_mapping;
1656         index = outarg.offset >> PAGE_SHIFT;
1657         offset = outarg.offset & ~PAGE_MASK;
1658         file_size = i_size_read(inode);
1659         end = outarg.offset + outarg.size;
1660         if (end > file_size) {
1661                 file_size = end;
1662                 fuse_write_update_attr(inode, file_size, outarg.size);
1663         }
1664
1665         num = outarg.size;
1666         while (num) {
1667                 struct folio *folio;
1668                 struct page *page;
1669                 unsigned int this_num;
1670
1671                 folio = filemap_grab_folio(mapping, index);
1672                 err = PTR_ERR(folio);
1673                 if (IS_ERR(folio))
1674                         goto out_iput;
1675
1676                 page = &folio->page;
1677                 this_num = min_t(unsigned, num, folio_size(folio) - offset);
1678                 err = fuse_copy_page(cs, &page, offset, this_num, 0);
1679                 if (!folio_test_uptodate(folio) && !err && offset == 0 &&
1680                     (this_num == folio_size(folio) || file_size == end)) {
1681                         folio_zero_segment(folio, this_num, folio_size(folio));
1682                         folio_mark_uptodate(folio);
1683                 }
1684                 folio_unlock(folio);
1685                 folio_put(folio);
1686
1687                 if (err)
1688                         goto out_iput;
1689
1690                 num -= this_num;
1691                 offset = 0;
1692                 index++;
1693         }
1694
1695         err = 0;
1696
1697 out_iput:
1698         iput(inode);
1699 out_up_killsb:
1700         up_read(&fc->killsb);
1701 out_finish:
1702         fuse_copy_finish(cs);
1703         return err;
1704 }
1705
1706 struct fuse_retrieve_args {
1707         struct fuse_args_pages ap;
1708         struct fuse_notify_retrieve_in inarg;
1709 };
1710
1711 static void fuse_retrieve_end(struct fuse_mount *fm, struct fuse_args *args,
1712                               int error)
1713 {
1714         struct fuse_retrieve_args *ra =
1715                 container_of(args, typeof(*ra), ap.args);
1716
1717         release_pages(ra->ap.folios, ra->ap.num_folios);
1718         kfree(ra);
1719 }
1720
1721 static int fuse_retrieve(struct fuse_mount *fm, struct inode *inode,
1722                          struct fuse_notify_retrieve_out *outarg)
1723 {
1724         int err;
1725         struct address_space *mapping = inode->i_mapping;
1726         pgoff_t index;
1727         loff_t file_size;
1728         unsigned int num;
1729         unsigned int offset;
1730         size_t total_len = 0;
1731         unsigned int num_pages, cur_pages = 0;
1732         struct fuse_conn *fc = fm->fc;
1733         struct fuse_retrieve_args *ra;
1734         size_t args_size = sizeof(*ra);
1735         struct fuse_args_pages *ap;
1736         struct fuse_args *args;
1737
1738         offset = outarg->offset & ~PAGE_MASK;
1739         file_size = i_size_read(inode);
1740
1741         num = min(outarg->size, fc->max_write);
1742         if (outarg->offset > file_size)
1743                 num = 0;
1744         else if (outarg->offset + num > file_size)
1745                 num = file_size - outarg->offset;
1746
1747         num_pages = (num + offset + PAGE_SIZE - 1) >> PAGE_SHIFT;
1748         num_pages = min(num_pages, fc->max_pages);
1749
1750         args_size += num_pages * (sizeof(ap->folios[0]) + sizeof(ap->descs[0]));
1751
1752         ra = kzalloc(args_size, GFP_KERNEL);
1753         if (!ra)
1754                 return -ENOMEM;
1755
1756         ap = &ra->ap;
1757         ap->folios = (void *) (ra + 1);
1758         ap->descs = (void *) (ap->folios + num_pages);
1759
1760         args = &ap->args;
1761         args->nodeid = outarg->nodeid;
1762         args->opcode = FUSE_NOTIFY_REPLY;
1763         args->in_numargs = 2;
1764         args->in_pages = true;
1765         args->end = fuse_retrieve_end;
1766
1767         index = outarg->offset >> PAGE_SHIFT;
1768
1769         while (num && cur_pages < num_pages) {
1770                 struct folio *folio;
1771                 unsigned int this_num;
1772
1773                 folio = filemap_get_folio(mapping, index);
1774                 if (IS_ERR(folio))
1775                         break;
1776
1777                 this_num = min_t(unsigned, num, PAGE_SIZE - offset);
1778                 ap->folios[ap->num_folios] = folio;
1779                 ap->descs[ap->num_folios].offset = offset;
1780                 ap->descs[ap->num_folios].length = this_num;
1781                 ap->num_folios++;
1782                 cur_pages++;
1783
1784                 offset = 0;
1785                 num -= this_num;
1786                 total_len += this_num;
1787                 index++;
1788         }
1789         ra->inarg.offset = outarg->offset;
1790         ra->inarg.size = total_len;
1791         args->in_args[0].size = sizeof(ra->inarg);
1792         args->in_args[0].value = &ra->inarg;
1793         args->in_args[1].size = total_len;
1794
1795         err = fuse_simple_notify_reply(fm, args, outarg->notify_unique);
1796         if (err)
1797                 fuse_retrieve_end(fm, args, err);
1798
1799         return err;
1800 }
1801
1802 static int fuse_notify_retrieve(struct fuse_conn *fc, unsigned int size,
1803                                 struct fuse_copy_state *cs)
1804 {
1805         struct fuse_notify_retrieve_out outarg;
1806         struct fuse_mount *fm;
1807         struct inode *inode;
1808         u64 nodeid;
1809         int err;
1810
1811         err = -EINVAL;
1812         if (size != sizeof(outarg))
1813                 goto copy_finish;
1814
1815         err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1816         if (err)
1817                 goto copy_finish;
1818
1819         fuse_copy_finish(cs);
1820
1821         down_read(&fc->killsb);
1822         err = -ENOENT;
1823         nodeid = outarg.nodeid;
1824
1825         inode = fuse_ilookup(fc, nodeid, &fm);
1826         if (inode) {
1827                 err = fuse_retrieve(fm, inode, &outarg);
1828                 iput(inode);
1829         }
1830         up_read(&fc->killsb);
1831
1832         return err;
1833
1834 copy_finish:
1835         fuse_copy_finish(cs);
1836         return err;
1837 }
1838
1839 /*
1840  * Resending all processing queue requests.
1841  *
1842  * During a FUSE daemon panics and failover, it is possible for some inflight
1843  * requests to be lost and never returned. As a result, applications awaiting
1844  * replies would become stuck forever. To address this, we can use notification
1845  * to trigger resending of these pending requests to the FUSE daemon, ensuring
1846  * they are properly processed again.
1847  *
1848  * Please note that this strategy is applicable only to idempotent requests or
1849  * if the FUSE daemon takes careful measures to avoid processing duplicated
1850  * non-idempotent requests.
1851  */
1852 static void fuse_resend(struct fuse_conn *fc)
1853 {
1854         struct fuse_dev *fud;
1855         struct fuse_req *req, *next;
1856         struct fuse_iqueue *fiq = &fc->iq;
1857         LIST_HEAD(to_queue);
1858         unsigned int i;
1859
1860         spin_lock(&fc->lock);
1861         if (!fc->connected) {
1862                 spin_unlock(&fc->lock);
1863                 return;
1864         }
1865
1866         list_for_each_entry(fud, &fc->devices, entry) {
1867                 struct fuse_pqueue *fpq = &fud->pq;
1868
1869                 spin_lock(&fpq->lock);
1870                 for (i = 0; i < FUSE_PQ_HASH_SIZE; i++)
1871                         list_splice_tail_init(&fpq->processing[i], &to_queue);
1872                 spin_unlock(&fpq->lock);
1873         }
1874         spin_unlock(&fc->lock);
1875
1876         list_for_each_entry_safe(req, next, &to_queue, list) {
1877                 set_bit(FR_PENDING, &req->flags);
1878                 clear_bit(FR_SENT, &req->flags);
1879                 /* mark the request as resend request */
1880                 req->in.h.unique |= FUSE_UNIQUE_RESEND;
1881         }
1882
1883         spin_lock(&fiq->lock);
1884         if (!fiq->connected) {
1885                 spin_unlock(&fiq->lock);
1886                 list_for_each_entry(req, &to_queue, list)
1887                         clear_bit(FR_PENDING, &req->flags);
1888                 end_requests(&to_queue);
1889                 return;
1890         }
1891         /* iq and pq requests are both oldest to newest */
1892         list_splice(&to_queue, &fiq->pending);
1893         fuse_dev_wake_and_unlock(fiq);
1894 }
1895
1896 static int fuse_notify_resend(struct fuse_conn *fc)
1897 {
1898         fuse_resend(fc);
1899         return 0;
1900 }
1901
1902 static int fuse_notify(struct fuse_conn *fc, enum fuse_notify_code code,
1903                        unsigned int size, struct fuse_copy_state *cs)
1904 {
1905         /* Don't try to move pages (yet) */
1906         cs->move_pages = 0;
1907
1908         switch (code) {
1909         case FUSE_NOTIFY_POLL:
1910                 return fuse_notify_poll(fc, size, cs);
1911
1912         case FUSE_NOTIFY_INVAL_INODE:
1913                 return fuse_notify_inval_inode(fc, size, cs);
1914
1915         case FUSE_NOTIFY_INVAL_ENTRY:
1916                 return fuse_notify_inval_entry(fc, size, cs);
1917
1918         case FUSE_NOTIFY_STORE:
1919                 return fuse_notify_store(fc, size, cs);
1920
1921         case FUSE_NOTIFY_RETRIEVE:
1922                 return fuse_notify_retrieve(fc, size, cs);
1923
1924         case FUSE_NOTIFY_DELETE:
1925                 return fuse_notify_delete(fc, size, cs);
1926
1927         case FUSE_NOTIFY_RESEND:
1928                 return fuse_notify_resend(fc);
1929
1930         default:
1931                 fuse_copy_finish(cs);
1932                 return -EINVAL;
1933         }
1934 }
1935
1936 /* Look up request on processing list by unique ID */
1937 static struct fuse_req *request_find(struct fuse_pqueue *fpq, u64 unique)
1938 {
1939         unsigned int hash = fuse_req_hash(unique);
1940         struct fuse_req *req;
1941
1942         list_for_each_entry(req, &fpq->processing[hash], list) {
1943                 if (req->in.h.unique == unique)
1944                         return req;
1945         }
1946         return NULL;
1947 }
1948
1949 static int copy_out_args(struct fuse_copy_state *cs, struct fuse_args *args,
1950                          unsigned nbytes)
1951 {
1952         unsigned reqsize = sizeof(struct fuse_out_header);
1953
1954         reqsize += fuse_len_args(args->out_numargs, args->out_args);
1955
1956         if (reqsize < nbytes || (reqsize > nbytes && !args->out_argvar))
1957                 return -EINVAL;
1958         else if (reqsize > nbytes) {
1959                 struct fuse_arg *lastarg = &args->out_args[args->out_numargs-1];
1960                 unsigned diffsize = reqsize - nbytes;
1961
1962                 if (diffsize > lastarg->size)
1963                         return -EINVAL;
1964                 lastarg->size -= diffsize;
1965         }
1966         return fuse_copy_args(cs, args->out_numargs, args->out_pages,
1967                               args->out_args, args->page_zeroing);
1968 }
1969
1970 /*
1971  * Write a single reply to a request.  First the header is copied from
1972  * the write buffer.  The request is then searched on the processing
1973  * list by the unique ID found in the header.  If found, then remove
1974  * it from the list and copy the rest of the buffer to the request.
1975  * The request is finished by calling fuse_request_end().
1976  */
1977 static ssize_t fuse_dev_do_write(struct fuse_dev *fud,
1978                                  struct fuse_copy_state *cs, size_t nbytes)
1979 {
1980         int err;
1981         struct fuse_conn *fc = fud->fc;
1982         struct fuse_pqueue *fpq = &fud->pq;
1983         struct fuse_req *req;
1984         struct fuse_out_header oh;
1985
1986         err = -EINVAL;
1987         if (nbytes < sizeof(struct fuse_out_header))
1988                 goto out;
1989
1990         err = fuse_copy_one(cs, &oh, sizeof(oh));
1991         if (err)
1992                 goto copy_finish;
1993
1994         err = -EINVAL;
1995         if (oh.len != nbytes)
1996                 goto copy_finish;
1997
1998         /*
1999          * Zero oh.unique indicates unsolicited notification message
2000          * and error contains notification code.
2001          */
2002         if (!oh.unique) {
2003                 err = fuse_notify(fc, oh.error, nbytes - sizeof(oh), cs);
2004                 goto out;
2005         }
2006
2007         err = -EINVAL;
2008         if (oh.error <= -512 || oh.error > 0)
2009                 goto copy_finish;
2010
2011         spin_lock(&fpq->lock);
2012         req = NULL;
2013         if (fpq->connected)
2014                 req = request_find(fpq, oh.unique & ~FUSE_INT_REQ_BIT);
2015
2016         err = -ENOENT;
2017         if (!req) {
2018                 spin_unlock(&fpq->lock);
2019                 goto copy_finish;
2020         }
2021
2022         /* Is it an interrupt reply ID? */
2023         if (oh.unique & FUSE_INT_REQ_BIT) {
2024                 __fuse_get_request(req);
2025                 spin_unlock(&fpq->lock);
2026
2027                 err = 0;
2028                 if (nbytes != sizeof(struct fuse_out_header))
2029                         err = -EINVAL;
2030                 else if (oh.error == -ENOSYS)
2031                         fc->no_interrupt = 1;
2032                 else if (oh.error == -EAGAIN)
2033                         err = queue_interrupt(req);
2034
2035                 fuse_put_request(req);
2036
2037                 goto copy_finish;
2038         }
2039
2040         clear_bit(FR_SENT, &req->flags);
2041         list_move(&req->list, &fpq->io);
2042         req->out.h = oh;
2043         set_bit(FR_LOCKED, &req->flags);
2044         spin_unlock(&fpq->lock);
2045         cs->req = req;
2046         if (!req->args->page_replace)
2047                 cs->move_pages = 0;
2048
2049         if (oh.error)
2050                 err = nbytes != sizeof(oh) ? -EINVAL : 0;
2051         else
2052                 err = copy_out_args(cs, req->args, nbytes);
2053         fuse_copy_finish(cs);
2054
2055         spin_lock(&fpq->lock);
2056         clear_bit(FR_LOCKED, &req->flags);
2057         if (!fpq->connected)
2058                 err = -ENOENT;
2059         else if (err)
2060                 req->out.h.error = -EIO;
2061         if (!test_bit(FR_PRIVATE, &req->flags))
2062                 list_del_init(&req->list);
2063         spin_unlock(&fpq->lock);
2064
2065         fuse_request_end(req);
2066 out:
2067         return err ? err : nbytes;
2068
2069 copy_finish:
2070         fuse_copy_finish(cs);
2071         goto out;
2072 }
2073
2074 static ssize_t fuse_dev_write(struct kiocb *iocb, struct iov_iter *from)
2075 {
2076         struct fuse_copy_state cs;
2077         struct fuse_dev *fud = fuse_get_dev(iocb->ki_filp);
2078
2079         if (!fud)
2080                 return -EPERM;
2081
2082         if (!user_backed_iter(from))
2083                 return -EINVAL;
2084
2085         fuse_copy_init(&cs, 0, from);
2086
2087         return fuse_dev_do_write(fud, &cs, iov_iter_count(from));
2088 }
2089
2090 static ssize_t fuse_dev_splice_write(struct pipe_inode_info *pipe,
2091                                      struct file *out, loff_t *ppos,
2092                                      size_t len, unsigned int flags)
2093 {
2094         unsigned int head, tail, mask, count;
2095         unsigned nbuf;
2096         unsigned idx;
2097         struct pipe_buffer *bufs;
2098         struct fuse_copy_state cs;
2099         struct fuse_dev *fud;
2100         size_t rem;
2101         ssize_t ret;
2102
2103         fud = fuse_get_dev(out);
2104         if (!fud)
2105                 return -EPERM;
2106
2107         pipe_lock(pipe);
2108
2109         head = pipe->head;
2110         tail = pipe->tail;
2111         mask = pipe->ring_size - 1;
2112         count = head - tail;
2113
2114         bufs = kvmalloc_array(count, sizeof(struct pipe_buffer), GFP_KERNEL);
2115         if (!bufs) {
2116                 pipe_unlock(pipe);
2117                 return -ENOMEM;
2118         }
2119
2120         nbuf = 0;
2121         rem = 0;
2122         for (idx = tail; idx != head && rem < len; idx++)
2123                 rem += pipe->bufs[idx & mask].len;
2124
2125         ret = -EINVAL;
2126         if (rem < len)
2127                 goto out_free;
2128
2129         rem = len;
2130         while (rem) {
2131                 struct pipe_buffer *ibuf;
2132                 struct pipe_buffer *obuf;
2133
2134                 if (WARN_ON(nbuf >= count || tail == head))
2135                         goto out_free;
2136
2137                 ibuf = &pipe->bufs[tail & mask];
2138                 obuf = &bufs[nbuf];
2139
2140                 if (rem >= ibuf->len) {
2141                         *obuf = *ibuf;
2142                         ibuf->ops = NULL;
2143                         tail++;
2144                         pipe->tail = tail;
2145                 } else {
2146                         if (!pipe_buf_get(pipe, ibuf))
2147                                 goto out_free;
2148
2149                         *obuf = *ibuf;
2150                         obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
2151                         obuf->len = rem;
2152                         ibuf->offset += obuf->len;
2153                         ibuf->len -= obuf->len;
2154                 }
2155                 nbuf++;
2156                 rem -= obuf->len;
2157         }
2158         pipe_unlock(pipe);
2159
2160         fuse_copy_init(&cs, 0, NULL);
2161         cs.pipebufs = bufs;
2162         cs.nr_segs = nbuf;
2163         cs.pipe = pipe;
2164
2165         if (flags & SPLICE_F_MOVE)
2166                 cs.move_pages = 1;
2167
2168         ret = fuse_dev_do_write(fud, &cs, len);
2169
2170         pipe_lock(pipe);
2171 out_free:
2172         for (idx = 0; idx < nbuf; idx++) {
2173                 struct pipe_buffer *buf = &bufs[idx];
2174
2175                 if (buf->ops)
2176                         pipe_buf_release(pipe, buf);
2177         }
2178         pipe_unlock(pipe);
2179
2180         kvfree(bufs);
2181         return ret;
2182 }
2183
2184 static __poll_t fuse_dev_poll(struct file *file, poll_table *wait)
2185 {
2186         __poll_t mask = EPOLLOUT | EPOLLWRNORM;
2187         struct fuse_iqueue *fiq;
2188         struct fuse_dev *fud = fuse_get_dev(file);
2189
2190         if (!fud)
2191                 return EPOLLERR;
2192
2193         fiq = &fud->fc->iq;
2194         poll_wait(file, &fiq->waitq, wait);
2195
2196         spin_lock(&fiq->lock);
2197         if (!fiq->connected)
2198                 mask = EPOLLERR;
2199         else if (request_pending(fiq))
2200                 mask |= EPOLLIN | EPOLLRDNORM;
2201         spin_unlock(&fiq->lock);
2202
2203         return mask;
2204 }
2205
2206 /* Abort all requests on the given list (pending or processing) */
2207 static void end_requests(struct list_head *head)
2208 {
2209         while (!list_empty(head)) {
2210                 struct fuse_req *req;
2211                 req = list_entry(head->next, struct fuse_req, list);
2212                 req->out.h.error = -ECONNABORTED;
2213                 clear_bit(FR_SENT, &req->flags);
2214                 list_del_init(&req->list);
2215                 fuse_request_end(req);
2216         }
2217 }
2218
2219 static void end_polls(struct fuse_conn *fc)
2220 {
2221         struct rb_node *p;
2222
2223         p = rb_first(&fc->polled_files);
2224
2225         while (p) {
2226                 struct fuse_file *ff;
2227                 ff = rb_entry(p, struct fuse_file, polled_node);
2228                 wake_up_interruptible_all(&ff->poll_wait);
2229
2230                 p = rb_next(p);
2231         }
2232 }
2233
2234 /*
2235  * Abort all requests.
2236  *
2237  * Emergency exit in case of a malicious or accidental deadlock, or just a hung
2238  * filesystem.
2239  *
2240  * The same effect is usually achievable through killing the filesystem daemon
2241  * and all users of the filesystem.  The exception is the combination of an
2242  * asynchronous request and the tricky deadlock (see
2243  * Documentation/filesystems/fuse.rst).
2244  *
2245  * Aborting requests under I/O goes as follows: 1: Separate out unlocked
2246  * requests, they should be finished off immediately.  Locked requests will be
2247  * finished after unlock; see unlock_request(). 2: Finish off the unlocked
2248  * requests.  It is possible that some request will finish before we can.  This
2249  * is OK, the request will in that case be removed from the list before we touch
2250  * it.
2251  */
2252 void fuse_abort_conn(struct fuse_conn *fc)
2253 {
2254         struct fuse_iqueue *fiq = &fc->iq;
2255
2256         spin_lock(&fc->lock);
2257         if (fc->connected) {
2258                 struct fuse_dev *fud;
2259                 struct fuse_req *req, *next;
2260                 LIST_HEAD(to_end);
2261                 unsigned int i;
2262
2263                 /* Background queuing checks fc->connected under bg_lock */
2264                 spin_lock(&fc->bg_lock);
2265                 fc->connected = 0;
2266                 spin_unlock(&fc->bg_lock);
2267
2268                 fuse_set_initialized(fc);
2269                 list_for_each_entry(fud, &fc->devices, entry) {
2270                         struct fuse_pqueue *fpq = &fud->pq;
2271
2272                         spin_lock(&fpq->lock);
2273                         fpq->connected = 0;
2274                         list_for_each_entry_safe(req, next, &fpq->io, list) {
2275                                 req->out.h.error = -ECONNABORTED;
2276                                 spin_lock(&req->waitq.lock);
2277                                 set_bit(FR_ABORTED, &req->flags);
2278                                 if (!test_bit(FR_LOCKED, &req->flags)) {
2279                                         set_bit(FR_PRIVATE, &req->flags);
2280                                         __fuse_get_request(req);
2281                                         list_move(&req->list, &to_end);
2282                                 }
2283                                 spin_unlock(&req->waitq.lock);
2284                         }
2285                         for (i = 0; i < FUSE_PQ_HASH_SIZE; i++)
2286                                 list_splice_tail_init(&fpq->processing[i],
2287                                                       &to_end);
2288                         spin_unlock(&fpq->lock);
2289                 }
2290                 spin_lock(&fc->bg_lock);
2291                 fc->blocked = 0;
2292                 fc->max_background = UINT_MAX;
2293                 flush_bg_queue(fc);
2294                 spin_unlock(&fc->bg_lock);
2295
2296                 spin_lock(&fiq->lock);
2297                 fiq->connected = 0;
2298                 list_for_each_entry(req, &fiq->pending, list)
2299                         clear_bit(FR_PENDING, &req->flags);
2300                 list_splice_tail_init(&fiq->pending, &to_end);
2301                 while (forget_pending(fiq))
2302                         kfree(fuse_dequeue_forget(fiq, 1, NULL));
2303                 wake_up_all(&fiq->waitq);
2304                 spin_unlock(&fiq->lock);
2305                 kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
2306                 end_polls(fc);
2307                 wake_up_all(&fc->blocked_waitq);
2308                 spin_unlock(&fc->lock);
2309
2310                 end_requests(&to_end);
2311         } else {
2312                 spin_unlock(&fc->lock);
2313         }
2314 }
2315 EXPORT_SYMBOL_GPL(fuse_abort_conn);
2316
2317 void fuse_wait_aborted(struct fuse_conn *fc)
2318 {
2319         /* matches implicit memory barrier in fuse_drop_waiting() */
2320         smp_mb();
2321         wait_event(fc->blocked_waitq, atomic_read(&fc->num_waiting) == 0);
2322 }
2323
2324 int fuse_dev_release(struct inode *inode, struct file *file)
2325 {
2326         struct fuse_dev *fud = fuse_get_dev(file);
2327
2328         if (fud) {
2329                 struct fuse_conn *fc = fud->fc;
2330                 struct fuse_pqueue *fpq = &fud->pq;
2331                 LIST_HEAD(to_end);
2332                 unsigned int i;
2333
2334                 spin_lock(&fpq->lock);
2335                 WARN_ON(!list_empty(&fpq->io));
2336                 for (i = 0; i < FUSE_PQ_HASH_SIZE; i++)
2337                         list_splice_init(&fpq->processing[i], &to_end);
2338                 spin_unlock(&fpq->lock);
2339
2340                 end_requests(&to_end);
2341
2342                 /* Are we the last open device? */
2343                 if (atomic_dec_and_test(&fc->dev_count)) {
2344                         WARN_ON(fc->iq.fasync != NULL);
2345                         fuse_abort_conn(fc);
2346                 }
2347                 fuse_dev_free(fud);
2348         }
2349         return 0;
2350 }
2351 EXPORT_SYMBOL_GPL(fuse_dev_release);
2352
2353 static int fuse_dev_fasync(int fd, struct file *file, int on)
2354 {
2355         struct fuse_dev *fud = fuse_get_dev(file);
2356
2357         if (!fud)
2358                 return -EPERM;
2359
2360         /* No locking - fasync_helper does its own locking */
2361         return fasync_helper(fd, file, on, &fud->fc->iq.fasync);
2362 }
2363
2364 static int fuse_device_clone(struct fuse_conn *fc, struct file *new)
2365 {
2366         struct fuse_dev *fud;
2367
2368         if (new->private_data)
2369                 return -EINVAL;
2370
2371         fud = fuse_dev_alloc_install(fc);
2372         if (!fud)
2373                 return -ENOMEM;
2374
2375         new->private_data = fud;
2376         atomic_inc(&fc->dev_count);
2377
2378         return 0;
2379 }
2380
2381 static long fuse_dev_ioctl_clone(struct file *file, __u32 __user *argp)
2382 {
2383         int res;
2384         int oldfd;
2385         struct fuse_dev *fud = NULL;
2386
2387         if (get_user(oldfd, argp))
2388                 return -EFAULT;
2389
2390         CLASS(fd, f)(oldfd);
2391         if (fd_empty(f))
2392                 return -EINVAL;
2393
2394         /*
2395          * Check against file->f_op because CUSE
2396          * uses the same ioctl handler.
2397          */
2398         if (fd_file(f)->f_op == file->f_op)
2399                 fud = fuse_get_dev(fd_file(f));
2400
2401         res = -EINVAL;
2402         if (fud) {
2403                 mutex_lock(&fuse_mutex);
2404                 res = fuse_device_clone(fud->fc, file);
2405                 mutex_unlock(&fuse_mutex);
2406         }
2407
2408         return res;
2409 }
2410
2411 static long fuse_dev_ioctl_backing_open(struct file *file,
2412                                         struct fuse_backing_map __user *argp)
2413 {
2414         struct fuse_dev *fud = fuse_get_dev(file);
2415         struct fuse_backing_map map;
2416
2417         if (!fud)
2418                 return -EPERM;
2419
2420         if (!IS_ENABLED(CONFIG_FUSE_PASSTHROUGH))
2421                 return -EOPNOTSUPP;
2422
2423         if (copy_from_user(&map, argp, sizeof(map)))
2424                 return -EFAULT;
2425
2426         return fuse_backing_open(fud->fc, &map);
2427 }
2428
2429 static long fuse_dev_ioctl_backing_close(struct file *file, __u32 __user *argp)
2430 {
2431         struct fuse_dev *fud = fuse_get_dev(file);
2432         int backing_id;
2433
2434         if (!fud)
2435                 return -EPERM;
2436
2437         if (!IS_ENABLED(CONFIG_FUSE_PASSTHROUGH))
2438                 return -EOPNOTSUPP;
2439
2440         if (get_user(backing_id, argp))
2441                 return -EFAULT;
2442
2443         return fuse_backing_close(fud->fc, backing_id);
2444 }
2445
2446 static long fuse_dev_ioctl(struct file *file, unsigned int cmd,
2447                            unsigned long arg)
2448 {
2449         void __user *argp = (void __user *)arg;
2450
2451         switch (cmd) {
2452         case FUSE_DEV_IOC_CLONE:
2453                 return fuse_dev_ioctl_clone(file, argp);
2454
2455         case FUSE_DEV_IOC_BACKING_OPEN:
2456                 return fuse_dev_ioctl_backing_open(file, argp);
2457
2458         case FUSE_DEV_IOC_BACKING_CLOSE:
2459                 return fuse_dev_ioctl_backing_close(file, argp);
2460
2461         default:
2462                 return -ENOTTY;
2463         }
2464 }
2465
2466 const struct file_operations fuse_dev_operations = {
2467         .owner          = THIS_MODULE,
2468         .open           = fuse_dev_open,
2469         .read_iter      = fuse_dev_read,
2470         .splice_read    = fuse_dev_splice_read,
2471         .write_iter     = fuse_dev_write,
2472         .splice_write   = fuse_dev_splice_write,
2473         .poll           = fuse_dev_poll,
2474         .release        = fuse_dev_release,
2475         .fasync         = fuse_dev_fasync,
2476         .unlocked_ioctl = fuse_dev_ioctl,
2477         .compat_ioctl   = compat_ptr_ioctl,
2478 };
2479 EXPORT_SYMBOL_GPL(fuse_dev_operations);
2480
2481 static struct miscdevice fuse_miscdevice = {
2482         .minor = FUSE_MINOR,
2483         .name  = "fuse",
2484         .fops = &fuse_dev_operations,
2485 };
2486
2487 int __init fuse_dev_init(void)
2488 {
2489         int err = -ENOMEM;
2490         fuse_req_cachep = kmem_cache_create("fuse_request",
2491                                             sizeof(struct fuse_req),
2492                                             0, 0, NULL);
2493         if (!fuse_req_cachep)
2494                 goto out;
2495
2496         err = misc_register(&fuse_miscdevice);
2497         if (err)
2498                 goto out_cache_clean;
2499
2500         return 0;
2501
2502  out_cache_clean:
2503         kmem_cache_destroy(fuse_req_cachep);
2504  out:
2505         return err;
2506 }
2507
2508 void fuse_dev_cleanup(void)
2509 {
2510         misc_deregister(&fuse_miscdevice);
2511         kmem_cache_destroy(fuse_req_cachep);
2512 }