fs/fuse/file.c

   1 /*
   2   FUSE: Filesystem in Userspace
   3   Copyright (C) 2001-2008  Miklos Szeredi <[email protected]>
   4
   5   This program can be distributed under the terms of the GNU GPL.
   6   See the file COPYING.
   7 */
   8
   9 #include "fuse_i.h"
  10
  11 #include <linux/pagemap.h>
  12 #include <linux/slab.h>
  13 #include <linux/kernel.h>
  14 #include <linux/sched.h>
  15 #include <linux/sched/signal.h>
  16 #include <linux/module.h>
  17 #include <linux/compat.h>
  18 #include <linux/swap.h>
  19 #include <linux/falloc.h>
  20 #include <linux/uio.h>
  21 #include <linux/fs.h>
  22
  23 static struct page **fuse_pages_alloc(unsigned int npages, gfp_t flags,
  24                                       struct fuse_page_desc **desc)
  25 {
  26         struct page **pages;
  27
  28         pages = kzalloc(npages * (sizeof(struct page *) +
  29                                   sizeof(struct fuse_page_desc)), flags);
  30         *desc = (void *) (pages + npages);
  31
  32         return pages;
  33 }
  34
  35 static int fuse_send_open(struct fuse_mount *fm, u64 nodeid, struct file *file,
  36                           int opcode, struct fuse_open_out *outargp)
  37 {
  38         struct fuse_open_in inarg;
  39         FUSE_ARGS(args);
  40
  41         memset(&inarg, 0, sizeof(inarg));
  42         inarg.flags = file->f_flags & ~(O_CREAT | O_EXCL | O_NOCTTY);
  43         if (!fm->fc->atomic_o_trunc)
  44                 inarg.flags &= ~O_TRUNC;
  45         args.opcode = opcode;
  46         args.nodeid = nodeid;
  47         args.in_numargs = 1;
  48         args.in_args[0].size = sizeof(inarg);
  49         args.in_args[0].value = &inarg;
  50         args.out_numargs = 1;
  51         args.out_args[0].size = sizeof(*outargp);
  52         args.out_args[0].value = outargp;
  53
  54         return fuse_simple_request(fm, &args);
  55 }
  56
  57 struct fuse_release_args {
  58         struct fuse_args args;
  59         struct fuse_release_in inarg;
  60         struct inode *inode;
  61 };
  62
  63 struct fuse_file *fuse_file_alloc(struct fuse_mount *fm)
  64 {
  65         struct fuse_file *ff;
  66
  67         ff = kzalloc(sizeof(struct fuse_file), GFP_KERNEL_ACCOUNT);
  68         if (unlikely(!ff))
  69                 return NULL;
  70
  71         ff->fm = fm;
  72         ff->release_args = kzalloc(sizeof(*ff->release_args),
  73                                    GFP_KERNEL_ACCOUNT);
  74         if (!ff->release_args) {
  75                 kfree(ff);
  76                 return NULL;
  77         }
  78
  79         INIT_LIST_HEAD(&ff->write_entry);
  80         mutex_init(&ff->readdir.lock);
  81         refcount_set(&ff->count, 1);
  82         RB_CLEAR_NODE(&ff->polled_node);
  83         init_waitqueue_head(&ff->poll_wait);
  84
  85         ff->kh = atomic64_inc_return(&fm->fc->khctr);
  86
  87         return ff;
  88 }
  89
  90 void fuse_file_free(struct fuse_file *ff)
  91 {
  92         kfree(ff->release_args);
  93         mutex_destroy(&ff->readdir.lock);
  94         kfree(ff);
  95 }
  96
  97 static struct fuse_file *fuse_file_get(struct fuse_file *ff)
  98 {
  99         refcount_inc(&ff->count);
 100         return ff;
 101 }
 102
 103 static void fuse_release_end(struct fuse_mount *fm, struct fuse_args *args,
 104                              int error)
 105 {
 106         struct fuse_release_args *ra = container_of(args, typeof(*ra), args);
 107
 108         iput(ra->inode);
 109         kfree(ra);
 110 }
 111
 112 static void fuse_file_put(struct fuse_file *ff, bool sync, bool isdir)
 113 {
 114         if (refcount_dec_and_test(&ff->count)) {
 115                 struct fuse_args *args = &ff->release_args->args;
 116
 117                 if (isdir ? ff->fm->fc->no_opendir : ff->fm->fc->no_open) {
 118                         /* Do nothing when client does not implement 'open' */
 119                         fuse_release_end(ff->fm, args, 0);
 120                 } else if (sync) {
 121                         fuse_simple_request(ff->fm, args);
 122                         fuse_release_end(ff->fm, args, 0);
 123                 } else {
 124                         args->end = fuse_release_end;
 125                         if (fuse_simple_background(ff->fm, args,
 126                                                    GFP_KERNEL | __GFP_NOFAIL))
 127                                 fuse_release_end(ff->fm, args, -ENOTCONN);
 128                 }
 129                 kfree(ff);
 130         }
 131 }
 132
 133 int fuse_do_open(struct fuse_mount *fm, u64 nodeid, struct file *file,
 134                  bool isdir)
 135 {
 136         struct fuse_conn *fc = fm->fc;
 137         struct fuse_file *ff;
 138         int opcode = isdir ? FUSE_OPENDIR : FUSE_OPEN;
 139
 140         ff = fuse_file_alloc(fm);
 141         if (!ff)
 142                 return -ENOMEM;
 143
 144         ff->fh = 0;
 145         /* Default for no-open */
 146         ff->open_flags = FOPEN_KEEP_CACHE | (isdir ? FOPEN_CACHE_DIR : 0);
 147         if (isdir ? !fc->no_opendir : !fc->no_open) {
 148                 struct fuse_open_out outarg;
 149                 int err;
 150
 151                 err = fuse_send_open(fm, nodeid, file, opcode, &outarg);
 152                 if (!err) {
 153                         ff->fh = outarg.fh;
 154                         ff->open_flags = outarg.open_flags;
 155
 156                 } else if (err != -ENOSYS) {
 157                         fuse_file_free(ff);
 158                         return err;
 159                 } else {
 160                         if (isdir)
 161                                 fc->no_opendir = 1;
 162                         else
 163                                 fc->no_open = 1;
 164                 }
 165         }
 166
 167         if (isdir)
 168                 ff->open_flags &= ~FOPEN_DIRECT_IO;
 169
 170         ff->nodeid = nodeid;
 171         file->private_data = ff;
 172
 173         return 0;
 174 }
 175 EXPORT_SYMBOL_GPL(fuse_do_open);
 176
 177 static void fuse_link_write_file(struct file *file)
 178 {
 179         struct inode *inode = file_inode(file);
 180         struct fuse_inode *fi = get_fuse_inode(inode);
 181         struct fuse_file *ff = file->private_data;
 182         /*
 183          * file may be written through mmap, so chain it onto the
 184          * inodes's write_file list
 185          */
 186         spin_lock(&fi->lock);
 187         if (list_empty(&ff->write_entry))
 188                 list_add(&ff->write_entry, &fi->write_files);
 189         spin_unlock(&fi->lock);
 190 }
 191
 192 void fuse_finish_open(struct inode *inode, struct file *file)
 193 {
 194         struct fuse_file *ff = file->private_data;
 195         struct fuse_conn *fc = get_fuse_conn(inode);
 196
 197         if (!(ff->open_flags & FOPEN_KEEP_CACHE))
 198                 invalidate_inode_pages2(inode->i_mapping);
 199         if (ff->open_flags & FOPEN_STREAM)
 200                 stream_open(inode, file);
 201         else if (ff->open_flags & FOPEN_NONSEEKABLE)
 202                 nonseekable_open(inode, file);
 203         if (fc->atomic_o_trunc && (file->f_flags & O_TRUNC)) {
 204                 struct fuse_inode *fi = get_fuse_inode(inode);
 205
 206                 spin_lock(&fi->lock);
 207                 fi->attr_version = atomic64_inc_return(&fc->attr_version);
 208                 i_size_write(inode, 0);
 209                 spin_unlock(&fi->lock);
 210                 fuse_invalidate_attr(inode);
 211                 if (fc->writeback_cache)
 212                         file_update_time(file);
 213         }
 214         if ((file->f_mode & FMODE_WRITE) && fc->writeback_cache)
 215                 fuse_link_write_file(file);
 216 }
 217
 218 int fuse_open_common(struct inode *inode, struct file *file, bool isdir)
 219 {
 220         struct fuse_mount *fm = get_fuse_mount(inode);
 221         struct fuse_conn *fc = fm->fc;
 222         int err;
 223         bool is_wb_truncate = (file->f_flags & O_TRUNC) &&
 224                           fc->atomic_o_trunc &&
 225                           fc->writeback_cache;
 226         bool dax_truncate = (file->f_flags & O_TRUNC) &&
 227                           fc->atomic_o_trunc && FUSE_IS_DAX(inode);
 228
 229         err = generic_file_open(inode, file);
 230         if (err)
 231                 return err;
 232
 233         if (is_wb_truncate || dax_truncate) {
 234                 inode_lock(inode);
 235                 fuse_set_nowrite(inode);
 236         }
 237
 238         if (dax_truncate) {
 239                 down_write(&get_fuse_inode(inode)->i_mmap_sem);
 240                 err = fuse_dax_break_layouts(inode, 0, 0);
 241                 if (err)
 242                         goto out;
 243         }
 244
 245         err = fuse_do_open(fm, get_node_id(inode), file, isdir);
 246         if (!err)
 247                 fuse_finish_open(inode, file);
 248
 249 out:
 250         if (dax_truncate)
 251                 up_write(&get_fuse_inode(inode)->i_mmap_sem);
 252
 253         if (is_wb_truncate | dax_truncate) {
 254                 fuse_release_nowrite(inode);
 255                 inode_unlock(inode);
 256         }
 257
 258         return err;
 259 }
 260
 261 static void fuse_prepare_release(struct fuse_inode *fi, struct fuse_file *ff,
 262                                  int flags, int opcode)
 263 {
 264         struct fuse_conn *fc = ff->fm->fc;
 265         struct fuse_release_args *ra = ff->release_args;
 266
 267         /* Inode is NULL on error path of fuse_create_open() */
 268         if (likely(fi)) {
 269                 spin_lock(&fi->lock);
 270                 list_del(&ff->write_entry);
 271                 spin_unlock(&fi->lock);
 272         }
 273         spin_lock(&fc->lock);
 274         if (!RB_EMPTY_NODE(&ff->polled_node))
 275                 rb_erase(&ff->polled_node, &fc->polled_files);
 276         spin_unlock(&fc->lock);
 277
 278         wake_up_interruptible_all(&ff->poll_wait);
 279
 280         ra->inarg.fh = ff->fh;
 281         ra->inarg.flags = flags;
 282         ra->args.in_numargs = 1;
 283         ra->args.in_args[0].size = sizeof(struct fuse_release_in);
 284         ra->args.in_args[0].value = &ra->inarg;
 285         ra->args.opcode = opcode;
 286         ra->args.nodeid = ff->nodeid;
 287         ra->args.force = true;
 288         ra->args.nocreds = true;
 289 }
 290
 291 void fuse_release_common(struct file *file, bool isdir)
 292 {
 293         struct fuse_inode *fi = get_fuse_inode(file_inode(file));
 294         struct fuse_file *ff = file->private_data;
 295         struct fuse_release_args *ra = ff->release_args;
 296         int opcode = isdir ? FUSE_RELEASEDIR : FUSE_RELEASE;
 297
 298         fuse_prepare_release(fi, ff, file->f_flags, opcode);
 299
 300         if (ff->flock) {
 301                 ra->inarg.release_flags |= FUSE_RELEASE_FLOCK_UNLOCK;
 302                 ra->inarg.lock_owner = fuse_lock_owner_id(ff->fm->fc,
 303                                                           (fl_owner_t) file);
 304         }
 305         /* Hold inode until release is finished */
 306         ra->inode = igrab(file_inode(file));
 307
 308         /*
 309          * Normally this will send the RELEASE request, however if
 310          * some asynchronous READ or WRITE requests are outstanding,
 311          * the sending will be delayed.
 312          *
 313          * Make the release synchronous if this is a fuseblk mount,
 314          * synchronous RELEASE is allowed (and desirable) in this case
 315          * because the server can be trusted not to screw up.
 316          */
 317         fuse_file_put(ff, ff->fm->fc->destroy, isdir);
 318 }
 319
 320 static int fuse_open(struct inode *inode, struct file *file)
 321 {
 322         return fuse_open_common(inode, file, false);
 323 }
 324
 325 static int fuse_release(struct inode *inode, struct file *file)
 326 {
 327         struct fuse_conn *fc = get_fuse_conn(inode);
 328
 329         /* see fuse_vma_close() for !writeback_cache case */
 330         if (fc->writeback_cache)
 331                 write_inode_now(inode, 1);
 332
 333         fuse_release_common(file, false);
 334
 335         /* return value is ignored by VFS */
 336         return 0;
 337 }
 338
 339 void fuse_sync_release(struct fuse_inode *fi, struct fuse_file *ff, int flags)
 340 {
 341         WARN_ON(refcount_read(&ff->count) > 1);
 342         fuse_prepare_release(fi, ff, flags, FUSE_RELEASE);
 343         /*
 344          * iput(NULL) is a no-op and since the refcount is 1 and everything's
 345          * synchronous, we are fine with not doing igrab() here"
 346          */
 347         fuse_file_put(ff, true, false);
 348 }
 349 EXPORT_SYMBOL_GPL(fuse_sync_release);
 350
 351 /*
 352  * Scramble the ID space with XTEA, so that the value of the files_struct
 353  * pointer is not exposed to userspace.
 354  */
 355 u64 fuse_lock_owner_id(struct fuse_conn *fc, fl_owner_t id)
 356 {
 357         u32 *k = fc->scramble_key;
 358         u64 v = (unsigned long) id;
 359         u32 v0 = v;
 360         u32 v1 = v >> 32;
 361         u32 sum = 0;
 362         int i;
 363
 364         for (i = 0; i < 32; i++) {
 365                 v0 += ((v1 << 4 ^ v1 >> 5) + v1) ^ (sum + k[sum & 3]);
 366                 sum += 0x9E3779B9;
 367                 v1 += ((v0 << 4 ^ v0 >> 5) + v0) ^ (sum + k[sum>>11 & 3]);
 368         }
 369
 370         return (u64) v0 + ((u64) v1 << 32);
 371 }
 372
 373 struct fuse_writepage_args {
 374         struct fuse_io_args ia;
 375         struct rb_node writepages_entry;
 376         struct list_head queue_entry;
 377         struct fuse_writepage_args *next;
 378         struct inode *inode;
 379 };
 380
 381 static struct fuse_writepage_args *fuse_find_writeback(struct fuse_inode *fi,
 382                                             pgoff_t idx_from, pgoff_t idx_to)
 383 {
 384         struct rb_node *n;
 385
 386         n = fi->writepages.rb_node;
 387
 388         while (n) {
 389                 struct fuse_writepage_args *wpa;
 390                 pgoff_t curr_index;
 391
 392                 wpa = rb_entry(n, struct fuse_writepage_args, writepages_entry);
 393                 WARN_ON(get_fuse_inode(wpa->inode) != fi);
 394                 curr_index = wpa->ia.write.in.offset >> PAGE_SHIFT;
 395                 if (idx_from >= curr_index + wpa->ia.ap.num_pages)
 396                         n = n->rb_right;
 397                 else if (idx_to < curr_index)
 398                         n = n->rb_left;
 399                 else
 400                         return wpa;
 401         }
 402         return NULL;
 403 }
 404
 405 /*
 406  * Check if any page in a range is under writeback
 407  *
 408  * This is currently done by walking the list of writepage requests
 409  * for the inode, which can be pretty inefficient.
 410  */
 411 static bool fuse_range_is_writeback(struct inode *inode, pgoff_t idx_from,
 412                                    pgoff_t idx_to)
 413 {
 414         struct fuse_inode *fi = get_fuse_inode(inode);
 415         bool found;
 416
 417         spin_lock(&fi->lock);
 418         found = fuse_find_writeback(fi, idx_from, idx_to);
 419         spin_unlock(&fi->lock);
 420
 421         return found;
 422 }
 423
 424 static inline bool fuse_page_is_writeback(struct inode *inode, pgoff_t index)
 425 {
 426         return fuse_range_is_writeback(inode, index, index);
 427 }
 428
 429 /*
 430  * Wait for page writeback to be completed.
 431  *
 432  * Since fuse doesn't rely on the VM writeback tracking, this has to
 433  * use some other means.
 434  */
 435 static void fuse_wait_on_page_writeback(struct inode *inode, pgoff_t index)
 436 {
 437         struct fuse_inode *fi = get_fuse_inode(inode);
 438
 439         wait_event(fi->page_waitq, !fuse_page_is_writeback(inode, index));
 440 }
 441
 442 /*
 443  * Wait for all pending writepages on the inode to finish.
 444  *
 445  * This is currently done by blocking further writes with FUSE_NOWRITE
 446  * and waiting for all sent writes to complete.
 447  *
 448  * This must be called under i_mutex, otherwise the FUSE_NOWRITE usage
 449  * could conflict with truncation.
 450  */
 451 static void fuse_sync_writes(struct inode *inode)
 452 {
 453         fuse_set_nowrite(inode);
 454         fuse_release_nowrite(inode);
 455 }
 456
 457 static int fuse_flush(struct file *file, fl_owner_t id)
 458 {
 459         struct inode *inode = file_inode(file);
 460         struct fuse_mount *fm = get_fuse_mount(inode);
 461         struct fuse_file *ff = file->private_data;
 462         struct fuse_flush_in inarg;
 463         FUSE_ARGS(args);
 464         int err;
 465
 466         if (is_bad_inode(inode))
 467                 return -EIO;
 468
 469         err = write_inode_now(inode, 1);
 470         if (err)
 471                 return err;
 472
 473         inode_lock(inode);
 474         fuse_sync_writes(inode);
 475         inode_unlock(inode);
 476
 477         err = filemap_check_errors(file->f_mapping);
 478         if (err)
 479                 return err;
 480
 481         err = 0;
 482         if (fm->fc->no_flush)
 483                 goto inval_attr_out;
 484
 485         memset(&inarg, 0, sizeof(inarg));
 486         inarg.fh = ff->fh;
 487         inarg.lock_owner = fuse_lock_owner_id(fm->fc, id);
 488         args.opcode = FUSE_FLUSH;
 489         args.nodeid = get_node_id(inode);
 490         args.in_numargs = 1;
 491         args.in_args[0].size = sizeof(inarg);
 492         args.in_args[0].value = &inarg;
 493         args.force = true;
 494
 495         err = fuse_simple_request(fm, &args);
 496         if (err == -ENOSYS) {
 497                 fm->fc->no_flush = 1;
 498                 err = 0;
 499         }
 500
 501 inval_attr_out:
 502         /*
 503          * In memory i_blocks is not maintained by fuse, if writeback cache is
 504          * enabled, i_blocks from cached attr may not be accurate.
 505          */
 506         if (!err && fm->fc->writeback_cache)
 507                 fuse_invalidate_attr(inode);
 508         return err;
 509 }
 510
 511 int fuse_fsync_common(struct file *file, loff_t start, loff_t end,
 512                       int datasync, int opcode)
 513 {
 514         struct inode *inode = file->f_mapping->host;
 515         struct fuse_mount *fm = get_fuse_mount(inode);
 516         struct fuse_file *ff = file->private_data;
 517         FUSE_ARGS(args);
 518         struct fuse_fsync_in inarg;
 519
 520         memset(&inarg, 0, sizeof(inarg));
 521         inarg.fh = ff->fh;
 522         inarg.fsync_flags = datasync ? FUSE_FSYNC_FDATASYNC : 0;
 523         args.opcode = opcode;
 524         args.nodeid = get_node_id(inode);
 525         args.in_numargs = 1;
 526         args.in_args[0].size = sizeof(inarg);
 527         args.in_args[0].value = &inarg;
 528         return fuse_simple_request(fm, &args);
 529 }
 530
 531 static int fuse_fsync(struct file *file, loff_t start, loff_t end,
 532                       int datasync)
 533 {
 534         struct inode *inode = file->f_mapping->host;
 535         struct fuse_conn *fc = get_fuse_conn(inode);
 536         int err;
 537
 538         if (is_bad_inode(inode))
 539                 return -EIO;
 540
 541         inode_lock(inode);
 542
 543         /*
 544          * Start writeback against all dirty pages of the inode, then
 545          * wait for all outstanding writes, before sending the FSYNC
 546          * request.
 547          */
 548         err = file_write_and_wait_range(file, start, end);
 549         if (err)
 550                 goto out;
 551
 552         fuse_sync_writes(inode);
 553
 554         /*
 555          * Due to implementation of fuse writeback
 556          * file_write_and_wait_range() does not catch errors.
 557          * We have to do this directly after fuse_sync_writes()
 558          */
 559         err = file_check_and_advance_wb_err(file);
 560         if (err)
 561                 goto out;
 562
 563         err = sync_inode_metadata(inode, 1);
 564         if (err)
 565                 goto out;
 566
 567         if (fc->no_fsync)
 568                 goto out;
 569
 570         err = fuse_fsync_common(file, start, end, datasync, FUSE_FSYNC);
 571         if (err == -ENOSYS) {
 572                 fc->no_fsync = 1;
 573                 err = 0;
 574         }
 575 out:
 576         inode_unlock(inode);
 577
 578         return err;
 579 }
 580
 581 void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos,
 582                          size_t count, int opcode)
 583 {
 584         struct fuse_file *ff = file->private_data;
 585         struct fuse_args *args = &ia->ap.args;
 586
 587         ia->read.in.fh = ff->fh;
 588         ia->read.in.offset = pos;
 589         ia->read.in.size = count;
 590         ia->read.in.flags = file->f_flags;
 591         args->opcode = opcode;
 592         args->nodeid = ff->nodeid;
 593         args->in_numargs = 1;
 594         args->in_args[0].size = sizeof(ia->read.in);
 595         args->in_args[0].value = &ia->read.in;
 596         args->out_argvar = true;
 597         args->out_numargs = 1;
 598         args->out_args[0].size = count;
 599 }
 600
 601 static void fuse_release_user_pages(struct fuse_args_pages *ap,
 602                                     bool should_dirty)
 603 {
 604         unsigned int i;
 605
 606         for (i = 0; i < ap->num_pages; i++) {
 607                 if (should_dirty)
 608                         set_page_dirty_lock(ap->pages[i]);
 609                 put_page(ap->pages[i]);
 610         }
 611 }
 612
 613 static void fuse_io_release(struct kref *kref)
 614 {
 615         kfree(container_of(kref, struct fuse_io_priv, refcnt));
 616 }
 617
 618 static ssize_t fuse_get_res_by_io(struct fuse_io_priv *io)
 619 {
 620         if (io->err)
 621                 return io->err;
 622
 623         if (io->bytes >= 0 && io->write)
 624                 return -EIO;
 625
 626         return io->bytes < 0 ? io->size : io->bytes;
 627 }
 628
 629 /**
 630  * In case of short read, the caller sets 'pos' to the position of
 631  * actual end of fuse request in IO request. Otherwise, if bytes_requested
 632  * == bytes_transferred or rw == WRITE, the caller sets 'pos' to -1.
 633  *
 634  * An example:
 635  * User requested DIO read of 64K. It was splitted into two 32K fuse requests,
 636  * both submitted asynchronously. The first of them was ACKed by userspace as
 637  * fully completed (req->out.args[0].size == 32K) resulting in pos == -1. The
 638  * second request was ACKed as short, e.g. only 1K was read, resulting in
 639  * pos == 33K.
 640  *
 641  * Thus, when all fuse requests are completed, the minimal non-negative 'pos'
 642  * will be equal to the length of the longest contiguous fragment of
 643  * transferred data starting from the beginning of IO request.
 644  */
 645 static void fuse_aio_complete(struct fuse_io_priv *io, int err, ssize_t pos)
 646 {
 647         int left;
 648
 649         spin_lock(&io->lock);
 650         if (err)
 651                 io->err = io->err ? : err;
 652         else if (pos >= 0 && (io->bytes < 0 || pos < io->bytes))
 653                 io->bytes = pos;
 654
 655         left = --io->reqs;
 656         if (!left && io->blocking)
 657                 complete(io->done);
 658         spin_unlock(&io->lock);
 659
 660         if (!left && !io->blocking) {
 661                 ssize_t res = fuse_get_res_by_io(io);
 662
 663                 if (res >= 0) {
 664                         struct inode *inode = file_inode(io->iocb->ki_filp);
 665                         struct fuse_conn *fc = get_fuse_conn(inode);
 666                         struct fuse_inode *fi = get_fuse_inode(inode);
 667
 668                         spin_lock(&fi->lock);
 669                         fi->attr_version = atomic64_inc_return(&fc->attr_version);
 670                         spin_unlock(&fi->lock);
 671                 }
 672
 673                 io->iocb->ki_complete(io->iocb, res, 0);
 674         }
 675
 676         kref_put(&io->refcnt, fuse_io_release);
 677 }
 678
 679 static struct fuse_io_args *fuse_io_alloc(struct fuse_io_priv *io,
 680                                           unsigned int npages)
 681 {
 682         struct fuse_io_args *ia;
 683
 684         ia = kzalloc(sizeof(*ia), GFP_KERNEL);
 685         if (ia) {
 686                 ia->io = io;
 687                 ia->ap.pages = fuse_pages_alloc(npages, GFP_KERNEL,
 688                                                 &ia->ap.descs);
 689                 if (!ia->ap.pages) {
 690                         kfree(ia);
 691                         ia = NULL;
 692                 }
 693         }
 694         return ia;
 695 }
 696
 697 static void fuse_io_free(struct fuse_io_args *ia)
 698 {
 699         kfree(ia->ap.pages);
 700         kfree(ia);
 701 }
 702
 703 static void fuse_aio_complete_req(struct fuse_mount *fm, struct fuse_args *args,
 704                                   int err)
 705 {
 706         struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args);
 707         struct fuse_io_priv *io = ia->io;
 708         ssize_t pos = -1;
 709
 710         fuse_release_user_pages(&ia->ap, io->should_dirty);
 711
 712         if (err) {
 713                 /* Nothing */
 714         } else if (io->write) {
 715                 if (ia->write.out.size > ia->write.in.size) {
 716                         err = -EIO;
 717                 } else if (ia->write.in.size != ia->write.out.size) {
 718                         pos = ia->write.in.offset - io->offset +
 719                                 ia->write.out.size;
 720                 }
 721         } else {
 722                 u32 outsize = args->out_args[0].size;
 723
 724                 if (ia->read.in.size != outsize)
 725                         pos = ia->read.in.offset - io->offset + outsize;
 726         }
 727
 728         fuse_aio_complete(io, err, pos);
 729         fuse_io_free(ia);
 730 }
 731
 732 static ssize_t fuse_async_req_send(struct fuse_mount *fm,
 733                                    struct fuse_io_args *ia, size_t num_bytes)
 734 {
 735         ssize_t err;
 736         struct fuse_io_priv *io = ia->io;
 737
 738         spin_lock(&io->lock);
 739         kref_get(&io->refcnt);
 740         io->size += num_bytes;
 741         io->reqs++;
 742         spin_unlock(&io->lock);
 743
 744         ia->ap.args.end = fuse_aio_complete_req;
 745         ia->ap.args.may_block = io->should_dirty;
 746         err = fuse_simple_background(fm, &ia->ap.args, GFP_KERNEL);
 747         if (err)
 748                 fuse_aio_complete_req(fm, &ia->ap.args, err);
 749
 750         return num_bytes;
 751 }
 752
 753 static ssize_t fuse_send_read(struct fuse_io_args *ia, loff_t pos, size_t count,
 754                               fl_owner_t owner)
 755 {
 756         struct file *file = ia->io->iocb->ki_filp;
 757         struct fuse_file *ff = file->private_data;
 758         struct fuse_mount *fm = ff->fm;
 759
 760         fuse_read_args_fill(ia, file, pos, count, FUSE_READ);
 761         if (owner != NULL) {
 762                 ia->read.in.read_flags |= FUSE_READ_LOCKOWNER;
 763                 ia->read.in.lock_owner = fuse_lock_owner_id(fm->fc, owner);
 764         }
 765
 766         if (ia->io->async)
 767                 return fuse_async_req_send(fm, ia, count);
 768
 769         return fuse_simple_request(fm, &ia->ap.args);
 770 }
 771
 772 static void fuse_read_update_size(struct inode *inode, loff_t size,
 773                                   u64 attr_ver)
 774 {
 775         struct fuse_conn *fc = get_fuse_conn(inode);
 776         struct fuse_inode *fi = get_fuse_inode(inode);
 777
 778         spin_lock(&fi->lock);
 779         if (attr_ver == fi->attr_version && size < inode->i_size &&
 780             !test_bit(FUSE_I_SIZE_UNSTABLE, &fi->state)) {
 781                 fi->attr_version = atomic64_inc_return(&fc->attr_version);
 782                 i_size_write(inode, size);
 783         }
 784         spin_unlock(&fi->lock);
 785 }
 786
 787 static void fuse_short_read(struct inode *inode, u64 attr_ver, size_t num_read,
 788                             struct fuse_args_pages *ap)
 789 {
 790         struct fuse_conn *fc = get_fuse_conn(inode);
 791
 792         if (fc->writeback_cache) {
 793                 /*
 794                  * A hole in a file. Some data after the hole are in page cache,
 795                  * but have not reached the client fs yet. So, the hole is not
 796                  * present there.
 797                  */
 798                 int i;
 799                 int start_idx = num_read >> PAGE_SHIFT;
 800                 size_t off = num_read & (PAGE_SIZE - 1);
 801
 802                 for (i = start_idx; i < ap->num_pages; i++) {
 803                         zero_user_segment(ap->pages[i], off, PAGE_SIZE);
 804                         off = 0;
 805                 }
 806         } else {
 807                 loff_t pos = page_offset(ap->pages[0]) + num_read;
 808                 fuse_read_update_size(inode, pos, attr_ver);
 809         }
 810 }
 811
 812 static int fuse_do_readpage(struct file *file, struct page *page)
 813 {
 814         struct inode *inode = page->mapping->host;
 815         struct fuse_mount *fm = get_fuse_mount(inode);
 816         loff_t pos = page_offset(page);
 817         struct fuse_page_desc desc = { .length = PAGE_SIZE };
 818         struct fuse_io_args ia = {
 819                 .ap.args.page_zeroing = true,
 820                 .ap.args.out_pages = true,
 821                 .ap.num_pages = 1,
 822                 .ap.pages = &page,
 823                 .ap.descs = &desc,
 824         };
 825         ssize_t res;
 826         u64 attr_ver;
 827
 828         /*
 829          * Page writeback can extend beyond the lifetime of the
 830          * page-cache page, so make sure we read a properly synced
 831          * page.
 832          */
 833         fuse_wait_on_page_writeback(inode, page->index);
 834
 835         attr_ver = fuse_get_attr_version(fm->fc);
 836
 837         /* Don't overflow end offset */
 838         if (pos + (desc.length - 1) == LLONG_MAX)
 839                 desc.length--;
 840
 841         fuse_read_args_fill(&ia, file, pos, desc.length, FUSE_READ);
 842         res = fuse_simple_request(fm, &ia.ap.args);
 843         if (res < 0)
 844                 return res;
 845         /*
 846          * Short read means EOF.  If file size is larger, truncate it
 847          */
 848         if (res < desc.length)
 849                 fuse_short_read(inode, attr_ver, res, &ia.ap);
 850
 851         SetPageUptodate(page);
 852
 853         return 0;
 854 }
 855
 856 static int fuse_readpage(struct file *file, struct page *page)
 857 {
 858         struct inode *inode = page->mapping->host;
 859         int err;
 860
 861         err = -EIO;
 862         if (is_bad_inode(inode))
 863                 goto out;
 864
 865         err = fuse_do_readpage(file, page);
 866         fuse_invalidate_atime(inode);
 867  out:
 868         unlock_page(page);
 869         return err;
 870 }
 871
 872 static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args,
 873                                int err)
 874 {
 875         int i;
 876         struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args);
 877         struct fuse_args_pages *ap = &ia->ap;
 878         size_t count = ia->read.in.size;
 879         size_t num_read = args->out_args[0].size;
 880         struct address_space *mapping = NULL;
 881
 882         for (i = 0; mapping == NULL && i < ap->num_pages; i++)
 883                 mapping = ap->pages[i]->mapping;
 884
 885         if (mapping) {
 886                 struct inode *inode = mapping->host;
 887
 888                 /*
 889                  * Short read means EOF. If file size is larger, truncate it
 890                  */
 891                 if (!err && num_read < count)
 892                         fuse_short_read(inode, ia->read.attr_ver, num_read, ap);
 893
 894                 fuse_invalidate_atime(inode);
 895         }
 896
 897         for (i = 0; i < ap->num_pages; i++) {
 898                 struct page *page = ap->pages[i];
 899
 900                 if (!err)
 901                         SetPageUptodate(page);
 902                 else
 903                         SetPageError(page);
 904                 unlock_page(page);
 905                 put_page(page);
 906         }
 907         if (ia->ff)
 908                 fuse_file_put(ia->ff, false, false);
 909
 910         fuse_io_free(ia);
 911 }
 912
 913 static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file)
 914 {
 915         struct fuse_file *ff = file->private_data;
 916         struct fuse_mount *fm = ff->fm;
 917         struct fuse_args_pages *ap = &ia->ap;
 918         loff_t pos = page_offset(ap->pages[0]);
 919         size_t count = ap->num_pages << PAGE_SHIFT;
 920         ssize_t res;
 921         int err;
 922
 923         ap->args.out_pages = true;
 924         ap->args.page_zeroing = true;
 925         ap->args.page_replace = true;
 926
 927         /* Don't overflow end offset */
 928         if (pos + (count - 1) == LLONG_MAX) {
 929                 count--;
 930                 ap->descs[ap->num_pages - 1].length--;
 931         }
 932         WARN_ON((loff_t) (pos + count) < 0);
 933
 934         fuse_read_args_fill(ia, file, pos, count, FUSE_READ);
 935         ia->read.attr_ver = fuse_get_attr_version(fm->fc);
 936         if (fm->fc->async_read) {
 937                 ia->ff = fuse_file_get(ff);
 938                 ap->args.end = fuse_readpages_end;
 939                 err = fuse_simple_background(fm, &ap->args, GFP_KERNEL);
 940                 if (!err)
 941                         return;
 942         } else {
 943                 res = fuse_simple_request(fm, &ap->args);
 944                 err = res < 0 ? res : 0;
 945         }
 946         fuse_readpages_end(fm, &ap->args, err);
 947 }
 948
 949 static void fuse_readahead(struct readahead_control *rac)
 950 {
 951         struct inode *inode = rac->mapping->host;
 952         struct fuse_conn *fc = get_fuse_conn(inode);
 953         unsigned int i, max_pages, nr_pages = 0;
 954
 955         if (is_bad_inode(inode))
 956                 return;
 957
 958         max_pages = min_t(unsigned int, fc->max_pages,
 959                         fc->max_read / PAGE_SIZE);
 960
 961         for (;;) {
 962                 struct fuse_io_args *ia;
 963                 struct fuse_args_pages *ap;
 964
 965                 nr_pages = readahead_count(rac) - nr_pages;
 966                 if (nr_pages > max_pages)
 967                         nr_pages = max_pages;
 968                 if (nr_pages == 0)
 969                         break;
 970                 ia = fuse_io_alloc(NULL, nr_pages);
 971                 if (!ia)
 972                         return;
 973                 ap = &ia->ap;
 974                 nr_pages = __readahead_batch(rac, ap->pages, nr_pages);
 975                 for (i = 0; i < nr_pages; i++) {
 976                         fuse_wait_on_page_writeback(inode,
 977                                                     readahead_index(rac) + i);
 978                         ap->descs[i].length = PAGE_SIZE;
 979                 }
 980                 ap->num_pages = nr_pages;
 981                 fuse_send_readpages(ia, rac->file);
 982         }
 983 }
 984
 985 static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to)
 986 {
 987         struct inode *inode = iocb->ki_filp->f_mapping->host;
 988         struct fuse_conn *fc = get_fuse_conn(inode);
 989
 990         /*
 991          * In auto invalidate mode, always update attributes on read.
 992          * Otherwise, only update if we attempt to read past EOF (to ensure
 993          * i_size is up to date).
 994          */
 995         if (fc->auto_inval_data ||
 996             (iocb->ki_pos + iov_iter_count(to) > i_size_read(inode))) {
 997                 int err;
 998                 err = fuse_update_attributes(inode, iocb->ki_filp);
 999                 if (err)
1000                         return err;
1001         }
1002
1003         return generic_file_read_iter(iocb, to);
1004 }
1005
1006 static void fuse_write_args_fill(struct fuse_io_args *ia, struct fuse_file *ff,
1007                                  loff_t pos, size_t count)
1008 {
1009         struct fuse_args *args = &ia->ap.args;
1010
1011         ia->write.in.fh = ff->fh;
1012         ia->write.in.offset = pos;
1013         ia->write.in.size = count;
1014         args->opcode = FUSE_WRITE;
1015         args->nodeid = ff->nodeid;
1016         args->in_numargs = 2;
1017         if (ff->fm->fc->minor < 9)
1018                 args->in_args[0].size = FUSE_COMPAT_WRITE_IN_SIZE;
1019         else
1020                 args->in_args[0].size = sizeof(ia->write.in);
1021         args->in_args[0].value = &ia->write.in;
1022         args->in_args[1].size = count;
1023         args->out_numargs = 1;
1024         args->out_args[0].size = sizeof(ia->write.out);
1025         args->out_args[0].value = &ia->write.out;
1026 }
1027
1028 static unsigned int fuse_write_flags(struct kiocb *iocb)
1029 {
1030         unsigned int flags = iocb->ki_filp->f_flags;
1031
1032         if (iocb->ki_flags & IOCB_DSYNC)
1033                 flags |= O_DSYNC;
1034         if (iocb->ki_flags & IOCB_SYNC)
1035                 flags |= O_SYNC;
1036
1037         return flags;
1038 }
1039
1040 static ssize_t fuse_send_write(struct fuse_io_args *ia, loff_t pos,
1041                                size_t count, fl_owner_t owner)
1042 {
1043         struct kiocb *iocb = ia->io->iocb;
1044         struct file *file = iocb->ki_filp;
1045         struct fuse_file *ff = file->private_data;
1046         struct fuse_mount *fm = ff->fm;
1047         struct fuse_write_in *inarg = &ia->write.in;
1048         ssize_t err;
1049
1050         fuse_write_args_fill(ia, ff, pos, count);
1051         inarg->flags = fuse_write_flags(iocb);
1052         if (owner != NULL) {
1053                 inarg->write_flags |= FUSE_WRITE_LOCKOWNER;
1054                 inarg->lock_owner = fuse_lock_owner_id(fm->fc, owner);
1055         }
1056
1057         if (ia->io->async)
1058                 return fuse_async_req_send(fm, ia, count);
1059
1060         err = fuse_simple_request(fm, &ia->ap.args);
1061         if (!err && ia->write.out.size > count)
1062                 err = -EIO;
1063
1064         return err ?: ia->write.out.size;
1065 }
1066
1067 bool fuse_write_update_size(struct inode *inode, loff_t pos)
1068 {
1069         struct fuse_conn *fc = get_fuse_conn(inode);
1070         struct fuse_inode *fi = get_fuse_inode(inode);
1071         bool ret = false;
1072
1073         spin_lock(&fi->lock);
1074         fi->attr_version = atomic64_inc_return(&fc->attr_version);
1075         if (pos > inode->i_size) {
1076                 i_size_write(inode, pos);
1077                 ret = true;
1078         }
1079         spin_unlock(&fi->lock);
1080
1081         return ret;
1082 }
1083
1084 static ssize_t fuse_send_write_pages(struct fuse_io_args *ia,
1085                                      struct kiocb *iocb, struct inode *inode,
1086                                      loff_t pos, size_t count)
1087 {
1088         struct fuse_args_pages *ap = &ia->ap;
1089         struct file *file = iocb->ki_filp;
1090         struct fuse_file *ff = file->private_data;
1091         struct fuse_mount *fm = ff->fm;
1092         unsigned int offset, i;
1093         int err;
1094
1095         for (i = 0; i < ap->num_pages; i++)
1096                 fuse_wait_on_page_writeback(inode, ap->pages[i]->index);
1097
1098         fuse_write_args_fill(ia, ff, pos, count);
1099         ia->write.in.flags = fuse_write_flags(iocb);
1100
1101         err = fuse_simple_request(fm, &ap->args);
1102         if (!err && ia->write.out.size > count)
1103                 err = -EIO;
1104
1105         offset = ap->descs[0].offset;
1106         count = ia->write.out.size;
1107         for (i = 0; i < ap->num_pages; i++) {
1108                 struct page *page = ap->pages[i];
1109
1110                 if (!err && !offset && count >= PAGE_SIZE)
1111                         SetPageUptodate(page);
1112
1113                 if (count > PAGE_SIZE - offset)
1114                         count -= PAGE_SIZE - offset;
1115                 else
1116                         count = 0;
1117                 offset = 0;
1118
1119                 unlock_page(page);
1120                 put_page(page);
1121         }
1122
1123         return err;
1124 }
1125
1126 static ssize_t fuse_fill_write_pages(struct fuse_args_pages *ap,
1127                                      struct address_space *mapping,
1128                                      struct iov_iter *ii, loff_t pos,
1129                                      unsigned int max_pages)
1130 {
1131         struct fuse_conn *fc = get_fuse_conn(mapping->host);
1132         unsigned offset = pos & (PAGE_SIZE - 1);
1133         size_t count = 0;
1134         int err;
1135
1136         ap->args.in_pages = true;
1137         ap->descs[0].offset = offset;
1138
1139         do {
1140                 size_t tmp;
1141                 struct page *page;
1142                 pgoff_t index = pos >> PAGE_SHIFT;
1143                 size_t bytes = min_t(size_t, PAGE_SIZE - offset,
1144                                      iov_iter_count(ii));
1145
1146                 bytes = min_t(size_t, bytes, fc->max_write - count);
1147
1148  again:
1149                 err = -EFAULT;
1150                 if (iov_iter_fault_in_readable(ii, bytes))
1151                         break;
1152
1153                 err = -ENOMEM;
1154                 page = grab_cache_page_write_begin(mapping, index, 0);
1155                 if (!page)
1156                         break;
1157
1158                 if (mapping_writably_mapped(mapping))
1159                         flush_dcache_page(page);
1160
1161                 tmp = iov_iter_copy_from_user_atomic(page, ii, offset, bytes);
1162                 flush_dcache_page(page);
1163
1164                 iov_iter_advance(ii, tmp);
1165                 if (!tmp) {
1166                         unlock_page(page);
1167                         put_page(page);
1168                         bytes = min(bytes, iov_iter_single_seg_count(ii));
1169                         goto again;
1170                 }
1171
1172                 err = 0;
1173                 ap->pages[ap->num_pages] = page;
1174                 ap->descs[ap->num_pages].length = tmp;
1175                 ap->num_pages++;
1176
1177                 count += tmp;
1178                 pos += tmp;
1179                 offset += tmp;
1180                 if (offset == PAGE_SIZE)
1181                         offset = 0;
1182
1183                 if (!fc->big_writes)
1184                         break;
1185         } while (iov_iter_count(ii) && count < fc->max_write &&
1186                  ap->num_pages < max_pages && offset == 0);
1187
1188         return count > 0 ? count : err;
1189 }
1190
1191 static inline unsigned int fuse_wr_pages(loff_t pos, size_t len,
1192                                      unsigned int max_pages)
1193 {
1194         return min_t(unsigned int,
1195                      ((pos + len - 1) >> PAGE_SHIFT) -
1196                      (pos >> PAGE_SHIFT) + 1,
1197                      max_pages);
1198 }
1199
1200 static ssize_t fuse_perform_write(struct kiocb *iocb,
1201                                   struct address_space *mapping,
1202                                   struct iov_iter *ii, loff_t pos)
1203 {
1204         struct inode *inode = mapping->host;
1205         struct fuse_conn *fc = get_fuse_conn(inode);
1206         struct fuse_inode *fi = get_fuse_inode(inode);
1207         int err = 0;
1208         ssize_t res = 0;
1209
1210         if (inode->i_size < pos + iov_iter_count(ii))
1211                 set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
1212
1213         do {
1214                 ssize_t count;
1215                 struct fuse_io_args ia = {};
1216                 struct fuse_args_pages *ap = &ia.ap;
1217                 unsigned int nr_pages = fuse_wr_pages(pos, iov_iter_count(ii),
1218                                                       fc->max_pages);
1219
1220                 ap->pages = fuse_pages_alloc(nr_pages, GFP_KERNEL, &ap->descs);
1221                 if (!ap->pages) {
1222                         err = -ENOMEM;
1223                         break;
1224                 }
1225
1226                 count = fuse_fill_write_pages(ap, mapping, ii, pos, nr_pages);
1227                 if (count <= 0) {
1228                         err = count;
1229                 } else {
1230                         err = fuse_send_write_pages(&ia, iocb, inode,
1231                                                     pos, count);
1232                         if (!err) {
1233                                 size_t num_written = ia.write.out.size;
1234
1235                                 res += num_written;
1236                                 pos += num_written;
1237
1238                                 /* break out of the loop on short write */
1239                                 if (num_written != count)
1240                                         err = -EIO;
1241                         }
1242                 }
1243                 kfree(ap->pages);
1244         } while (!err && iov_iter_count(ii));
1245
1246         if (res > 0)
1247                 fuse_write_update_size(inode, pos);
1248
1249         clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
1250         fuse_invalidate_attr(inode);
1251
1252         return res > 0 ? res : err;
1253 }
1254
1255 static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from)
1256 {
1257         struct file *file = iocb->ki_filp;
1258         struct address_space *mapping = file->f_mapping;
1259         ssize_t written = 0;
1260         ssize_t written_buffered = 0;
1261         struct inode *inode = mapping->host;
1262         ssize_t err;
1263         loff_t endbyte = 0;
1264
1265         if (get_fuse_conn(inode)->writeback_cache) {
1266                 /* Update size (EOF optimization) and mode (SUID clearing) */
1267                 err = fuse_update_attributes(mapping->host, file);
1268                 if (err)
1269                         return err;
1270
1271                 return generic_file_write_iter(iocb, from);
1272         }
1273
1274         inode_lock(inode);
1275
1276         /* We can write back this queue in page reclaim */
1277         current->backing_dev_info = inode_to_bdi(inode);
1278
1279         err = generic_write_checks(iocb, from);
1280         if (err <= 0)
1281                 goto out;
1282
1283         err = file_remove_privs(file);
1284         if (err)
1285                 goto out;
1286
1287         err = file_update_time(file);
1288         if (err)
1289                 goto out;
1290
1291         if (iocb->ki_flags & IOCB_DIRECT) {
1292                 loff_t pos = iocb->ki_pos;
1293                 written = generic_file_direct_write(iocb, from);
1294                 if (written < 0 || !iov_iter_count(from))
1295                         goto out;
1296
1297                 pos += written;
1298
1299                 written_buffered = fuse_perform_write(iocb, mapping, from, pos);
1300                 if (written_buffered < 0) {
1301                         err = written_buffered;
1302                         goto out;
1303                 }
1304                 endbyte = pos + written_buffered - 1;
1305
1306                 err = filemap_write_and_wait_range(file->f_mapping, pos,
1307                                                    endbyte);
1308                 if (err)
1309                         goto out;
1310
1311                 invalidate_mapping_pages(file->f_mapping,
1312                                          pos >> PAGE_SHIFT,
1313                                          endbyte >> PAGE_SHIFT);
1314
1315                 written += written_buffered;
1316                 iocb->ki_pos = pos + written_buffered;
1317         } else {
1318                 written = fuse_perform_write(iocb, mapping, from, iocb->ki_pos);
1319                 if (written >= 0)
1320                         iocb->ki_pos += written;
1321         }
1322 out:
1323         current->backing_dev_info = NULL;
1324         inode_unlock(inode);
1325         if (written > 0)
1326                 written = generic_write_sync(iocb, written);
1327
1328         return written ? written : err;
1329 }
1330
1331 static inline void fuse_page_descs_length_init(struct fuse_page_desc *descs,
1332                                                unsigned int index,
1333                                                unsigned int nr_pages)
1334 {
1335         int i;
1336
1337         for (i = index; i < index + nr_pages; i++)
1338                 descs[i].length = PAGE_SIZE - descs[i].offset;
1339 }
1340
1341 static inline unsigned long fuse_get_user_addr(const struct iov_iter *ii)
1342 {
1343         return (unsigned long)ii->iov->iov_base + ii->iov_offset;
1344 }
1345
1346 static inline size_t fuse_get_frag_size(const struct iov_iter *ii,
1347                                         size_t max_size)
1348 {
1349         return min(iov_iter_single_seg_count(ii), max_size);
1350 }
1351
1352 static int fuse_get_user_pages(struct fuse_args_pages *ap, struct iov_iter *ii,
1353                                size_t *nbytesp, int write,
1354                                unsigned int max_pages)
1355 {
1356         size_t nbytes = 0;  /* # bytes already packed in req */
1357         ssize_t ret = 0;
1358
1359         /* Special case for kernel I/O: can copy directly into the buffer */
1360         if (iov_iter_is_kvec(ii)) {
1361                 unsigned long user_addr = fuse_get_user_addr(ii);
1362                 size_t frag_size = fuse_get_frag_size(ii, *nbytesp);
1363
1364                 if (write)
1365                         ap->args.in_args[1].value = (void *) user_addr;
1366                 else
1367                         ap->args.out_args[0].value = (void *) user_addr;
1368
1369                 iov_iter_advance(ii, frag_size);
1370                 *nbytesp = frag_size;
1371                 return 0;
1372         }
1373
1374         while (nbytes < *nbytesp && ap->num_pages < max_pages) {
1375                 unsigned npages;
1376                 size_t start;
1377                 ret = iov_iter_get_pages(ii, &ap->pages[ap->num_pages],
1378                                         *nbytesp - nbytes,
1379                                         max_pages - ap->num_pages,
1380                                         &start);
1381                 if (ret < 0)
1382                         break;
1383
1384                 iov_iter_advance(ii, ret);
1385                 nbytes += ret;
1386
1387                 ret += start;
1388                 npages = (ret + PAGE_SIZE - 1) / PAGE_SIZE;
1389
1390                 ap->descs[ap->num_pages].offset = start;
1391                 fuse_page_descs_length_init(ap->descs, ap->num_pages, npages);
1392
1393                 ap->num_pages += npages;
1394                 ap->descs[ap->num_pages - 1].length -=
1395                         (PAGE_SIZE - ret) & (PAGE_SIZE - 1);
1396         }
1397
1398         if (write)
1399                 ap->args.in_pages = true;
1400         else
1401                 ap->args.out_pages = true;
1402
1403         *nbytesp = nbytes;
1404
1405         return ret < 0 ? ret : 0;
1406 }
1407
1408 ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter,
1409                        loff_t *ppos, int flags)
1410 {
1411         int write = flags & FUSE_DIO_WRITE;
1412         int cuse = flags & FUSE_DIO_CUSE;
1413         struct file *file = io->iocb->ki_filp;
1414         struct inode *inode = file->f_mapping->host;
1415         struct fuse_file *ff = file->private_data;
1416         struct fuse_conn *fc = ff->fm->fc;
1417         size_t nmax = write ? fc->max_write : fc->max_read;
1418         loff_t pos = *ppos;
1419         size_t count = iov_iter_count(iter);
1420         pgoff_t idx_from = pos >> PAGE_SHIFT;
1421         pgoff_t idx_to = (pos + count - 1) >> PAGE_SHIFT;
1422         ssize_t res = 0;
1423         int err = 0;
1424         struct fuse_io_args *ia;
1425         unsigned int max_pages;
1426
1427         max_pages = iov_iter_npages(iter, fc->max_pages);
1428         ia = fuse_io_alloc(io, max_pages);
1429         if (!ia)
1430                 return -ENOMEM;
1431
1432         ia->io = io;
1433         if (!cuse && fuse_range_is_writeback(inode, idx_from, idx_to)) {
1434                 if (!write)
1435                         inode_lock(inode);
1436                 fuse_sync_writes(inode);
1437                 if (!write)
1438                         inode_unlock(inode);
1439         }
1440
1441         io->should_dirty = !write && iter_is_iovec(iter);
1442         while (count) {
1443                 ssize_t nres;
1444                 fl_owner_t owner = current->files;
1445                 size_t nbytes = min(count, nmax);
1446
1447                 err = fuse_get_user_pages(&ia->ap, iter, &nbytes, write,
1448                                           max_pages);
1449                 if (err && !nbytes)
1450                         break;
1451
1452                 if (write) {
1453                         if (!capable(CAP_FSETID))
1454                                 ia->write.in.write_flags |= FUSE_WRITE_KILL_PRIV;
1455
1456                         nres = fuse_send_write(ia, pos, nbytes, owner);
1457                 } else {
1458                         nres = fuse_send_read(ia, pos, nbytes, owner);
1459                 }
1460
1461                 if (!io->async || nres < 0) {
1462                         fuse_release_user_pages(&ia->ap, io->should_dirty);
1463                         fuse_io_free(ia);
1464                 }
1465                 ia = NULL;
1466                 if (nres < 0) {
1467                         iov_iter_revert(iter, nbytes);
1468                         err = nres;
1469                         break;
1470                 }
1471                 WARN_ON(nres > nbytes);
1472
1473                 count -= nres;
1474                 res += nres;
1475                 pos += nres;
1476                 if (nres != nbytes) {
1477                         iov_iter_revert(iter, nbytes - nres);
1478                         break;
1479                 }
1480                 if (count) {
1481                         max_pages = iov_iter_npages(iter, fc->max_pages);
1482                         ia = fuse_io_alloc(io, max_pages);
1483                         if (!ia)
1484                                 break;
1485                 }
1486         }
1487         if (ia)
1488                 fuse_io_free(ia);
1489         if (res > 0)
1490                 *ppos = pos;
1491
1492         return res > 0 ? res : err;
1493 }
1494 EXPORT_SYMBOL_GPL(fuse_direct_io);
1495
1496 static ssize_t __fuse_direct_read(struct fuse_io_priv *io,
1497                                   struct iov_iter *iter,
1498                                   loff_t *ppos)
1499 {
1500         ssize_t res;
1501         struct inode *inode = file_inode(io->iocb->ki_filp);
1502
1503         res = fuse_direct_io(io, iter, ppos, 0);
1504
1505         fuse_invalidate_atime(inode);
1506
1507         return res;
1508 }
1509
1510 static ssize_t fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter);
1511
1512 static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to)
1513 {
1514         ssize_t res;
1515
1516         if (!is_sync_kiocb(iocb) && iocb->ki_flags & IOCB_DIRECT) {
1517                 res = fuse_direct_IO(iocb, to);
1518         } else {
1519                 struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);
1520
1521                 res = __fuse_direct_read(&io, to, &iocb->ki_pos);
1522         }
1523
1524         return res;
1525 }
1526
1527 static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from)
1528 {
1529         struct inode *inode = file_inode(iocb->ki_filp);
1530         struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);
1531         ssize_t res;
1532
1533         /* Don't allow parallel writes to the same file */
1534         inode_lock(inode);
1535         res = generic_write_checks(iocb, from);
1536         if (res > 0) {
1537                 if (!is_sync_kiocb(iocb) && iocb->ki_flags & IOCB_DIRECT) {
1538                         res = fuse_direct_IO(iocb, from);
1539                 } else {
1540                         res = fuse_direct_io(&io, from, &iocb->ki_pos,
1541                                              FUSE_DIO_WRITE);
1542                 }
1543         }
1544         fuse_invalidate_attr(inode);
1545         if (res > 0)
1546                 fuse_write_update_size(inode, iocb->ki_pos);
1547         inode_unlock(inode);
1548
1549         return res;
1550 }
1551
1552 static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
1553 {
1554         struct file *file = iocb->ki_filp;
1555         struct fuse_file *ff = file->private_data;
1556         struct inode *inode = file_inode(file);
1557
1558         if (is_bad_inode(inode))
1559                 return -EIO;
1560
1561         if (FUSE_IS_DAX(inode))
1562                 return fuse_dax_read_iter(iocb, to);
1563
1564         if (!(ff->open_flags & FOPEN_DIRECT_IO))
1565                 return fuse_cache_read_iter(iocb, to);
1566         else
1567                 return fuse_direct_read_iter(iocb, to);
1568 }
1569
1570 static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
1571 {
1572         struct file *file = iocb->ki_filp;
1573         struct fuse_file *ff = file->private_data;
1574         struct inode *inode = file_inode(file);
1575
1576         if (is_bad_inode(inode))
1577                 return -EIO;
1578
1579         if (FUSE_IS_DAX(inode))
1580                 return fuse_dax_write_iter(iocb, from);
1581
1582         if (!(ff->open_flags & FOPEN_DIRECT_IO))
1583                 return fuse_cache_write_iter(iocb, from);
1584         else
1585                 return fuse_direct_write_iter(iocb, from);
1586 }
1587
1588 static void fuse_writepage_free(struct fuse_writepage_args *wpa)
1589 {
1590         struct fuse_args_pages *ap = &wpa->ia.ap;
1591         int i;
1592
1593         for (i = 0; i < ap->num_pages; i++)
1594                 __free_page(ap->pages[i]);
1595
1596         if (wpa->ia.ff)
1597                 fuse_file_put(wpa->ia.ff, false, false);
1598
1599         kfree(ap->pages);
1600         kfree(wpa);
1601 }
1602
1603 static void fuse_writepage_finish(struct fuse_mount *fm,
1604                                   struct fuse_writepage_args *wpa)
1605 {
1606         struct fuse_args_pages *ap = &wpa->ia.ap;
1607         struct inode *inode = wpa->inode;
1608         struct fuse_inode *fi = get_fuse_inode(inode);
1609         struct backing_dev_info *bdi = inode_to_bdi(inode);
1610         int i;
1611
1612         for (i = 0; i < ap->num_pages; i++) {
1613                 dec_wb_stat(&bdi->wb, WB_WRITEBACK);
1614                 dec_node_page_state(ap->pages[i], NR_WRITEBACK_TEMP);
1615                 wb_writeout_inc(&bdi->wb);
1616         }
1617         wake_up(&fi->page_waitq);
1618 }
1619
1620 /* Called under fi->lock, may release and reacquire it */
1621 static void fuse_send_writepage(struct fuse_mount *fm,
1622                                 struct fuse_writepage_args *wpa, loff_t size)
1623 __releases(fi->lock)
1624 __acquires(fi->lock)
1625 {
1626         struct fuse_writepage_args *aux, *next;
1627         struct fuse_inode *fi = get_fuse_inode(wpa->inode);
1628         struct fuse_write_in *inarg = &wpa->ia.write.in;
1629         struct fuse_args *args = &wpa->ia.ap.args;
1630         __u64 data_size = wpa->ia.ap.num_pages * PAGE_SIZE;
1631         int err;
1632
1633         fi->writectr++;
1634         if (inarg->offset + data_size <= size) {
1635                 inarg->size = data_size;
1636         } else if (inarg->offset < size) {
1637                 inarg->size = size - inarg->offset;
1638         } else {
1639                 /* Got truncated off completely */
1640                 goto out_free;
1641         }
1642
1643         args->in_args[1].size = inarg->size;
1644         args->force = true;
1645         args->nocreds = true;
1646
1647         err = fuse_simple_background(fm, args, GFP_ATOMIC);
1648         if (err == -ENOMEM) {
1649                 spin_unlock(&fi->lock);
1650                 err = fuse_simple_background(fm, args, GFP_NOFS | __GFP_NOFAIL);
1651                 spin_lock(&fi->lock);
1652         }
1653
1654         /* Fails on broken connection only */
1655         if (unlikely(err))
1656                 goto out_free;
1657
1658         return;
1659
1660  out_free:
1661         fi->writectr--;
1662         rb_erase(&wpa->writepages_entry, &fi->writepages);
1663         fuse_writepage_finish(fm, wpa);
1664         spin_unlock(&fi->lock);
1665
1666         /* After fuse_writepage_finish() aux request list is private */
1667         for (aux = wpa->next; aux; aux = next) {
1668                 next = aux->next;
1669                 aux->next = NULL;
1670                 fuse_writepage_free(aux);
1671         }
1672
1673         fuse_writepage_free(wpa);
1674         spin_lock(&fi->lock);
1675 }
1676
1677 /*
1678  * If fi->writectr is positive (no truncate or fsync going on) send
1679  * all queued writepage requests.
1680  *
1681  * Called with fi->lock
1682  */
1683 void fuse_flush_writepages(struct inode *inode)
1684 __releases(fi->lock)
1685 __acquires(fi->lock)
1686 {
1687         struct fuse_mount *fm = get_fuse_mount(inode);
1688         struct fuse_inode *fi = get_fuse_inode(inode);
1689         loff_t crop = i_size_read(inode);
1690         struct fuse_writepage_args *wpa;
1691
1692         while (fi->writectr >= 0 && !list_empty(&fi->queued_writes)) {
1693                 wpa = list_entry(fi->queued_writes.next,
1694                                  struct fuse_writepage_args, queue_entry);
1695                 list_del_init(&wpa->queue_entry);
1696                 fuse_send_writepage(fm, wpa, crop);
1697         }
1698 }
1699
1700 static struct fuse_writepage_args *fuse_insert_writeback(struct rb_root *root,
1701                                                 struct fuse_writepage_args *wpa)
1702 {
1703         pgoff_t idx_from = wpa->ia.write.in.offset >> PAGE_SHIFT;
1704         pgoff_t idx_to = idx_from + wpa->ia.ap.num_pages - 1;
1705         struct rb_node **p = &root->rb_node;
1706         struct rb_node  *parent = NULL;
1707
1708         WARN_ON(!wpa->ia.ap.num_pages);
1709         while (*p) {
1710                 struct fuse_writepage_args *curr;
1711                 pgoff_t curr_index;
1712
1713                 parent = *p;
1714                 curr = rb_entry(parent, struct fuse_writepage_args,
1715                                 writepages_entry);
1716                 WARN_ON(curr->inode != wpa->inode);
1717                 curr_index = curr->ia.write.in.offset >> PAGE_SHIFT;
1718
1719                 if (idx_from >= curr_index + curr->ia.ap.num_pages)
1720                         p = &(*p)->rb_right;
1721                 else if (idx_to < curr_index)
1722                         p = &(*p)->rb_left;
1723                 else
1724                         return curr;
1725         }
1726
1727         rb_link_node(&wpa->writepages_entry, parent, p);
1728         rb_insert_color(&wpa->writepages_entry, root);
1729         return NULL;
1730 }
1731
1732 static void tree_insert(struct rb_root *root, struct fuse_writepage_args *wpa)
1733 {
1734         WARN_ON(fuse_insert_writeback(root, wpa));
1735 }
1736
1737 static void fuse_writepage_end(struct fuse_mount *fm, struct fuse_args *args,
1738                                int error)
1739 {
1740         struct fuse_writepage_args *wpa =
1741                 container_of(args, typeof(*wpa), ia.ap.args);
1742         struct inode *inode = wpa->inode;
1743         struct fuse_inode *fi = get_fuse_inode(inode);
1744
1745         mapping_set_error(inode->i_mapping, error);
1746         spin_lock(&fi->lock);
1747         rb_erase(&wpa->writepages_entry, &fi->writepages);
1748         while (wpa->next) {
1749                 struct fuse_mount *fm = get_fuse_mount(inode);
1750                 struct fuse_write_in *inarg = &wpa->ia.write.in;
1751                 struct fuse_writepage_args *next = wpa->next;
1752
1753                 wpa->next = next->next;
1754                 next->next = NULL;
1755                 next->ia.ff = fuse_file_get(wpa->ia.ff);
1756                 tree_insert(&fi->writepages, next);
1757
1758                 /*
1759                  * Skip fuse_flush_writepages() to make it easy to crop requests
1760                  * based on primary request size.
1761                  *
1762                  * 1st case (trivial): there are no concurrent activities using
1763                  * fuse_set/release_nowrite.  Then we're on safe side because
1764                  * fuse_flush_writepages() would call fuse_send_writepage()
1765                  * anyway.
1766                  *
1767                  * 2nd case: someone called fuse_set_nowrite and it is waiting
1768                  * now for completion of all in-flight requests.  This happens
1769                  * rarely and no more than once per page, so this should be
1770                  * okay.
1771                  *
1772                  * 3rd case: someone (e.g. fuse_do_setattr()) is in the middle
1773                  * of fuse_set_nowrite..fuse_release_nowrite section.  The fact
1774                  * that fuse_set_nowrite returned implies that all in-flight
1775                  * requests were completed along with all of their secondary
1776                  * requests.  Further primary requests are blocked by negative
1777                  * writectr.  Hence there cannot be any in-flight requests and
1778                  * no invocations of fuse_writepage_end() while we're in
1779                  * fuse_set_nowrite..fuse_release_nowrite section.
1780                  */
1781                 fuse_send_writepage(fm, next, inarg->offset + inarg->size);
1782         }
1783         fi->writectr--;
1784         fuse_writepage_finish(fm, wpa);
1785         spin_unlock(&fi->lock);
1786         fuse_writepage_free(wpa);
1787 }
1788
1789 static struct fuse_file *__fuse_write_file_get(struct fuse_conn *fc,
1790                                                struct fuse_inode *fi)
1791 {
1792         struct fuse_file *ff = NULL;
1793
1794         spin_lock(&fi->lock);
1795         if (!list_empty(&fi->write_files)) {
1796                 ff = list_entry(fi->write_files.next, struct fuse_file,
1797                                 write_entry);
1798                 fuse_file_get(ff);
1799         }
1800         spin_unlock(&fi->lock);
1801
1802         return ff;
1803 }
1804
1805 static struct fuse_file *fuse_write_file_get(struct fuse_conn *fc,
1806                                              struct fuse_inode *fi)
1807 {
1808         struct fuse_file *ff = __fuse_write_file_get(fc, fi);
1809         WARN_ON(!ff);
1810         return ff;
1811 }
1812
1813 int fuse_write_inode(struct inode *inode, struct writeback_control *wbc)
1814 {
1815         struct fuse_conn *fc = get_fuse_conn(inode);
1816         struct fuse_inode *fi = get_fuse_inode(inode);
1817         struct fuse_file *ff;
1818         int err;
1819
1820         ff = __fuse_write_file_get(fc, fi);
1821         err = fuse_flush_times(inode, ff);
1822         if (ff)
1823                 fuse_file_put(ff, false, false);
1824
1825         return err;
1826 }
1827
1828 static struct fuse_writepage_args *fuse_writepage_args_alloc(void)
1829 {
1830         struct fuse_writepage_args *wpa;
1831         struct fuse_args_pages *ap;
1832
1833         wpa = kzalloc(sizeof(*wpa), GFP_NOFS);
1834         if (wpa) {
1835                 ap = &wpa->ia.ap;
1836                 ap->num_pages = 0;
1837                 ap->pages = fuse_pages_alloc(1, GFP_NOFS, &ap->descs);
1838                 if (!ap->pages) {
1839                         kfree(wpa);
1840                         wpa = NULL;
1841                 }
1842         }
1843         return wpa;
1844
1845 }
1846
1847 static int fuse_writepage_locked(struct page *page)
1848 {
1849         struct address_space *mapping = page->mapping;
1850         struct inode *inode = mapping->host;
1851         struct fuse_conn *fc = get_fuse_conn(inode);
1852         struct fuse_inode *fi = get_fuse_inode(inode);
1853         struct fuse_writepage_args *wpa;
1854         struct fuse_args_pages *ap;
1855         struct page *tmp_page;
1856         int error = -ENOMEM;
1857
1858         set_page_writeback(page);
1859
1860         wpa = fuse_writepage_args_alloc();
1861         if (!wpa)
1862                 goto err;
1863         ap = &wpa->ia.ap;
1864
1865         tmp_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
1866         if (!tmp_page)
1867                 goto err_free;
1868
1869         error = -EIO;
1870         wpa->ia.ff = fuse_write_file_get(fc, fi);
1871         if (!wpa->ia.ff)
1872                 goto err_nofile;
1873
1874         fuse_write_args_fill(&wpa->ia, wpa->ia.ff, page_offset(page), 0);
1875
1876         copy_highpage(tmp_page, page);
1877         wpa->ia.write.in.write_flags |= FUSE_WRITE_CACHE;
1878         wpa->next = NULL;
1879         ap->args.in_pages = true;
1880         ap->num_pages = 1;
1881         ap->pages[0] = tmp_page;
1882         ap->descs[0].offset = 0;
1883         ap->descs[0].length = PAGE_SIZE;
1884         ap->args.end = fuse_writepage_end;
1885         wpa->inode = inode;
1886
1887         inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK);
1888         inc_node_page_state(tmp_page, NR_WRITEBACK_TEMP);
1889
1890         spin_lock(&fi->lock);
1891         tree_insert(&fi->writepages, wpa);
1892         list_add_tail(&wpa->queue_entry, &fi->queued_writes);
1893         fuse_flush_writepages(inode);
1894         spin_unlock(&fi->lock);
1895
1896         end_page_writeback(page);
1897
1898         return 0;
1899
1900 err_nofile:
1901         __free_page(tmp_page);
1902 err_free:
1903         kfree(wpa);
1904 err:
1905         mapping_set_error(page->mapping, error);
1906         end_page_writeback(page);
1907         return error;
1908 }
1909
1910 static int fuse_writepage(struct page *page, struct writeback_control *wbc)
1911 {
1912         int err;
1913
1914         if (fuse_page_is_writeback(page->mapping->host, page->index)) {
1915                 /*
1916                  * ->writepages() should be called for sync() and friends.  We
1917                  * should only get here on direct reclaim and then we are
1918                  * allowed to skip a page which is already in flight
1919                  */
1920                 WARN_ON(wbc->sync_mode == WB_SYNC_ALL);
1921
1922                 redirty_page_for_writepage(wbc, page);
1923                 unlock_page(page);
1924                 return 0;
1925         }
1926
1927         err = fuse_writepage_locked(page);
1928         unlock_page(page);
1929
1930         return err;
1931 }
1932
1933 struct fuse_fill_wb_data {
1934         struct fuse_writepage_args *wpa;
1935         struct fuse_file *ff;
1936         struct inode *inode;
1937         struct page **orig_pages;
1938         unsigned int max_pages;
1939 };
1940
1941 static bool fuse_pages_realloc(struct fuse_fill_wb_data *data)
1942 {
1943         struct fuse_args_pages *ap = &data->wpa->ia.ap;
1944         struct fuse_conn *fc = get_fuse_conn(data->inode);
1945         struct page **pages;
1946         struct fuse_page_desc *descs;
1947         unsigned int npages = min_t(unsigned int,
1948                                     max_t(unsigned int, data->max_pages * 2,
1949                                           FUSE_DEFAULT_MAX_PAGES_PER_REQ),
1950                                     fc->max_pages);
1951         WARN_ON(npages <= data->max_pages);
1952
1953         pages = fuse_pages_alloc(npages, GFP_NOFS, &descs);
1954         if (!pages)
1955                 return false;
1956
1957         memcpy(pages, ap->pages, sizeof(struct page *) * ap->num_pages);
1958         memcpy(descs, ap->descs, sizeof(struct fuse_page_desc) * ap->num_pages);
1959         kfree(ap->pages);
1960         ap->pages = pages;
1961         ap->descs = descs;
1962         data->max_pages = npages;
1963
1964         return true;
1965 }
1966
1967 static void fuse_writepages_send(struct fuse_fill_wb_data *data)
1968 {
1969         struct fuse_writepage_args *wpa = data->wpa;
1970         struct inode *inode = data->inode;
1971         struct fuse_inode *fi = get_fuse_inode(inode);
1972         int num_pages = wpa->ia.ap.num_pages;
1973         int i;
1974
1975         wpa->ia.ff = fuse_file_get(data->ff);
1976         spin_lock(&fi->lock);
1977         list_add_tail(&wpa->queue_entry, &fi->queued_writes);
1978         fuse_flush_writepages(inode);
1979         spin_unlock(&fi->lock);
1980
1981         for (i = 0; i < num_pages; i++)
1982                 end_page_writeback(data->orig_pages[i]);
1983 }
1984
1985 /*
1986  * Check under fi->lock if the page is under writeback, and insert it onto the
1987  * rb_tree if not. Otherwise iterate auxiliary write requests, to see if there's
1988  * one already added for a page at this offset.  If there's none, then insert
1989  * this new request onto the auxiliary list, otherwise reuse the existing one by
1990  * swapping the new temp page with the old one.
1991  */
1992 static bool fuse_writepage_add(struct fuse_writepage_args *new_wpa,
1993                                struct page *page)
1994 {
1995         struct fuse_inode *fi = get_fuse_inode(new_wpa->inode);
1996         struct fuse_writepage_args *tmp;
1997         struct fuse_writepage_args *old_wpa;
1998         struct fuse_args_pages *new_ap = &new_wpa->ia.ap;
1999
2000         WARN_ON(new_ap->num_pages != 0);
2001         new_ap->num_pages = 1;
2002
2003         spin_lock(&fi->lock);
2004         old_wpa = fuse_insert_writeback(&fi->writepages, new_wpa);
2005         if (!old_wpa) {
2006                 spin_unlock(&fi->lock);
2007                 return true;
2008         }
2009
2010         for (tmp = old_wpa->next; tmp; tmp = tmp->next) {
2011                 pgoff_t curr_index;
2012
2013                 WARN_ON(tmp->inode != new_wpa->inode);
2014                 curr_index = tmp->ia.write.in.offset >> PAGE_SHIFT;
2015                 if (curr_index == page->index) {
2016                         WARN_ON(tmp->ia.ap.num_pages != 1);
2017                         swap(tmp->ia.ap.pages[0], new_ap->pages[0]);
2018                         break;
2019                 }
2020         }
2021
2022         if (!tmp) {
2023                 new_wpa->next = old_wpa->next;
2024                 old_wpa->next = new_wpa;
2025         }
2026
2027         spin_unlock(&fi->lock);
2028
2029         if (tmp) {
2030                 struct backing_dev_info *bdi = inode_to_bdi(new_wpa->inode);
2031
2032                 dec_wb_stat(&bdi->wb, WB_WRITEBACK);
2033                 dec_node_page_state(new_ap->pages[0], NR_WRITEBACK_TEMP);
2034                 wb_writeout_inc(&bdi->wb);
2035                 fuse_writepage_free(new_wpa);
2036         }
2037
2038         return false;
2039 }
2040
2041 static bool fuse_writepage_need_send(struct fuse_conn *fc, struct page *page,
2042                                      struct fuse_args_pages *ap,
2043                                      struct fuse_fill_wb_data *data)
2044 {
2045         WARN_ON(!ap->num_pages);
2046
2047         /*
2048          * Being under writeback is unlikely but possible.  For example direct
2049          * read to an mmaped fuse file will set the page dirty twice; once when
2050          * the pages are faulted with get_user_pages(), and then after the read
2051          * completed.
2052          */
2053         if (fuse_page_is_writeback(data->inode, page->index))
2054                 return true;
2055
2056         /* Reached max pages */
2057         if (ap->num_pages == fc->max_pages)
2058                 return true;
2059
2060         /* Reached max write bytes */
2061         if ((ap->num_pages + 1) * PAGE_SIZE > fc->max_write)
2062                 return true;
2063
2064         /* Discontinuity */
2065         if (data->orig_pages[ap->num_pages - 1]->index + 1 != page->index)
2066                 return true;
2067
2068         /* Need to grow the pages array?  If so, did the expansion fail? */
2069         if (ap->num_pages == data->max_pages && !fuse_pages_realloc(data))
2070                 return true;
2071
2072         return false;
2073 }
2074
2075 static int fuse_writepages_fill(struct page *page,
2076                 struct writeback_control *wbc, void *_data)
2077 {
2078         struct fuse_fill_wb_data *data = _data;
2079         struct fuse_writepage_args *wpa = data->wpa;
2080         struct fuse_args_pages *ap = &wpa->ia.ap;
2081         struct inode *inode = data->inode;
2082         struct fuse_inode *fi = get_fuse_inode(inode);
2083         struct fuse_conn *fc = get_fuse_conn(inode);
2084         struct page *tmp_page;
2085         int err;
2086
2087         if (!data->ff) {
2088                 err = -EIO;
2089                 data->ff = fuse_write_file_get(fc, fi);
2090                 if (!data->ff)
2091                         goto out_unlock;
2092         }
2093
2094         if (wpa && fuse_writepage_need_send(fc, page, ap, data)) {
2095                 fuse_writepages_send(data);
2096                 data->wpa = NULL;
2097         }
2098
2099         err = -ENOMEM;
2100         tmp_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
2101         if (!tmp_page)
2102                 goto out_unlock;
2103
2104         /*
2105          * The page must not be redirtied until the writeout is completed
2106          * (i.e. userspace has sent a reply to the write request).  Otherwise
2107          * there could be more than one temporary page instance for each real
2108          * page.
2109          *
2110          * This is ensured by holding the page lock in page_mkwrite() while
2111          * checking fuse_page_is_writeback().  We already hold the page lock
2112          * since clear_page_dirty_for_io() and keep it held until we add the
2113          * request to the fi->writepages list and increment ap->num_pages.
2114          * After this fuse_page_is_writeback() will indicate that the page is
2115          * under writeback, so we can release the page lock.
2116          */
2117         if (data->wpa == NULL) {
2118                 err = -ENOMEM;
2119                 wpa = fuse_writepage_args_alloc();
2120                 if (!wpa) {
2121                         __free_page(tmp_page);
2122                         goto out_unlock;
2123                 }
2124                 data->max_pages = 1;
2125
2126                 ap = &wpa->ia.ap;
2127                 fuse_write_args_fill(&wpa->ia, data->ff, page_offset(page), 0);
2128                 wpa->ia.write.in.write_flags |= FUSE_WRITE_CACHE;
2129                 wpa->next = NULL;
2130                 ap->args.in_pages = true;
2131                 ap->args.end = fuse_writepage_end;
2132                 ap->num_pages = 0;
2133                 wpa->inode = inode;
2134         }
2135         set_page_writeback(page);
2136
2137         copy_highpage(tmp_page, page);
2138         ap->pages[ap->num_pages] = tmp_page;
2139         ap->descs[ap->num_pages].offset = 0;
2140         ap->descs[ap->num_pages].length = PAGE_SIZE;
2141         data->orig_pages[ap->num_pages] = page;
2142
2143         inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK);
2144         inc_node_page_state(tmp_page, NR_WRITEBACK_TEMP);
2145
2146         err = 0;
2147         if (data->wpa) {
2148                 /*
2149                  * Protected by fi->lock against concurrent access by
2150                  * fuse_page_is_writeback().
2151                  */
2152                 spin_lock(&fi->lock);
2153                 ap->num_pages++;
2154                 spin_unlock(&fi->lock);
2155         } else if (fuse_writepage_add(wpa, page)) {
2156                 data->wpa = wpa;
2157         } else {
2158                 end_page_writeback(page);
2159         }
2160 out_unlock:
2161         unlock_page(page);
2162
2163         return err;
2164 }
2165
2166 static int fuse_writepages(struct address_space *mapping,
2167                            struct writeback_control *wbc)
2168 {
2169         struct inode *inode = mapping->host;
2170         struct fuse_conn *fc = get_fuse_conn(inode);
2171         struct fuse_fill_wb_data data;
2172         int err;
2173
2174         err = -EIO;
2175         if (is_bad_inode(inode))
2176                 goto out;
2177
2178         data.inode = inode;
2179         data.wpa = NULL;
2180         data.ff = NULL;
2181
2182         err = -ENOMEM;
2183         data.orig_pages = kcalloc(fc->max_pages,
2184                                   sizeof(struct page *),
2185                                   GFP_NOFS);
2186         if (!data.orig_pages)
2187                 goto out;
2188
2189         err = write_cache_pages(mapping, wbc, fuse_writepages_fill, &data);
2190         if (data.wpa) {
2191                 WARN_ON(!data.wpa->ia.ap.num_pages);
2192                 fuse_writepages_send(&data);
2193         }
2194         if (data.ff)
2195                 fuse_file_put(data.ff, false, false);
2196
2197         kfree(data.orig_pages);
2198 out:
2199         return err;
2200 }
2201
2202 /*
2203  * It's worthy to make sure that space is reserved on disk for the write,
2204  * but how to implement it without killing performance need more thinking.
2205  */
2206 static int fuse_write_begin(struct file *file, struct address_space *mapping,
2207                 loff_t pos, unsigned len, unsigned flags,
2208                 struct page **pagep, void **fsdata)
2209 {
2210         pgoff_t index = pos >> PAGE_SHIFT;
2211         struct fuse_conn *fc = get_fuse_conn(file_inode(file));
2212         struct page *page;
2213         loff_t fsize;
2214         int err = -ENOMEM;
2215
2216         WARN_ON(!fc->writeback_cache);
2217
2218         page = grab_cache_page_write_begin(mapping, index, flags);
2219         if (!page)
2220                 goto error;
2221
2222         fuse_wait_on_page_writeback(mapping->host, page->index);
2223
2224         if (PageUptodate(page) || len == PAGE_SIZE)
2225                 goto success;
2226         /*
2227          * Check if the start this page comes after the end of file, in which
2228          * case the readpage can be optimized away.
2229          */
2230         fsize = i_size_read(mapping->host);
2231         if (fsize <= (pos & PAGE_MASK)) {
2232                 size_t off = pos & ~PAGE_MASK;
2233                 if (off)
2234                         zero_user_segment(page, 0, off);
2235                 goto success;
2236         }
2237         err = fuse_do_readpage(file, page);
2238         if (err)
2239                 goto cleanup;
2240 success:
2241         *pagep = page;
2242         return 0;
2243
2244 cleanup:
2245         unlock_page(page);
2246         put_page(page);
2247 error:
2248         return err;
2249 }
2250
2251 static int fuse_write_end(struct file *file, struct address_space *mapping,
2252                 loff_t pos, unsigned len, unsigned copied,
2253                 struct page *page, void *fsdata)
2254 {
2255         struct inode *inode = page->mapping->host;
2256
2257         /* Haven't copied anything?  Skip zeroing, size extending, dirtying. */
2258         if (!copied)
2259                 goto unlock;
2260
2261         if (!PageUptodate(page)) {
2262                 /* Zero any unwritten bytes at the end of the page */
2263                 size_t endoff = (pos + copied) & ~PAGE_MASK;
2264                 if (endoff)
2265                         zero_user_segment(page, endoff, PAGE_SIZE);
2266                 SetPageUptodate(page);
2267         }
2268
2269         fuse_write_update_size(inode, pos + copied);
2270         set_page_dirty(page);
2271
2272 unlock:
2273         unlock_page(page);
2274         put_page(page);
2275
2276         return copied;
2277 }
2278
2279 static int fuse_launder_page(struct page *page)
2280 {
2281         int err = 0;
2282         if (clear_page_dirty_for_io(page)) {
2283                 struct inode *inode = page->mapping->host;
2284                 err = fuse_writepage_locked(page);
2285                 if (!err)
2286                         fuse_wait_on_page_writeback(inode, page->index);
2287         }
2288         return err;
2289 }
2290
2291 /*
2292  * Write back dirty pages now, because there may not be any suitable
2293  * open files later
2294  */
2295 static void fuse_vma_close(struct vm_area_struct *vma)
2296 {
2297         filemap_write_and_wait(vma->vm_file->f_mapping);
2298 }
2299
2300 /*
2301  * Wait for writeback against this page to complete before allowing it
2302  * to be marked dirty again, and hence written back again, possibly
2303  * before the previous writepage completed.
2304  *
2305  * Block here, instead of in ->writepage(), so that the userspace fs
2306  * can only block processes actually operating on the filesystem.
2307  *
2308  * Otherwise unprivileged userspace fs would be able to block
2309  * unrelated:
2310  *
2311  * - page migration
2312  * - sync(2)
2313  * - try_to_free_pages() with order > PAGE_ALLOC_COSTLY_ORDER
2314  */
2315 static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf)
2316 {
2317         struct page *page = vmf->page;
2318         struct inode *inode = file_inode(vmf->vma->vm_file);
2319
2320         file_update_time(vmf->vma->vm_file);
2321         lock_page(page);
2322         if (page->mapping != inode->i_mapping) {
2323                 unlock_page(page);
2324                 return VM_FAULT_NOPAGE;
2325         }
2326
2327         fuse_wait_on_page_writeback(inode, page->index);
2328         return VM_FAULT_LOCKED;
2329 }
2330
2331 static const struct vm_operations_struct fuse_file_vm_ops = {
2332         .close          = fuse_vma_close,
2333         .fault          = filemap_fault,
2334         .map_pages      = filemap_map_pages,
2335         .page_mkwrite   = fuse_page_mkwrite,
2336 };
2337
2338 static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma)
2339 {
2340         struct fuse_file *ff = file->private_data;
2341
2342         /* DAX mmap is superior to direct_io mmap */
2343         if (FUSE_IS_DAX(file_inode(file)))
2344                 return fuse_dax_mmap(file, vma);
2345
2346         if (ff->open_flags & FOPEN_DIRECT_IO) {
2347                 /* Can't provide the coherency needed for MAP_SHARED */
2348                 if (vma->vm_flags & VM_MAYSHARE)
2349                         return -ENODEV;
2350
2351                 invalidate_inode_pages2(file->f_mapping);
2352
2353                 return generic_file_mmap(file, vma);
2354         }
2355
2356         if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
2357                 fuse_link_write_file(file);
2358
2359         file_accessed(file);
2360         vma->vm_ops = &fuse_file_vm_ops;
2361         return 0;
2362 }
2363
2364 static int convert_fuse_file_lock(struct fuse_conn *fc,
2365                                   const struct fuse_file_lock *ffl,
2366                                   struct file_lock *fl)
2367 {
2368         switch (ffl->type) {
2369         case F_UNLCK:
2370                 break;
2371
2372         case F_RDLCK:
2373         case F_WRLCK:
2374                 if (ffl->start > OFFSET_MAX || ffl->end > OFFSET_MAX ||
2375                     ffl->end < ffl->start)
2376                         return -EIO;
2377
2378                 fl->fl_start = ffl->start;
2379                 fl->fl_end = ffl->end;
2380
2381                 /*
2382                  * Convert pid into init's pid namespace.  The locks API will
2383                  * translate it into the caller's pid namespace.
2384                  */
2385                 rcu_read_lock();
2386                 fl->fl_pid = pid_nr_ns(find_pid_ns(ffl->pid, fc->pid_ns), &init_pid_ns);
2387                 rcu_read_unlock();
2388                 break;
2389
2390         default:
2391                 return -EIO;
2392         }
2393         fl->fl_type = ffl->type;
2394         return 0;
2395 }
2396
2397 static void fuse_lk_fill(struct fuse_args *args, struct file *file,
2398                          const struct file_lock *fl, int opcode, pid_t pid,
2399                          int flock, struct fuse_lk_in *inarg)
2400 {
2401         struct inode *inode = file_inode(file);
2402         struct fuse_conn *fc = get_fuse_conn(inode);
2403         struct fuse_file *ff = file->private_data;
2404
2405         memset(inarg, 0, sizeof(*inarg));
2406         inarg->fh = ff->fh;
2407         inarg->owner = fuse_lock_owner_id(fc, fl->fl_owner);
2408         inarg->lk.start = fl->fl_start;
2409         inarg->lk.end = fl->fl_end;
2410         inarg->lk.type = fl->fl_type;
2411         inarg->lk.pid = pid;
2412         if (flock)
2413                 inarg->lk_flags |= FUSE_LK_FLOCK;
2414         args->opcode = opcode;
2415         args->nodeid = get_node_id(inode);
2416         args->in_numargs = 1;
2417         args->in_args[0].size = sizeof(*inarg);
2418         args->in_args[0].value = inarg;
2419 }
2420
2421 static int fuse_getlk(struct file *file, struct file_lock *fl)
2422 {
2423         struct inode *inode = file_inode(file);
2424         struct fuse_mount *fm = get_fuse_mount(inode);
2425         FUSE_ARGS(args);
2426         struct fuse_lk_in inarg;
2427         struct fuse_lk_out outarg;
2428         int err;
2429
2430         fuse_lk_fill(&args, file, fl, FUSE_GETLK, 0, 0, &inarg);
2431         args.out_numargs = 1;
2432         args.out_args[0].size = sizeof(outarg);
2433         args.out_args[0].value = &outarg;
2434         err = fuse_simple_request(fm, &args);
2435         if (!err)
2436                 err = convert_fuse_file_lock(fm->fc, &outarg.lk, fl);
2437
2438         return err;
2439 }
2440
2441 static int fuse_setlk(struct file *file, struct file_lock *fl, int flock)
2442 {
2443         struct inode *inode = file_inode(file);
2444         struct fuse_mount *fm = get_fuse_mount(inode);
2445         FUSE_ARGS(args);
2446         struct fuse_lk_in inarg;
2447         int opcode = (fl->fl_flags & FL_SLEEP) ? FUSE_SETLKW : FUSE_SETLK;
2448         struct pid *pid = fl->fl_type != F_UNLCK ? task_tgid(current) : NULL;
2449         pid_t pid_nr = pid_nr_ns(pid, fm->fc->pid_ns);
2450         int err;
2451
2452         if (fl->fl_lmops && fl->fl_lmops->lm_grant) {
2453                 /* NLM needs asynchronous locks, which we don't support yet */
2454                 return -ENOLCK;
2455         }
2456
2457         /* Unlock on close is handled by the flush method */
2458         if ((fl->fl_flags & FL_CLOSE_POSIX) == FL_CLOSE_POSIX)
2459                 return 0;
2460
2461         fuse_lk_fill(&args, file, fl, opcode, pid_nr, flock, &inarg);
2462         err = fuse_simple_request(fm, &args);
2463
2464         /* locking is restartable */
2465         if (err == -EINTR)
2466                 err = -ERESTARTSYS;
2467
2468         return err;
2469 }
2470
2471 static int fuse_file_lock(struct file *file, int cmd, struct file_lock *fl)
2472 {
2473         struct inode *inode = file_inode(file);
2474         struct fuse_conn *fc = get_fuse_conn(inode);
2475         int err;
2476
2477         if (cmd == F_CANCELLK) {
2478                 err = 0;
2479         } else if (cmd == F_GETLK) {
2480                 if (fc->no_lock) {
2481                         posix_test_lock(file, fl);
2482                         err = 0;
2483                 } else
2484                         err = fuse_getlk(file, fl);
2485         } else {
2486                 if (fc->no_lock)
2487                         err = posix_lock_file(file, fl, NULL);
2488                 else
2489                         err = fuse_setlk(file, fl, 0);
2490         }
2491         return err;
2492 }
2493
2494 static int fuse_file_flock(struct file *file, int cmd, struct file_lock *fl)
2495 {
2496         struct inode *inode = file_inode(file);
2497         struct fuse_conn *fc = get_fuse_conn(inode);
2498         int err;
2499
2500         if (fc->no_flock) {
2501                 err = locks_lock_file_wait(file, fl);
2502         } else {
2503                 struct fuse_file *ff = file->private_data;
2504
2505                 /* emulate flock with POSIX locks */
2506                 ff->flock = true;
2507                 err = fuse_setlk(file, fl, 1);
2508         }
2509
2510         return err;
2511 }
2512
2513 static sector_t fuse_bmap(struct address_space *mapping, sector_t block)
2514 {
2515         struct inode *inode = mapping->host;
2516         struct fuse_mount *fm = get_fuse_mount(inode);
2517         FUSE_ARGS(args);
2518         struct fuse_bmap_in inarg;
2519         struct fuse_bmap_out outarg;
2520         int err;
2521
2522         if (!inode->i_sb->s_bdev || fm->fc->no_bmap)
2523                 return 0;
2524
2525         memset(&inarg, 0, sizeof(inarg));
2526         inarg.block = block;
2527         inarg.blocksize = inode->i_sb->s_blocksize;
2528         args.opcode = FUSE_BMAP;
2529         args.nodeid = get_node_id(inode);
2530         args.in_numargs = 1;
2531         args.in_args[0].size = sizeof(inarg);
2532         args.in_args[0].value = &inarg;
2533         args.out_numargs = 1;
2534         args.out_args[0].size = sizeof(outarg);
2535         args.out_args[0].value = &outarg;
2536         err = fuse_simple_request(fm, &args);
2537         if (err == -ENOSYS)
2538                 fm->fc->no_bmap = 1;
2539
2540         return err ? 0 : outarg.block;
2541 }
2542
2543 static loff_t fuse_lseek(struct file *file, loff_t offset, int whence)
2544 {
2545         struct inode *inode = file->f_mapping->host;
2546         struct fuse_mount *fm = get_fuse_mount(inode);
2547         struct fuse_file *ff = file->private_data;
2548         FUSE_ARGS(args);
2549         struct fuse_lseek_in inarg = {
2550                 .fh = ff->fh,
2551                 .offset = offset,
2552                 .whence = whence
2553         };
2554         struct fuse_lseek_out outarg;
2555         int err;
2556
2557         if (fm->fc->no_lseek)
2558                 goto fallback;
2559
2560         args.opcode = FUSE_LSEEK;
2561         args.nodeid = ff->nodeid;
2562         args.in_numargs = 1;
2563         args.in_args[0].size = sizeof(inarg);
2564         args.in_args[0].value = &inarg;
2565         args.out_numargs = 1;
2566         args.out_args[0].size = sizeof(outarg);
2567         args.out_args[0].value = &outarg;
2568         err = fuse_simple_request(fm, &args);
2569         if (err) {
2570                 if (err == -ENOSYS) {
2571                         fm->fc->no_lseek = 1;
2572                         goto fallback;
2573                 }
2574                 return err;
2575         }
2576
2577         return vfs_setpos(file, outarg.offset, inode->i_sb->s_maxbytes);
2578
2579 fallback:
2580         err = fuse_update_attributes(inode, file);
2581         if (!err)
2582                 return generic_file_llseek(file, offset, whence);
2583         else
2584                 return err;
2585 }
2586
2587 static loff_t fuse_file_llseek(struct file *file, loff_t offset, int whence)
2588 {
2589         loff_t retval;
2590         struct inode *inode = file_inode(file);
2591
2592         switch (whence) {
2593         case SEEK_SET:
2594         case SEEK_CUR:
2595                  /* No i_mutex protection necessary for SEEK_CUR and SEEK_SET */
2596                 retval = generic_file_llseek(file, offset, whence);
2597                 break;
2598         case SEEK_END:
2599                 inode_lock(inode);
2600                 retval = fuse_update_attributes(inode, file);
2601                 if (!retval)
2602                         retval = generic_file_llseek(file, offset, whence);
2603                 inode_unlock(inode);
2604                 break;
2605         case SEEK_HOLE:
2606         case SEEK_DATA:
2607                 inode_lock(inode);
2608                 retval = fuse_lseek(file, offset, whence);
2609                 inode_unlock(inode);
2610                 break;
2611         default:
2612                 retval = -EINVAL;
2613         }
2614
2615         return retval;
2616 }
2617
2618 /*
2619  * CUSE servers compiled on 32bit broke on 64bit kernels because the
2620  * ABI was defined to be 'struct iovec' which is different on 32bit
2621  * and 64bit.  Fortunately we can determine which structure the server
2622  * used from the size of the reply.
2623  */
2624 static int fuse_copy_ioctl_iovec_old(struct iovec *dst, void *src,
2625                                      size_t transferred, unsigned count,
2626                                      bool is_compat)
2627 {
2628 #ifdef CONFIG_COMPAT
2629         if (count * sizeof(struct compat_iovec) == transferred) {
2630                 struct compat_iovec *ciov = src;
2631                 unsigned i;
2632
2633                 /*
2634                  * With this interface a 32bit server cannot support
2635                  * non-compat (i.e. ones coming from 64bit apps) ioctl
2636                  * requests
2637                  */
2638                 if (!is_compat)
2639                         return -EINVAL;
2640
2641                 for (i = 0; i < count; i++) {
2642                         dst[i].iov_base = compat_ptr(ciov[i].iov_base);
2643                         dst[i].iov_len = ciov[i].iov_len;
2644                 }
2645                 return 0;
2646         }
2647 #endif
2648
2649         if (count * sizeof(struct iovec) != transferred)
2650                 return -EIO;
2651
2652         memcpy(dst, src, transferred);
2653         return 0;
2654 }
2655
2656 /* Make sure iov_length() won't overflow */
2657 static int fuse_verify_ioctl_iov(struct fuse_conn *fc, struct iovec *iov,
2658                                  size_t count)
2659 {
2660         size_t n;
2661         u32 max = fc->max_pages << PAGE_SHIFT;
2662
2663         for (n = 0; n < count; n++, iov++) {
2664                 if (iov->iov_len > (size_t) max)
2665                         return -ENOMEM;
2666                 max -= iov->iov_len;
2667         }
2668         return 0;
2669 }
2670
2671 static int fuse_copy_ioctl_iovec(struct fuse_conn *fc, struct iovec *dst,
2672                                  void *src, size_t transferred, unsigned count,
2673                                  bool is_compat)
2674 {
2675         unsigned i;
2676         struct fuse_ioctl_iovec *fiov = src;
2677
2678         if (fc->minor < 16) {
2679                 return fuse_copy_ioctl_iovec_old(dst, src, transferred,
2680                                                  count, is_compat);
2681         }
2682
2683         if (count * sizeof(struct fuse_ioctl_iovec) != transferred)
2684                 return -EIO;
2685
2686         for (i = 0; i < count; i++) {
2687                 /* Did the server supply an inappropriate value? */
2688                 if (fiov[i].base != (unsigned long) fiov[i].base ||
2689                     fiov[i].len != (unsigned long) fiov[i].len)
2690                         return -EIO;
2691
2692                 dst[i].iov_base = (void __user *) (unsigned long) fiov[i].base;
2693                 dst[i].iov_len = (size_t) fiov[i].len;
2694
2695 #ifdef CONFIG_COMPAT
2696                 if (is_compat &&
2697                     (ptr_to_compat(dst[i].iov_base) != fiov[i].base ||
2698                      (compat_size_t) dst[i].iov_len != fiov[i].len))
2699                         return -EIO;
2700 #endif
2701         }
2702
2703         return 0;
2704 }
2705
2706
2707 /*
2708  * For ioctls, there is no generic way to determine how much memory
2709  * needs to be read and/or written.  Furthermore, ioctls are allowed
2710  * to dereference the passed pointer, so the parameter requires deep
2711  * copying but FUSE has no idea whatsoever about what to copy in or
2712  * out.
2713  *
2714  * This is solved by allowing FUSE server to retry ioctl with
2715  * necessary in/out iovecs.  Let's assume the ioctl implementation
2716  * needs to read in the following structure.
2717  *
2718  * struct a {
2719  *      char    *buf;
2720  *      size_t  buflen;
2721  * }
2722  *
2723  * On the first callout to FUSE server, inarg->in_size and
2724  * inarg->out_size will be NULL; then, the server completes the ioctl
2725  * with FUSE_IOCTL_RETRY set in out->flags, out->in_iovs set to 1 and
2726  * the actual iov array to
2727  *
2728  * { { .iov_base = inarg.arg,   .iov_len = sizeof(struct a) } }
2729  *
2730  * which tells FUSE to copy in the requested area and retry the ioctl.
2731  * On the second round, the server has access to the structure and
2732  * from that it can tell what to look for next, so on the invocation,
2733  * it sets FUSE_IOCTL_RETRY, out->in_iovs to 2 and iov array to
2734  *
2735  * { { .iov_base = inarg.arg,   .iov_len = sizeof(struct a)     },
2736  *   { .iov_base = a.buf,       .iov_len = a.buflen             } }
2737  *
2738  * FUSE will copy both struct a and the pointed buffer from the
2739  * process doing the ioctl and retry ioctl with both struct a and the
2740  * buffer.
2741  *
2742  * This time, FUSE server has everything it needs and completes ioctl
2743  * without FUSE_IOCTL_RETRY which finishes the ioctl call.
2744  *
2745  * Copying data out works the same way.
2746  *
2747  * Note that if FUSE_IOCTL_UNRESTRICTED is clear, the kernel
2748  * automatically initializes in and out iovs by decoding @cmd with
2749  * _IOC_* macros and the server is not allowed to request RETRY.  This
2750  * limits ioctl data transfers to well-formed ioctls and is the forced
2751  * behavior for all FUSE servers.
2752  */
2753 long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg,
2754                    unsigned int flags)
2755 {
2756         struct fuse_file *ff = file->private_data;
2757         struct fuse_mount *fm = ff->fm;
2758         struct fuse_ioctl_in inarg = {
2759                 .fh = ff->fh,
2760                 .cmd = cmd,
2761                 .arg = arg,
2762                 .flags = flags
2763         };
2764         struct fuse_ioctl_out outarg;
2765         struct iovec *iov_page = NULL;
2766         struct iovec *in_iov = NULL, *out_iov = NULL;
2767         unsigned int in_iovs = 0, out_iovs = 0, max_pages;
2768         size_t in_size, out_size, c;
2769         ssize_t transferred;
2770         int err, i;
2771         struct iov_iter ii;
2772         struct fuse_args_pages ap = {};
2773
2774 #if BITS_PER_LONG == 32
2775         inarg.flags |= FUSE_IOCTL_32BIT;
2776 #else
2777         if (flags & FUSE_IOCTL_COMPAT) {
2778                 inarg.flags |= FUSE_IOCTL_32BIT;
2779 #ifdef CONFIG_X86_X32
2780                 if (in_x32_syscall())
2781                         inarg.flags |= FUSE_IOCTL_COMPAT_X32;
2782 #endif
2783         }
2784 #endif
2785
2786         /* assume all the iovs returned by client always fits in a page */
2787         BUILD_BUG_ON(sizeof(struct fuse_ioctl_iovec) * FUSE_IOCTL_MAX_IOV > PAGE_SIZE);
2788
2789         err = -ENOMEM;
2790         ap.pages = fuse_pages_alloc(fm->fc->max_pages, GFP_KERNEL, &ap.descs);
2791         iov_page = (struct iovec *) __get_free_page(GFP_KERNEL);
2792         if (!ap.pages || !iov_page)
2793                 goto out;
2794
2795         fuse_page_descs_length_init(ap.descs, 0, fm->fc->max_pages);
2796
2797         /*
2798          * If restricted, initialize IO parameters as encoded in @cmd.
2799          * RETRY from server is not allowed.
2800          */
2801         if (!(flags & FUSE_IOCTL_UNRESTRICTED)) {
2802                 struct iovec *iov = iov_page;
2803
2804                 iov->iov_base = (void __user *)arg;
2805
2806                 switch (cmd) {
2807                 case FS_IOC_GETFLAGS:
2808                 case FS_IOC_SETFLAGS:
2809                         iov->iov_len = sizeof(int);
2810                         break;
2811                 default:
2812                         iov->iov_len = _IOC_SIZE(cmd);
2813                         break;
2814                 }
2815
2816                 if (_IOC_DIR(cmd) & _IOC_WRITE) {
2817                         in_iov = iov;
2818                         in_iovs = 1;
2819                 }
2820
2821                 if (_IOC_DIR(cmd) & _IOC_READ) {
2822                         out_iov = iov;
2823                         out_iovs = 1;
2824                 }
2825         }
2826
2827  retry:
2828         inarg.in_size = in_size = iov_length(in_iov, in_iovs);
2829         inarg.out_size = out_size = iov_length(out_iov, out_iovs);
2830
2831         /*
2832          * Out data can be used either for actual out data or iovs,
2833          * make sure there always is at least one page.
2834          */
2835         out_size = max_t(size_t, out_size, PAGE_SIZE);
2836         max_pages = DIV_ROUND_UP(max(in_size, out_size), PAGE_SIZE);
2837
2838         /* make sure there are enough buffer pages and init request with them */
2839         err = -ENOMEM;
2840         if (max_pages > fm->fc->max_pages)
2841                 goto out;
2842         while (ap.num_pages < max_pages) {
2843                 ap.pages[ap.num_pages] = alloc_page(GFP_KERNEL | __GFP_HIGHMEM);
2844                 if (!ap.pages[ap.num_pages])
2845                         goto out;
2846                 ap.num_pages++;
2847         }
2848
2849
2850         /* okay, let's send it to the client */
2851         ap.args.opcode = FUSE_IOCTL;
2852         ap.args.nodeid = ff->nodeid;
2853         ap.args.in_numargs = 1;
2854         ap.args.in_args[0].size = sizeof(inarg);
2855         ap.args.in_args[0].value = &inarg;
2856         if (in_size) {
2857                 ap.args.in_numargs++;
2858                 ap.args.in_args[1].size = in_size;
2859                 ap.args.in_pages = true;
2860
2861                 err = -EFAULT;
2862                 iov_iter_init(&ii, WRITE, in_iov, in_iovs, in_size);
2863                 for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= ap.num_pages); i++) {
2864                         c = copy_page_from_iter(ap.pages[i], 0, PAGE_SIZE, &ii);
2865                         if (c != PAGE_SIZE && iov_iter_count(&ii))
2866                                 goto out;
2867                 }
2868         }
2869
2870         ap.args.out_numargs = 2;
2871         ap.args.out_args[0].size = sizeof(outarg);
2872         ap.args.out_args[0].value = &outarg;
2873         ap.args.out_args[1].size = out_size;
2874         ap.args.out_pages = true;
2875         ap.args.out_argvar = true;
2876
2877         transferred = fuse_simple_request(fm, &ap.args);
2878         err = transferred;
2879         if (transferred < 0)
2880                 goto out;
2881
2882         /* did it ask for retry? */
2883         if (outarg.flags & FUSE_IOCTL_RETRY) {
2884                 void *vaddr;
2885
2886                 /* no retry if in restricted mode */
2887                 err = -EIO;
2888                 if (!(flags & FUSE_IOCTL_UNRESTRICTED))
2889                         goto out;
2890
2891                 in_iovs = outarg.in_iovs;
2892                 out_iovs = outarg.out_iovs;
2893
2894                 /*
2895                  * Make sure things are in boundary, separate checks
2896                  * are to protect against overflow.
2897                  */
2898                 err = -ENOMEM;
2899                 if (in_iovs > FUSE_IOCTL_MAX_IOV ||
2900                     out_iovs > FUSE_IOCTL_MAX_IOV ||
2901                     in_iovs + out_iovs > FUSE_IOCTL_MAX_IOV)
2902                         goto out;
2903
2904                 vaddr = kmap_atomic(ap.pages[0]);
2905                 err = fuse_copy_ioctl_iovec(fm->fc, iov_page, vaddr,
2906                                             transferred, in_iovs + out_iovs,
2907                                             (flags & FUSE_IOCTL_COMPAT) != 0);
2908                 kunmap_atomic(vaddr);
2909                 if (err)
2910                         goto out;
2911
2912                 in_iov = iov_page;
2913                 out_iov = in_iov + in_iovs;
2914
2915                 err = fuse_verify_ioctl_iov(fm->fc, in_iov, in_iovs);
2916                 if (err)
2917                         goto out;
2918
2919                 err = fuse_verify_ioctl_iov(fm->fc, out_iov, out_iovs);
2920                 if (err)
2921                         goto out;
2922
2923                 goto retry;
2924         }
2925
2926         err = -EIO;
2927         if (transferred > inarg.out_size)
2928                 goto out;
2929
2930         err = -EFAULT;
2931         iov_iter_init(&ii, READ, out_iov, out_iovs, transferred);
2932         for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= ap.num_pages); i++) {
2933                 c = copy_page_to_iter(ap.pages[i], 0, PAGE_SIZE, &ii);
2934                 if (c != PAGE_SIZE && iov_iter_count(&ii))
2935                         goto out;
2936         }
2937         err = 0;
2938  out:
2939         free_page((unsigned long) iov_page);
2940         while (ap.num_pages)
2941                 __free_page(ap.pages[--ap.num_pages]);
2942         kfree(ap.pages);
2943
2944         return err ? err : outarg.result;
2945 }
2946 EXPORT_SYMBOL_GPL(fuse_do_ioctl);
2947
2948 long fuse_ioctl_common(struct file *file, unsigned int cmd,
2949                        unsigned long arg, unsigned int flags)
2950 {
2951         struct inode *inode = file_inode(file);
2952         struct fuse_conn *fc = get_fuse_conn(inode);
2953
2954         if (!fuse_allow_current_process(fc))
2955                 return -EACCES;
2956
2957         if (is_bad_inode(inode))
2958                 return -EIO;
2959
2960         return fuse_do_ioctl(file, cmd, arg, flags);
2961 }
2962
2963 static long fuse_file_ioctl(struct file *file, unsigned int cmd,
2964                             unsigned long arg)
2965 {
2966         return fuse_ioctl_common(file, cmd, arg, 0);
2967 }
2968
2969 static long fuse_file_compat_ioctl(struct file *file, unsigned int cmd,
2970                                    unsigned long arg)
2971 {
2972         return fuse_ioctl_common(file, cmd, arg, FUSE_IOCTL_COMPAT);
2973 }
2974
2975 /*
2976  * All files which have been polled are linked to RB tree
2977  * fuse_conn->polled_files which is indexed by kh.  Walk the tree and
2978  * find the matching one.
2979  */
2980 static struct rb_node **fuse_find_polled_node(struct fuse_conn *fc, u64 kh,
2981                                               struct rb_node **parent_out)
2982 {
2983         struct rb_node **link = &fc->polled_files.rb_node;
2984         struct rb_node *last = NULL;
2985
2986         while (*link) {
2987                 struct fuse_file *ff;
2988
2989                 last = *link;
2990                 ff = rb_entry(last, struct fuse_file, polled_node);
2991
2992                 if (kh < ff->kh)
2993                         link = &last->rb_left;
2994                 else if (kh > ff->kh)
2995                         link = &last->rb_right;
2996                 else
2997                         return link;
2998         }
2999
3000         if (parent_out)
3001                 *parent_out = last;
3002         return link;
3003 }
3004
3005 /*
3006  * The file is about to be polled.  Make sure it's on the polled_files
3007  * RB tree.  Note that files once added to the polled_files tree are
3008  * not removed before the file is released.  This is because a file
3009  * polled once is likely to be polled again.
3010  */
3011 static void fuse_register_polled_file(struct fuse_conn *fc,
3012                                       struct fuse_file *ff)
3013 {
3014         spin_lock(&fc->lock);
3015         if (RB_EMPTY_NODE(&ff->polled_node)) {
3016                 struct rb_node **link, *parent;
3017
3018                 link = fuse_find_polled_node(fc, ff->kh, &parent);
3019                 BUG_ON(*link);
3020                 rb_link_node(&ff->polled_node, parent, link);
3021                 rb_insert_color(&ff->polled_node, &fc->polled_files);
3022         }
3023         spin_unlock(&fc->lock);
3024 }
3025
3026 __poll_t fuse_file_poll(struct file *file, poll_table *wait)
3027 {
3028         struct fuse_file *ff = file->private_data;
3029         struct fuse_mount *fm = ff->fm;
3030         struct fuse_poll_in inarg = { .fh = ff->fh, .kh = ff->kh };
3031         struct fuse_poll_out outarg;
3032         FUSE_ARGS(args);
3033         int err;
3034
3035         if (fm->fc->no_poll)
3036                 return DEFAULT_POLLMASK;
3037
3038         poll_wait(file, &ff->poll_wait, wait);
3039         inarg.events = mangle_poll(poll_requested_events(wait));
3040
3041         /*
3042          * Ask for notification iff there's someone waiting for it.
3043          * The client may ignore the flag and always notify.
3044          */
3045         if (waitqueue_active(&ff->poll_wait)) {
3046                 inarg.flags |= FUSE_POLL_SCHEDULE_NOTIFY;
3047                 fuse_register_polled_file(fm->fc, ff);
3048         }
3049
3050         args.opcode = FUSE_POLL;
3051         args.nodeid = ff->nodeid;
3052         args.in_numargs = 1;
3053         args.in_args[0].size = sizeof(inarg);
3054         args.in_args[0].value = &inarg;
3055         args.out_numargs = 1;
3056         args.out_args[0].size = sizeof(outarg);
3057         args.out_args[0].value = &outarg;
3058         err = fuse_simple_request(fm, &args);
3059
3060         if (!err)
3061                 return demangle_poll(outarg.revents);
3062         if (err == -ENOSYS) {
3063                 fm->fc->no_poll = 1;
3064                 return DEFAULT_POLLMASK;
3065         }
3066         return EPOLLERR;
3067 }
3068 EXPORT_SYMBOL_GPL(fuse_file_poll);
3069
3070 /*
3071  * This is called from fuse_handle_notify() on FUSE_NOTIFY_POLL and
3072  * wakes up the poll waiters.
3073  */
3074 int fuse_notify_poll_wakeup(struct fuse_conn *fc,
3075                             struct fuse_notify_poll_wakeup_out *outarg)
3076 {
3077         u64 kh = outarg->kh;
3078         struct rb_node **link;
3079
3080         spin_lock(&fc->lock);
3081
3082         link = fuse_find_polled_node(fc, kh, NULL);
3083         if (*link) {
3084                 struct fuse_file *ff;
3085
3086                 ff = rb_entry(*link, struct fuse_file, polled_node);
3087                 wake_up_interruptible_sync(&ff->poll_wait);
3088         }
3089
3090         spin_unlock(&fc->lock);
3091         return 0;
3092 }
3093
3094 static void fuse_do_truncate(struct file *file)
3095 {
3096         struct inode *inode = file->f_mapping->host;
3097         struct iattr attr;
3098
3099         attr.ia_valid = ATTR_SIZE;
3100         attr.ia_size = i_size_read(inode);
3101
3102         attr.ia_file = file;
3103         attr.ia_valid |= ATTR_FILE;
3104
3105         fuse_do_setattr(file_dentry(file), &attr, file);
3106 }
3107
3108 static inline loff_t fuse_round_up(struct fuse_conn *fc, loff_t off)
3109 {
3110         return round_up(off, fc->max_pages << PAGE_SHIFT);
3111 }
3112
3113 static ssize_t
3114 fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
3115 {
3116         DECLARE_COMPLETION_ONSTACK(wait);
3117         ssize_t ret = 0;
3118         struct file *file = iocb->ki_filp;
3119         struct fuse_file *ff = file->private_data;
3120         loff_t pos = 0;
3121         struct inode *inode;
3122         loff_t i_size;
3123         size_t count = iov_iter_count(iter), shortened = 0;
3124         loff_t offset = iocb->ki_pos;
3125         struct fuse_io_priv *io;
3126
3127         pos = offset;
3128         inode = file->f_mapping->host;
3129         i_size = i_size_read(inode);
3130
3131         if ((iov_iter_rw(iter) == READ) && (offset >= i_size))
3132                 return 0;
3133
3134         io = kmalloc(sizeof(struct fuse_io_priv), GFP_KERNEL);
3135         if (!io)
3136                 return -ENOMEM;
3137         spin_lock_init(&io->lock);
3138         kref_init(&io->refcnt);
3139         io->reqs = 1;
3140         io->bytes = -1;
3141         io->size = 0;
3142         io->offset = offset;
3143         io->write = (iov_iter_rw(iter) == WRITE);
3144         io->err = 0;
3145         /*
3146          * By default, we want to optimize all I/Os with async request
3147          * submission to the client filesystem if supported.
3148          */
3149         io->async = ff->fm->fc->async_dio;
3150         io->iocb = iocb;
3151         io->blocking = is_sync_kiocb(iocb);
3152
3153         /* optimization for short read */
3154         if (io->async && !io->write && offset + count > i_size) {
3155                 iov_iter_truncate(iter, fuse_round_up(ff->fm->fc, i_size - offset));
3156                 shortened = count - iov_iter_count(iter);
3157                 count -= shortened;
3158         }
3159
3160         /*
3161          * We cannot asynchronously extend the size of a file.
3162          * In such case the aio will behave exactly like sync io.
3163          */
3164         if ((offset + count > i_size) && io->write)
3165                 io->blocking = true;
3166
3167         if (io->async && io->blocking) {
3168                 /*
3169                  * Additional reference to keep io around after
3170                  * calling fuse_aio_complete()
3171                  */
3172                 kref_get(&io->refcnt);
3173                 io->done = &wait;
3174         }
3175
3176         if (iov_iter_rw(iter) == WRITE) {
3177                 ret = fuse_direct_io(io, iter, &pos, FUSE_DIO_WRITE);
3178                 fuse_invalidate_attr(inode);
3179         } else {
3180                 ret = __fuse_direct_read(io, iter, &pos);
3181         }
3182         iov_iter_reexpand(iter, iov_iter_count(iter) + shortened);
3183
3184         if (io->async) {
3185                 bool blocking = io->blocking;
3186
3187                 fuse_aio_complete(io, ret < 0 ? ret : 0, -1);
3188
3189                 /* we have a non-extending, async request, so return */
3190                 if (!blocking)
3191                         return -EIOCBQUEUED;
3192
3193                 wait_for_completion(&wait);
3194                 ret = fuse_get_res_by_io(io);
3195         }
3196
3197         kref_put(&io->refcnt, fuse_io_release);
3198
3199         if (iov_iter_rw(iter) == WRITE) {
3200                 if (ret > 0)
3201                         fuse_write_update_size(inode, pos);
3202                 else if (ret < 0 && offset + count > i_size)
3203                         fuse_do_truncate(file);
3204         }
3205
3206         return ret;
3207 }
3208
3209 static int fuse_writeback_range(struct inode *inode, loff_t start, loff_t end)
3210 {
3211         int err = filemap_write_and_wait_range(inode->i_mapping, start, end);
3212
3213         if (!err)
3214                 fuse_sync_writes(inode);
3215
3216         return err;
3217 }
3218
3219 static long fuse_file_fallocate(struct file *file, int mode, loff_t offset,
3220                                 loff_t length)
3221 {
3222         struct fuse_file *ff = file->private_data;
3223         struct inode *inode = file_inode(file);
3224         struct fuse_inode *fi = get_fuse_inode(inode);
3225         struct fuse_mount *fm = ff->fm;
3226         FUSE_ARGS(args);
3227         struct fuse_fallocate_in inarg = {
3228                 .fh = ff->fh,
3229                 .offset = offset,
3230                 .length = length,
3231                 .mode = mode
3232         };
3233         int err;
3234         bool lock_inode = !(mode & FALLOC_FL_KEEP_SIZE) ||
3235                            (mode & FALLOC_FL_PUNCH_HOLE);
3236
3237         bool block_faults = FUSE_IS_DAX(inode) && lock_inode;
3238
3239         if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
3240                 return -EOPNOTSUPP;
3241
3242         if (fm->fc->no_fallocate)
3243                 return -EOPNOTSUPP;
3244
3245         if (lock_inode) {
3246                 inode_lock(inode);
3247                 if (block_faults) {
3248                         down_write(&fi->i_mmap_sem);
3249                         err = fuse_dax_break_layouts(inode, 0, 0);
3250                         if (err)
3251                                 goto out;
3252                 }
3253
3254                 if (mode & FALLOC_FL_PUNCH_HOLE) {
3255                         loff_t endbyte = offset + length - 1;
3256
3257                         err = fuse_writeback_range(inode, offset, endbyte);
3258                         if (err)
3259                                 goto out;
3260                 }
3261         }
3262
3263         if (!(mode & FALLOC_FL_KEEP_SIZE) &&
3264             offset + length > i_size_read(inode)) {
3265                 err = inode_newsize_ok(inode, offset + length);
3266                 if (err)
3267                         goto out;
3268         }
3269
3270         if (!(mode & FALLOC_FL_KEEP_SIZE))
3271                 set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
3272
3273         args.opcode = FUSE_FALLOCATE;
3274         args.nodeid = ff->nodeid;
3275         args.in_numargs = 1;
3276         args.in_args[0].size = sizeof(inarg);
3277         args.in_args[0].value = &inarg;
3278         err = fuse_simple_request(fm, &args);
3279         if (err == -ENOSYS) {
3280                 fm->fc->no_fallocate = 1;
3281                 err = -EOPNOTSUPP;
3282         }
3283         if (err)
3284                 goto out;
3285
3286         /* we could have extended the file */
3287         if (!(mode & FALLOC_FL_KEEP_SIZE)) {
3288                 bool changed = fuse_write_update_size(inode, offset + length);
3289
3290                 if (changed && fm->fc->writeback_cache)
3291                         file_update_time(file);
3292         }
3293
3294         if (mode & FALLOC_FL_PUNCH_HOLE)
3295                 truncate_pagecache_range(inode, offset, offset + length - 1);
3296
3297         fuse_invalidate_attr(inode);
3298
3299 out:
3300         if (!(mode & FALLOC_FL_KEEP_SIZE))
3301                 clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
3302
3303         if (block_faults)
3304                 up_write(&fi->i_mmap_sem);
3305
3306         if (lock_inode)
3307                 inode_unlock(inode);
3308
3309         return err;
3310 }
3311
3312 static ssize_t __fuse_copy_file_range(struct file *file_in, loff_t pos_in,
3313                                       struct file *file_out, loff_t pos_out,
3314                                       size_t len, unsigned int flags)
3315 {
3316         struct fuse_file *ff_in = file_in->private_data;
3317         struct fuse_file *ff_out = file_out->private_data;
3318         struct inode *inode_in = file_inode(file_in);
3319         struct inode *inode_out = file_inode(file_out);
3320         struct fuse_inode *fi_out = get_fuse_inode(inode_out);
3321         struct fuse_mount *fm = ff_in->fm;
3322         struct fuse_conn *fc = fm->fc;
3323         FUSE_ARGS(args);
3324         struct fuse_copy_file_range_in inarg = {
3325                 .fh_in = ff_in->fh,
3326                 .off_in = pos_in,
3327                 .nodeid_out = ff_out->nodeid,
3328                 .fh_out = ff_out->fh,
3329                 .off_out = pos_out,
3330                 .len = len,
3331                 .flags = flags
3332         };
3333         struct fuse_write_out outarg;
3334         ssize_t err;
3335         /* mark unstable when write-back is not used, and file_out gets
3336          * extended */
3337         bool is_unstable = (!fc->writeback_cache) &&
3338                            ((pos_out + len) > inode_out->i_size);
3339
3340         if (fc->no_copy_file_range)
3341                 return -EOPNOTSUPP;
3342
3343         if (file_inode(file_in)->i_sb != file_inode(file_out)->i_sb)
3344                 return -EXDEV;
3345
3346         inode_lock(inode_in);
3347         err = fuse_writeback_range(inode_in, pos_in, pos_in + len - 1);
3348         inode_unlock(inode_in);
3349         if (err)
3350                 return err;
3351
3352         inode_lock(inode_out);
3353
3354         err = file_modified(file_out);
3355         if (err)
3356                 goto out;
3357
3358         /*
3359          * Write out dirty pages in the destination file before sending the COPY
3360          * request to userspace.  After the request is completed, truncate off
3361          * pages (including partial ones) from the cache that have been copied,
3362          * since these contain stale data at that point.
3363          *
3364          * This should be mostly correct, but if the COPY writes to partial
3365          * pages (at the start or end) and the parts not covered by the COPY are
3366          * written through a memory map after calling fuse_writeback_range(),
3367          * then these partial page modifications will be lost on truncation.
3368          *
3369          * It is unlikely that someone would rely on such mixed style
3370          * modifications.  Yet this does give less guarantees than if the
3371          * copying was performed with write(2).
3372          *
3373          * To fix this a i_mmap_sem style lock could be used to prevent new
3374          * faults while the copy is ongoing.
3375          */
3376         err = fuse_writeback_range(inode_out, pos_out, pos_out + len - 1);
3377         if (err)
3378                 goto out;
3379
3380         if (is_unstable)
3381                 set_bit(FUSE_I_SIZE_UNSTABLE, &fi_out->state);
3382
3383         args.opcode = FUSE_COPY_FILE_RANGE;
3384         args.nodeid = ff_in->nodeid;
3385         args.in_numargs = 1;
3386         args.in_args[0].size = sizeof(inarg);
3387         args.in_args[0].value = &inarg;
3388         args.out_numargs = 1;
3389         args.out_args[0].size = sizeof(outarg);
3390         args.out_args[0].value = &outarg;
3391         err = fuse_simple_request(fm, &args);
3392         if (err == -ENOSYS) {
3393                 fc->no_copy_file_range = 1;
3394                 err = -EOPNOTSUPP;
3395         }
3396         if (err)
3397                 goto out;
3398
3399         truncate_inode_pages_range(inode_out->i_mapping,
3400                                    ALIGN_DOWN(pos_out, PAGE_SIZE),
3401                                    ALIGN(pos_out + outarg.size, PAGE_SIZE) - 1);
3402
3403         if (fc->writeback_cache) {
3404                 fuse_write_update_size(inode_out, pos_out + outarg.size);
3405                 file_update_time(file_out);
3406         }
3407
3408         fuse_invalidate_attr(inode_out);
3409
3410         err = outarg.size;
3411 out:
3412         if (is_unstable)
3413                 clear_bit(FUSE_I_SIZE_UNSTABLE, &fi_out->state);
3414
3415         inode_unlock(inode_out);
3416         file_accessed(file_in);
3417
3418         return err;
3419 }
3420
3421 static ssize_t fuse_copy_file_range(struct file *src_file, loff_t src_off,
3422                                     struct file *dst_file, loff_t dst_off,
3423                                     size_t len, unsigned int flags)
3424 {
3425         ssize_t ret;
3426
3427         ret = __fuse_copy_file_range(src_file, src_off, dst_file, dst_off,
3428                                      len, flags);
3429
3430         if (ret == -EOPNOTSUPP || ret == -EXDEV)
3431                 ret = generic_copy_file_range(src_file, src_off, dst_file,
3432                                               dst_off, len, flags);
3433         return ret;
3434 }
3435
3436 static const struct file_operations fuse_file_operations = {
3437         .llseek         = fuse_file_llseek,
3438         .read_iter      = fuse_file_read_iter,
3439         .write_iter     = fuse_file_write_iter,
3440         .mmap           = fuse_file_mmap,
3441         .open           = fuse_open,
3442         .flush          = fuse_flush,
3443         .release        = fuse_release,
3444         .fsync          = fuse_fsync,
3445         .lock           = fuse_file_lock,
3446         .get_unmapped_area = thp_get_unmapped_area,
3447         .flock          = fuse_file_flock,
3448         .splice_read    = generic_file_splice_read,
3449         .splice_write   = iter_file_splice_write,
3450         .unlocked_ioctl = fuse_file_ioctl,
3451         .compat_ioctl   = fuse_file_compat_ioctl,
3452         .poll           = fuse_file_poll,
3453         .fallocate      = fuse_file_fallocate,
3454         .copy_file_range = fuse_copy_file_range,
3455 };
3456
3457 static const struct address_space_operations fuse_file_aops  = {
3458         .readpage       = fuse_readpage,
3459         .readahead      = fuse_readahead,
3460         .writepage      = fuse_writepage,
3461         .writepages     = fuse_writepages,
3462         .launder_page   = fuse_launder_page,
3463         .set_page_dirty = __set_page_dirty_nobuffers,
3464         .bmap           = fuse_bmap,
3465         .direct_IO      = fuse_direct_IO,
3466         .write_begin    = fuse_write_begin,
3467         .write_end      = fuse_write_end,
3468 };
3469
3470 void fuse_init_file_inode(struct inode *inode)
3471 {
3472         struct fuse_inode *fi = get_fuse_inode(inode);
3473
3474         inode->i_fop = &fuse_file_operations;
3475         inode->i_data.a_ops = &fuse_file_aops;
3476
3477         INIT_LIST_HEAD(&fi->write_files);
3478         INIT_LIST_HEAD(&fi->queued_writes);
3479         fi->writectr = 0;
3480         init_waitqueue_head(&fi->page_waitq);
3481         fi->writepages = RB_ROOT;
3482
3483         if (IS_ENABLED(CONFIG_FUSE_DAX))
3484                 fuse_dax_inode_init(inode);
3485 }