fs/xfs/xfs_iomap.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2000-2006 Silicon Graphics, Inc.
   4  * Copyright (c) 2016-2018 Christoph Hellwig.
   5  * All Rights Reserved.
   6  */
   7 #include "xfs.h"
   8 #include "xfs_fs.h"
   9 #include "xfs_shared.h"
  10 #include "xfs_format.h"
  11 #include "xfs_log_format.h"
  12 #include "xfs_trans_resv.h"
  13 #include "xfs_mount.h"
  14 #include "xfs_inode.h"
  15 #include "xfs_btree.h"
  16 #include "xfs_bmap_btree.h"
  17 #include "xfs_bmap.h"
  18 #include "xfs_bmap_util.h"
  19 #include "xfs_errortag.h"
  20 #include "xfs_error.h"
  21 #include "xfs_trans.h"
  22 #include "xfs_trans_space.h"
  23 #include "xfs_inode_item.h"
  24 #include "xfs_iomap.h"
  25 #include "xfs_trace.h"
  26 #include "xfs_quota.h"
  27 #include "xfs_dquot_item.h"
  28 #include "xfs_dquot.h"
  29 #include "xfs_reflink.h"
  30 #include "xfs_health.h"
  31 #include "xfs_rtbitmap.h"
  32
  33 #define XFS_ALLOC_ALIGN(mp, off) \
  34         (((off) >> mp->m_allocsize_log) << mp->m_allocsize_log)
  35
  36 static int
  37 xfs_alert_fsblock_zero(
  38         xfs_inode_t     *ip,
  39         xfs_bmbt_irec_t *imap)
  40 {
  41         xfs_alert_tag(ip->i_mount, XFS_PTAG_FSBLOCK_ZERO,
  42                         "Access to block zero in inode %llu "
  43                         "start_block: %llx start_off: %llx "
  44                         "blkcnt: %llx extent-state: %x",
  45                 (unsigned long long)ip->i_ino,
  46                 (unsigned long long)imap->br_startblock,
  47                 (unsigned long long)imap->br_startoff,
  48                 (unsigned long long)imap->br_blockcount,
  49                 imap->br_state);
  50         xfs_bmap_mark_sick(ip, XFS_DATA_FORK);
  51         return -EFSCORRUPTED;
  52 }
  53
  54 u64
  55 xfs_iomap_inode_sequence(
  56         struct xfs_inode        *ip,
  57         u16                     iomap_flags)
  58 {
  59         u64                     cookie = 0;
  60
  61         if (iomap_flags & IOMAP_F_XATTR)
  62                 return READ_ONCE(ip->i_af.if_seq);
  63         if ((iomap_flags & IOMAP_F_SHARED) && ip->i_cowfp)
  64                 cookie = (u64)READ_ONCE(ip->i_cowfp->if_seq) << 32;
  65         return cookie | READ_ONCE(ip->i_df.if_seq);
  66 }
  67
  68 /*
  69  * Check that the iomap passed to us is still valid for the given offset and
  70  * length.
  71  */
  72 static bool
  73 xfs_iomap_valid(
  74         struct inode            *inode,
  75         const struct iomap      *iomap)
  76 {
  77         struct xfs_inode        *ip = XFS_I(inode);
  78
  79         if (iomap->validity_cookie !=
  80                         xfs_iomap_inode_sequence(ip, iomap->flags)) {
  81                 trace_xfs_iomap_invalid(ip, iomap);
  82                 return false;
  83         }
  84
  85         XFS_ERRORTAG_DELAY(ip->i_mount, XFS_ERRTAG_WRITE_DELAY_MS);
  86         return true;
  87 }
  88
  89 static const struct iomap_folio_ops xfs_iomap_folio_ops = {
  90         .iomap_valid            = xfs_iomap_valid,
  91 };
  92
  93 int
  94 xfs_bmbt_to_iomap(
  95         struct xfs_inode        *ip,
  96         struct iomap            *iomap,
  97         struct xfs_bmbt_irec    *imap,
  98         unsigned int            mapping_flags,
  99         u16                     iomap_flags,
 100         u64                     sequence_cookie)
 101 {
 102         struct xfs_mount        *mp = ip->i_mount;
 103         struct xfs_buftarg      *target = xfs_inode_buftarg(ip);
 104
 105         if (unlikely(!xfs_valid_startblock(ip, imap->br_startblock))) {
 106                 xfs_bmap_mark_sick(ip, XFS_DATA_FORK);
 107                 return xfs_alert_fsblock_zero(ip, imap);
 108         }
 109
 110         if (imap->br_startblock == HOLESTARTBLOCK) {
 111                 iomap->addr = IOMAP_NULL_ADDR;
 112                 iomap->type = IOMAP_HOLE;
 113         } else if (imap->br_startblock == DELAYSTARTBLOCK ||
 114                    isnullstartblock(imap->br_startblock)) {
 115                 iomap->addr = IOMAP_NULL_ADDR;
 116                 iomap->type = IOMAP_DELALLOC;
 117         } else {
 118                 iomap->addr = BBTOB(xfs_fsb_to_db(ip, imap->br_startblock));
 119                 if (mapping_flags & IOMAP_DAX)
 120                         iomap->addr += target->bt_dax_part_off;
 121
 122                 if (imap->br_state == XFS_EXT_UNWRITTEN)
 123                         iomap->type = IOMAP_UNWRITTEN;
 124                 else
 125                         iomap->type = IOMAP_MAPPED;
 126
 127         }
 128         iomap->offset = XFS_FSB_TO_B(mp, imap->br_startoff);
 129         iomap->length = XFS_FSB_TO_B(mp, imap->br_blockcount);
 130         if (mapping_flags & IOMAP_DAX)
 131                 iomap->dax_dev = target->bt_daxdev;
 132         else
 133                 iomap->bdev = target->bt_bdev;
 134         iomap->flags = iomap_flags;
 135
 136         if (xfs_ipincount(ip) &&
 137             (ip->i_itemp->ili_fsync_fields & ~XFS_ILOG_TIMESTAMP))
 138                 iomap->flags |= IOMAP_F_DIRTY;
 139
 140         iomap->validity_cookie = sequence_cookie;
 141         iomap->folio_ops = &xfs_iomap_folio_ops;
 142         return 0;
 143 }
 144
 145 static void
 146 xfs_hole_to_iomap(
 147         struct xfs_inode        *ip,
 148         struct iomap            *iomap,
 149         xfs_fileoff_t           offset_fsb,
 150         xfs_fileoff_t           end_fsb)
 151 {
 152         struct xfs_buftarg      *target = xfs_inode_buftarg(ip);
 153
 154         iomap->addr = IOMAP_NULL_ADDR;
 155         iomap->type = IOMAP_HOLE;
 156         iomap->offset = XFS_FSB_TO_B(ip->i_mount, offset_fsb);
 157         iomap->length = XFS_FSB_TO_B(ip->i_mount, end_fsb - offset_fsb);
 158         iomap->bdev = target->bt_bdev;
 159         iomap->dax_dev = target->bt_daxdev;
 160 }
 161
 162 static inline xfs_fileoff_t
 163 xfs_iomap_end_fsb(
 164         struct xfs_mount        *mp,
 165         loff_t                  offset,
 166         loff_t                  count)
 167 {
 168         ASSERT(offset <= mp->m_super->s_maxbytes);
 169         return min(XFS_B_TO_FSB(mp, offset + count),
 170                    XFS_B_TO_FSB(mp, mp->m_super->s_maxbytes));
 171 }
 172
 173 static xfs_extlen_t
 174 xfs_eof_alignment(
 175         struct xfs_inode        *ip)
 176 {
 177         struct xfs_mount        *mp = ip->i_mount;
 178         xfs_extlen_t            align = 0;
 179
 180         if (!XFS_IS_REALTIME_INODE(ip)) {
 181                 /*
 182                  * Round up the allocation request to a stripe unit
 183                  * (m_dalign) boundary if the file size is >= stripe unit
 184                  * size, and we are allocating past the allocation eof.
 185                  *
 186                  * If mounted with the "-o swalloc" option the alignment is
 187                  * increased from the strip unit size to the stripe width.
 188                  */
 189                 if (mp->m_swidth && xfs_has_swalloc(mp))
 190                         align = mp->m_swidth;
 191                 else if (mp->m_dalign)
 192                         align = mp->m_dalign;
 193
 194                 if (align && XFS_ISIZE(ip) < XFS_FSB_TO_B(mp, align))
 195                         align = 0;
 196         }
 197
 198         return align;
 199 }
 200
 201 /*
 202  * Check if last_fsb is outside the last extent, and if so grow it to the next
 203  * stripe unit boundary.
 204  */
 205 xfs_fileoff_t
 206 xfs_iomap_eof_align_last_fsb(
 207         struct xfs_inode        *ip,
 208         xfs_fileoff_t           end_fsb)
 209 {
 210         struct xfs_ifork        *ifp = xfs_ifork_ptr(ip, XFS_DATA_FORK);
 211         xfs_extlen_t            extsz = xfs_get_extsz_hint(ip);
 212         xfs_extlen_t            align = xfs_eof_alignment(ip);
 213         struct xfs_bmbt_irec    irec;
 214         struct xfs_iext_cursor  icur;
 215
 216         ASSERT(!xfs_need_iread_extents(ifp));
 217
 218         /*
 219          * Always round up the allocation request to the extent hint boundary.
 220          */
 221         if (extsz) {
 222                 if (align)
 223                         align = roundup_64(align, extsz);
 224                 else
 225                         align = extsz;
 226         }
 227
 228         if (align) {
 229                 xfs_fileoff_t   aligned_end_fsb = roundup_64(end_fsb, align);
 230
 231                 xfs_iext_last(ifp, &icur);
 232                 if (!xfs_iext_get_extent(ifp, &icur, &irec) ||
 233                     aligned_end_fsb >= irec.br_startoff + irec.br_blockcount)
 234                         return aligned_end_fsb;
 235         }
 236
 237         return end_fsb;
 238 }
 239
 240 int
 241 xfs_iomap_write_direct(
 242         struct xfs_inode        *ip,
 243         xfs_fileoff_t           offset_fsb,
 244         xfs_fileoff_t           count_fsb,
 245         unsigned int            flags,
 246         struct xfs_bmbt_irec    *imap,
 247         u64                     *seq)
 248 {
 249         struct xfs_mount        *mp = ip->i_mount;
 250         struct xfs_trans        *tp;
 251         xfs_filblks_t           resaligned;
 252         int                     nimaps;
 253         unsigned int            dblocks, rblocks;
 254         bool                    force = false;
 255         int                     error;
 256         int                     bmapi_flags = XFS_BMAPI_PREALLOC;
 257         int                     nr_exts = XFS_IEXT_ADD_NOSPLIT_CNT;
 258
 259         ASSERT(count_fsb > 0);
 260
 261         resaligned = xfs_aligned_fsb_count(offset_fsb, count_fsb,
 262                                            xfs_get_extsz_hint(ip));
 263         if (unlikely(XFS_IS_REALTIME_INODE(ip))) {
 264                 dblocks = XFS_DIOSTRAT_SPACE_RES(mp, 0);
 265                 rblocks = resaligned;
 266         } else {
 267                 dblocks = XFS_DIOSTRAT_SPACE_RES(mp, resaligned);
 268                 rblocks = 0;
 269         }
 270
 271         error = xfs_qm_dqattach(ip);
 272         if (error)
 273                 return error;
 274
 275         /*
 276          * For DAX, we do not allocate unwritten extents, but instead we zero
 277          * the block before we commit the transaction.  Ideally we'd like to do
 278          * this outside the transaction context, but if we commit and then crash
 279          * we may not have zeroed the blocks and this will be exposed on
 280          * recovery of the allocation. Hence we must zero before commit.
 281          *
 282          * Further, if we are mapping unwritten extents here, we need to zero
 283          * and convert them to written so that we don't need an unwritten extent
 284          * callback for DAX. This also means that we need to be able to dip into
 285          * the reserve block pool for bmbt block allocation if there is no space
 286          * left but we need to do unwritten extent conversion.
 287          */
 288         if (flags & IOMAP_DAX) {
 289                 bmapi_flags = XFS_BMAPI_CONVERT | XFS_BMAPI_ZERO;
 290                 if (imap->br_state == XFS_EXT_UNWRITTEN) {
 291                         force = true;
 292                         nr_exts = XFS_IEXT_WRITE_UNWRITTEN_CNT;
 293                         dblocks = XFS_DIOSTRAT_SPACE_RES(mp, 0) << 1;
 294                 }
 295         }
 296
 297         error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, dblocks,
 298                         rblocks, force, &tp);
 299         if (error)
 300                 return error;
 301
 302         error = xfs_iext_count_extend(tp, ip, XFS_DATA_FORK, nr_exts);
 303         if (error)
 304                 goto out_trans_cancel;
 305
 306         /*
 307          * From this point onwards we overwrite the imap pointer that the
 308          * caller gave to us.
 309          */
 310         nimaps = 1;
 311         error = xfs_bmapi_write(tp, ip, offset_fsb, count_fsb, bmapi_flags, 0,
 312                                 imap, &nimaps);
 313         if (error)
 314                 goto out_trans_cancel;
 315
 316         /*
 317          * Complete the transaction
 318          */
 319         error = xfs_trans_commit(tp);
 320         if (error)
 321                 goto out_unlock;
 322
 323         if (unlikely(!xfs_valid_startblock(ip, imap->br_startblock))) {
 324                 xfs_bmap_mark_sick(ip, XFS_DATA_FORK);
 325                 error = xfs_alert_fsblock_zero(ip, imap);
 326         }
 327
 328 out_unlock:
 329         *seq = xfs_iomap_inode_sequence(ip, 0);
 330         xfs_iunlock(ip, XFS_ILOCK_EXCL);
 331         return error;
 332
 333 out_trans_cancel:
 334         xfs_trans_cancel(tp);
 335         goto out_unlock;
 336 }
 337
 338 STATIC bool
 339 xfs_quota_need_throttle(
 340         struct xfs_inode        *ip,
 341         xfs_dqtype_t            type,
 342         xfs_fsblock_t           alloc_blocks)
 343 {
 344         struct xfs_dquot        *dq = xfs_inode_dquot(ip, type);
 345
 346         if (!dq || !xfs_this_quota_on(ip->i_mount, type))
 347                 return false;
 348
 349         /* no hi watermark, no throttle */
 350         if (!dq->q_prealloc_hi_wmark)
 351                 return false;
 352
 353         /* under the lo watermark, no throttle */
 354         if (dq->q_blk.reserved + alloc_blocks < dq->q_prealloc_lo_wmark)
 355                 return false;
 356
 357         return true;
 358 }
 359
 360 STATIC void
 361 xfs_quota_calc_throttle(
 362         struct xfs_inode        *ip,
 363         xfs_dqtype_t            type,
 364         xfs_fsblock_t           *qblocks,
 365         int                     *qshift,
 366         int64_t                 *qfreesp)
 367 {
 368         struct xfs_dquot        *dq = xfs_inode_dquot(ip, type);
 369         int64_t                 freesp;
 370         int                     shift = 0;
 371
 372         /* no dq, or over hi wmark, squash the prealloc completely */
 373         if (!dq || dq->q_blk.reserved >= dq->q_prealloc_hi_wmark) {
 374                 *qblocks = 0;
 375                 *qfreesp = 0;
 376                 return;
 377         }
 378
 379         freesp = dq->q_prealloc_hi_wmark - dq->q_blk.reserved;
 380         if (freesp < dq->q_low_space[XFS_QLOWSP_5_PCNT]) {
 381                 shift = 2;
 382                 if (freesp < dq->q_low_space[XFS_QLOWSP_3_PCNT])
 383                         shift += 2;
 384                 if (freesp < dq->q_low_space[XFS_QLOWSP_1_PCNT])
 385                         shift += 2;
 386         }
 387
 388         if (freesp < *qfreesp)
 389                 *qfreesp = freesp;
 390
 391         /* only overwrite the throttle values if we are more aggressive */
 392         if ((freesp >> shift) < (*qblocks >> *qshift)) {
 393                 *qblocks = freesp;
 394                 *qshift = shift;
 395         }
 396 }
 397
 398 static int64_t
 399 xfs_iomap_freesp(
 400         struct percpu_counter   *counter,
 401         uint64_t                low_space[XFS_LOWSP_MAX],
 402         int                     *shift)
 403 {
 404         int64_t                 freesp;
 405
 406         freesp = percpu_counter_read_positive(counter);
 407         if (freesp < low_space[XFS_LOWSP_5_PCNT]) {
 408                 *shift = 2;
 409                 if (freesp < low_space[XFS_LOWSP_4_PCNT])
 410                         (*shift)++;
 411                 if (freesp < low_space[XFS_LOWSP_3_PCNT])
 412                         (*shift)++;
 413                 if (freesp < low_space[XFS_LOWSP_2_PCNT])
 414                         (*shift)++;
 415                 if (freesp < low_space[XFS_LOWSP_1_PCNT])
 416                         (*shift)++;
 417         }
 418         return freesp;
 419 }
 420
 421 /*
 422  * If we don't have a user specified preallocation size, dynamically increase
 423  * the preallocation size as the size of the file grows.  Cap the maximum size
 424  * at a single extent or less if the filesystem is near full. The closer the
 425  * filesystem is to being full, the smaller the maximum preallocation.
 426  */
 427 STATIC xfs_fsblock_t
 428 xfs_iomap_prealloc_size(
 429         struct xfs_inode        *ip,
 430         int                     whichfork,
 431         loff_t                  offset,
 432         loff_t                  count,
 433         struct xfs_iext_cursor  *icur)
 434 {
 435         struct xfs_iext_cursor  ncur = *icur;
 436         struct xfs_bmbt_irec    prev, got;
 437         struct xfs_mount        *mp = ip->i_mount;
 438         struct xfs_ifork        *ifp = xfs_ifork_ptr(ip, whichfork);
 439         xfs_fileoff_t           offset_fsb = XFS_B_TO_FSBT(mp, offset);
 440         int64_t                 freesp;
 441         xfs_fsblock_t           qblocks;
 442         xfs_fsblock_t           alloc_blocks = 0;
 443         xfs_extlen_t            plen;
 444         int                     shift = 0;
 445         int                     qshift = 0;
 446
 447         /*
 448          * As an exception we don't do any preallocation at all if the file is
 449          * smaller than the minimum preallocation and we are using the default
 450          * dynamic preallocation scheme, as it is likely this is the only write
 451          * to the file that is going to be done.
 452          */
 453         if (XFS_ISIZE(ip) < XFS_FSB_TO_B(mp, mp->m_allocsize_blocks))
 454                 return 0;
 455
 456         /*
 457          * Use the minimum preallocation size for small files or if we are
 458          * writing right after a hole.
 459          */
 460         if (XFS_ISIZE(ip) < XFS_FSB_TO_B(mp, mp->m_dalign) ||
 461             !xfs_iext_prev_extent(ifp, &ncur, &prev) ||
 462             prev.br_startoff + prev.br_blockcount < offset_fsb)
 463                 return mp->m_allocsize_blocks;
 464
 465         /*
 466          * Take the size of the preceding data extents as the basis for the
 467          * preallocation size. Note that we don't care if the previous extents
 468          * are written or not.
 469          */
 470         plen = prev.br_blockcount;
 471         while (xfs_iext_prev_extent(ifp, &ncur, &got)) {
 472                 if (plen > XFS_MAX_BMBT_EXTLEN / 2 ||
 473                     isnullstartblock(got.br_startblock) ||
 474                     got.br_startoff + got.br_blockcount != prev.br_startoff ||
 475                     got.br_startblock + got.br_blockcount != prev.br_startblock)
 476                         break;
 477                 plen += got.br_blockcount;
 478                 prev = got;
 479         }
 480
 481         /*
 482          * If the size of the extents is greater than half the maximum extent
 483          * length, then use the current offset as the basis.  This ensures that
 484          * for large files the preallocation size always extends to
 485          * XFS_BMBT_MAX_EXTLEN rather than falling short due to things like stripe
 486          * unit/width alignment of real extents.
 487          */
 488         alloc_blocks = plen * 2;
 489         if (alloc_blocks > XFS_MAX_BMBT_EXTLEN)
 490                 alloc_blocks = XFS_B_TO_FSB(mp, offset);
 491         qblocks = alloc_blocks;
 492
 493         /*
 494          * XFS_BMBT_MAX_EXTLEN is not a power of two value but we round the prealloc
 495          * down to the nearest power of two value after throttling. To prevent
 496          * the round down from unconditionally reducing the maximum supported
 497          * prealloc size, we round up first, apply appropriate throttling, round
 498          * down and cap the value to XFS_BMBT_MAX_EXTLEN.
 499          */
 500         alloc_blocks = XFS_FILEOFF_MIN(roundup_pow_of_two(XFS_MAX_BMBT_EXTLEN),
 501                                        alloc_blocks);
 502
 503         if (unlikely(XFS_IS_REALTIME_INODE(ip)))
 504                 freesp = xfs_rtx_to_rtb(mp,
 505                         xfs_iomap_freesp(&mp->m_frextents,
 506                                         mp->m_low_rtexts, &shift));
 507         else
 508                 freesp = xfs_iomap_freesp(&mp->m_fdblocks, mp->m_low_space,
 509                                 &shift);
 510
 511         /*
 512          * Check each quota to cap the prealloc size, provide a shift value to
 513          * throttle with and adjust amount of available space.
 514          */
 515         if (xfs_quota_need_throttle(ip, XFS_DQTYPE_USER, alloc_blocks))
 516                 xfs_quota_calc_throttle(ip, XFS_DQTYPE_USER, &qblocks, &qshift,
 517                                         &freesp);
 518         if (xfs_quota_need_throttle(ip, XFS_DQTYPE_GROUP, alloc_blocks))
 519                 xfs_quota_calc_throttle(ip, XFS_DQTYPE_GROUP, &qblocks, &qshift,
 520                                         &freesp);
 521         if (xfs_quota_need_throttle(ip, XFS_DQTYPE_PROJ, alloc_blocks))
 522                 xfs_quota_calc_throttle(ip, XFS_DQTYPE_PROJ, &qblocks, &qshift,
 523                                         &freesp);
 524
 525         /*
 526          * The final prealloc size is set to the minimum of free space available
 527          * in each of the quotas and the overall filesystem.
 528          *
 529          * The shift throttle value is set to the maximum value as determined by
 530          * the global low free space values and per-quota low free space values.
 531          */
 532         alloc_blocks = min(alloc_blocks, qblocks);
 533         shift = max(shift, qshift);
 534
 535         if (shift)
 536                 alloc_blocks >>= shift;
 537         /*
 538          * rounddown_pow_of_two() returns an undefined result if we pass in
 539          * alloc_blocks = 0.
 540          */
 541         if (alloc_blocks)
 542                 alloc_blocks = rounddown_pow_of_two(alloc_blocks);
 543         if (alloc_blocks > XFS_MAX_BMBT_EXTLEN)
 544                 alloc_blocks = XFS_MAX_BMBT_EXTLEN;
 545
 546         /*
 547          * If we are still trying to allocate more space than is
 548          * available, squash the prealloc hard. This can happen if we
 549          * have a large file on a small filesystem and the above
 550          * lowspace thresholds are smaller than XFS_BMBT_MAX_EXTLEN.
 551          */
 552         while (alloc_blocks && alloc_blocks >= freesp)
 553                 alloc_blocks >>= 4;
 554         if (alloc_blocks < mp->m_allocsize_blocks)
 555                 alloc_blocks = mp->m_allocsize_blocks;
 556         trace_xfs_iomap_prealloc_size(ip, alloc_blocks, shift,
 557                                       mp->m_allocsize_blocks);
 558         return alloc_blocks;
 559 }
 560
 561 int
 562 xfs_iomap_write_unwritten(
 563         xfs_inode_t     *ip,
 564         xfs_off_t       offset,
 565         xfs_off_t       count,
 566         bool            update_isize)
 567 {
 568         xfs_mount_t     *mp = ip->i_mount;
 569         xfs_fileoff_t   offset_fsb;
 570         xfs_filblks_t   count_fsb;
 571         xfs_filblks_t   numblks_fsb;
 572         int             nimaps;
 573         xfs_trans_t     *tp;
 574         xfs_bmbt_irec_t imap;
 575         struct inode    *inode = VFS_I(ip);
 576         xfs_fsize_t     i_size;
 577         uint            resblks;
 578         int             error;
 579
 580         trace_xfs_unwritten_convert(ip, offset, count);
 581
 582         offset_fsb = XFS_B_TO_FSBT(mp, offset);
 583         count_fsb = XFS_B_TO_FSB(mp, (xfs_ufsize_t)offset + count);
 584         count_fsb = (xfs_filblks_t)(count_fsb - offset_fsb);
 585
 586         /*
 587          * Reserve enough blocks in this transaction for two complete extent
 588          * btree splits.  We may be converting the middle part of an unwritten
 589          * extent and in this case we will insert two new extents in the btree
 590          * each of which could cause a full split.
 591          *
 592          * This reservation amount will be used in the first call to
 593          * xfs_bmbt_split() to select an AG with enough space to satisfy the
 594          * rest of the operation.
 595          */
 596         resblks = XFS_DIOSTRAT_SPACE_RES(mp, 0) << 1;
 597
 598         /* Attach dquots so that bmbt splits are accounted correctly. */
 599         error = xfs_qm_dqattach(ip);
 600         if (error)
 601                 return error;
 602
 603         do {
 604                 /*
 605                  * Set up a transaction to convert the range of extents
 606                  * from unwritten to real. Do allocations in a loop until
 607                  * we have covered the range passed in.
 608                  *
 609                  * Note that we can't risk to recursing back into the filesystem
 610                  * here as we might be asked to write out the same inode that we
 611                  * complete here and might deadlock on the iolock.
 612                  */
 613                 error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, resblks,
 614                                 0, true, &tp);
 615                 if (error)
 616                         return error;
 617
 618                 error = xfs_iext_count_extend(tp, ip, XFS_DATA_FORK,
 619                                 XFS_IEXT_WRITE_UNWRITTEN_CNT);
 620                 if (error)
 621                         goto error_on_bmapi_transaction;
 622
 623                 /*
 624                  * Modify the unwritten extent state of the buffer.
 625                  */
 626                 nimaps = 1;
 627                 error = xfs_bmapi_write(tp, ip, offset_fsb, count_fsb,
 628                                         XFS_BMAPI_CONVERT, resblks, &imap,
 629                                         &nimaps);
 630                 if (error)
 631                         goto error_on_bmapi_transaction;
 632
 633                 /*
 634                  * Log the updated inode size as we go.  We have to be careful
 635                  * to only log it up to the actual write offset if it is
 636                  * halfway into a block.
 637                  */
 638                 i_size = XFS_FSB_TO_B(mp, offset_fsb + count_fsb);
 639                 if (i_size > offset + count)
 640                         i_size = offset + count;
 641                 if (update_isize && i_size > i_size_read(inode))
 642                         i_size_write(inode, i_size);
 643                 i_size = xfs_new_eof(ip, i_size);
 644                 if (i_size) {
 645                         ip->i_disk_size = i_size;
 646                         xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
 647                 }
 648
 649                 error = xfs_trans_commit(tp);
 650                 xfs_iunlock(ip, XFS_ILOCK_EXCL);
 651                 if (error)
 652                         return error;
 653
 654                 if (unlikely(!xfs_valid_startblock(ip, imap.br_startblock))) {
 655                         xfs_bmap_mark_sick(ip, XFS_DATA_FORK);
 656                         return xfs_alert_fsblock_zero(ip, &imap);
 657                 }
 658
 659                 if ((numblks_fsb = imap.br_blockcount) == 0) {
 660                         /*
 661                          * The numblks_fsb value should always get
 662                          * smaller, otherwise the loop is stuck.
 663                          */
 664                         ASSERT(imap.br_blockcount);
 665                         break;
 666                 }
 667                 offset_fsb += numblks_fsb;
 668                 count_fsb -= numblks_fsb;
 669         } while (count_fsb > 0);
 670
 671         return 0;
 672
 673 error_on_bmapi_transaction:
 674         xfs_trans_cancel(tp);
 675         xfs_iunlock(ip, XFS_ILOCK_EXCL);
 676         return error;
 677 }
 678
 679 static inline bool
 680 imap_needs_alloc(
 681         struct inode            *inode,
 682         unsigned                flags,
 683         struct xfs_bmbt_irec    *imap,
 684         int                     nimaps)
 685 {
 686         /* don't allocate blocks when just zeroing */
 687         if (flags & IOMAP_ZERO)
 688                 return false;
 689         if (!nimaps ||
 690             imap->br_startblock == HOLESTARTBLOCK ||
 691             imap->br_startblock == DELAYSTARTBLOCK)
 692                 return true;
 693         /* we convert unwritten extents before copying the data for DAX */
 694         if ((flags & IOMAP_DAX) && imap->br_state == XFS_EXT_UNWRITTEN)
 695                 return true;
 696         return false;
 697 }
 698
 699 static inline bool
 700 imap_needs_cow(
 701         struct xfs_inode        *ip,
 702         unsigned int            flags,
 703         struct xfs_bmbt_irec    *imap,
 704         int                     nimaps)
 705 {
 706         if (!xfs_is_cow_inode(ip))
 707                 return false;
 708
 709         /* when zeroing we don't have to COW holes or unwritten extents */
 710         if (flags & IOMAP_ZERO) {
 711                 if (!nimaps ||
 712                     imap->br_startblock == HOLESTARTBLOCK ||
 713                     imap->br_state == XFS_EXT_UNWRITTEN)
 714                         return false;
 715         }
 716
 717         return true;
 718 }
 719
 720 static int
 721 xfs_ilock_for_iomap(
 722         struct xfs_inode        *ip,
 723         unsigned                flags,
 724         unsigned                *lockmode)
 725 {
 726         unsigned int            mode = *lockmode;
 727         bool                    is_write = flags & (IOMAP_WRITE | IOMAP_ZERO);
 728
 729         /*
 730          * COW writes may allocate delalloc space or convert unwritten COW
 731          * extents, so we need to make sure to take the lock exclusively here.
 732          */
 733         if (xfs_is_cow_inode(ip) && is_write)
 734                 mode = XFS_ILOCK_EXCL;
 735
 736         /*
 737          * Extents not yet cached requires exclusive access, don't block.  This
 738          * is an opencoded xfs_ilock_data_map_shared() call but with
 739          * non-blocking behaviour.
 740          */
 741         if (xfs_need_iread_extents(&ip->i_df)) {
 742                 if (flags & IOMAP_NOWAIT)
 743                         return -EAGAIN;
 744                 mode = XFS_ILOCK_EXCL;
 745         }
 746
 747 relock:
 748         if (flags & IOMAP_NOWAIT) {
 749                 if (!xfs_ilock_nowait(ip, mode))
 750                         return -EAGAIN;
 751         } else {
 752                 xfs_ilock(ip, mode);
 753         }
 754
 755         /*
 756          * The reflink iflag could have changed since the earlier unlocked
 757          * check, so if we got ILOCK_SHARED for a write and but we're now a
 758          * reflink inode we have to switch to ILOCK_EXCL and relock.
 759          */
 760         if (mode == XFS_ILOCK_SHARED && is_write && xfs_is_cow_inode(ip)) {
 761                 xfs_iunlock(ip, mode);
 762                 mode = XFS_ILOCK_EXCL;
 763                 goto relock;
 764         }
 765
 766         *lockmode = mode;
 767         return 0;
 768 }
 769
 770 /*
 771  * Check that the imap we are going to return to the caller spans the entire
 772  * range that the caller requested for the IO.
 773  */
 774 static bool
 775 imap_spans_range(
 776         struct xfs_bmbt_irec    *imap,
 777         xfs_fileoff_t           offset_fsb,
 778         xfs_fileoff_t           end_fsb)
 779 {
 780         if (imap->br_startoff > offset_fsb)
 781                 return false;
 782         if (imap->br_startoff + imap->br_blockcount < end_fsb)
 783                 return false;
 784         return true;
 785 }
 786
 787 static int
 788 xfs_direct_write_iomap_begin(
 789         struct inode            *inode,
 790         loff_t                  offset,
 791         loff_t                  length,
 792         unsigned                flags,
 793         struct iomap            *iomap,
 794         struct iomap            *srcmap)
 795 {
 796         struct xfs_inode        *ip = XFS_I(inode);
 797         struct xfs_mount        *mp = ip->i_mount;
 798         struct xfs_bmbt_irec    imap, cmap;
 799         xfs_fileoff_t           offset_fsb = XFS_B_TO_FSBT(mp, offset);
 800         xfs_fileoff_t           end_fsb = xfs_iomap_end_fsb(mp, offset, length);
 801         int                     nimaps = 1, error = 0;
 802         bool                    shared = false;
 803         u16                     iomap_flags = 0;
 804         unsigned int            lockmode = XFS_ILOCK_SHARED;
 805         u64                     seq;
 806
 807         ASSERT(flags & (IOMAP_WRITE | IOMAP_ZERO));
 808
 809         if (xfs_is_shutdown(mp))
 810                 return -EIO;
 811
 812         /*
 813          * Writes that span EOF might trigger an IO size update on completion,
 814          * so consider them to be dirty for the purposes of O_DSYNC even if
 815          * there is no other metadata changes pending or have been made here.
 816          */
 817         if (offset + length > i_size_read(inode))
 818                 iomap_flags |= IOMAP_F_DIRTY;
 819
 820         error = xfs_ilock_for_iomap(ip, flags, &lockmode);
 821         if (error)
 822                 return error;
 823
 824         error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb, &imap,
 825                                &nimaps, 0);
 826         if (error)
 827                 goto out_unlock;
 828
 829         if (imap_needs_cow(ip, flags, &imap, nimaps)) {
 830                 error = -EAGAIN;
 831                 if (flags & IOMAP_NOWAIT)
 832                         goto out_unlock;
 833
 834                 /* may drop and re-acquire the ilock */
 835                 error = xfs_reflink_allocate_cow(ip, &imap, &cmap, &shared,
 836                                 &lockmode,
 837                                 (flags & IOMAP_DIRECT) || IS_DAX(inode));
 838                 if (error)
 839                         goto out_unlock;
 840                 if (shared)
 841                         goto out_found_cow;
 842                 end_fsb = imap.br_startoff + imap.br_blockcount;
 843                 length = XFS_FSB_TO_B(mp, end_fsb) - offset;
 844         }
 845
 846         if (imap_needs_alloc(inode, flags, &imap, nimaps))
 847                 goto allocate_blocks;
 848
 849         /*
 850          * NOWAIT and OVERWRITE I/O needs to span the entire requested I/O with
 851          * a single map so that we avoid partial IO failures due to the rest of
 852          * the I/O range not covered by this map triggering an EAGAIN condition
 853          * when it is subsequently mapped and aborting the I/O.
 854          */
 855         if (flags & (IOMAP_NOWAIT | IOMAP_OVERWRITE_ONLY)) {
 856                 error = -EAGAIN;
 857                 if (!imap_spans_range(&imap, offset_fsb, end_fsb))
 858                         goto out_unlock;
 859         }
 860
 861         /*
 862          * For overwrite only I/O, we cannot convert unwritten extents without
 863          * requiring sub-block zeroing.  This can only be done under an
 864          * exclusive IOLOCK, hence return -EAGAIN if this is not a written
 865          * extent to tell the caller to try again.
 866          */
 867         if (flags & IOMAP_OVERWRITE_ONLY) {
 868                 error = -EAGAIN;
 869                 if (imap.br_state != XFS_EXT_NORM &&
 870                     ((offset | length) & mp->m_blockmask))
 871                         goto out_unlock;
 872         }
 873
 874         seq = xfs_iomap_inode_sequence(ip, iomap_flags);
 875         xfs_iunlock(ip, lockmode);
 876         trace_xfs_iomap_found(ip, offset, length, XFS_DATA_FORK, &imap);
 877         return xfs_bmbt_to_iomap(ip, iomap, &imap, flags, iomap_flags, seq);
 878
 879 allocate_blocks:
 880         error = -EAGAIN;
 881         if (flags & (IOMAP_NOWAIT | IOMAP_OVERWRITE_ONLY))
 882                 goto out_unlock;
 883
 884         /*
 885          * We cap the maximum length we map to a sane size  to keep the chunks
 886          * of work done where somewhat symmetric with the work writeback does.
 887          * This is a completely arbitrary number pulled out of thin air as a
 888          * best guess for initial testing.
 889          *
 890          * Note that the values needs to be less than 32-bits wide until the
 891          * lower level functions are updated.
 892          */
 893         length = min_t(loff_t, length, 1024 * PAGE_SIZE);
 894         end_fsb = xfs_iomap_end_fsb(mp, offset, length);
 895
 896         if (offset + length > XFS_ISIZE(ip))
 897                 end_fsb = xfs_iomap_eof_align_last_fsb(ip, end_fsb);
 898         else if (nimaps && imap.br_startblock == HOLESTARTBLOCK)
 899                 end_fsb = min(end_fsb, imap.br_startoff + imap.br_blockcount);
 900         xfs_iunlock(ip, lockmode);
 901
 902         error = xfs_iomap_write_direct(ip, offset_fsb, end_fsb - offset_fsb,
 903                         flags, &imap, &seq);
 904         if (error)
 905                 return error;
 906
 907         trace_xfs_iomap_alloc(ip, offset, length, XFS_DATA_FORK, &imap);
 908         return xfs_bmbt_to_iomap(ip, iomap, &imap, flags,
 909                                  iomap_flags | IOMAP_F_NEW, seq);
 910
 911 out_found_cow:
 912         length = XFS_FSB_TO_B(mp, cmap.br_startoff + cmap.br_blockcount);
 913         trace_xfs_iomap_found(ip, offset, length - offset, XFS_COW_FORK, &cmap);
 914         if (imap.br_startblock != HOLESTARTBLOCK) {
 915                 seq = xfs_iomap_inode_sequence(ip, 0);
 916                 error = xfs_bmbt_to_iomap(ip, srcmap, &imap, flags, 0, seq);
 917                 if (error)
 918                         goto out_unlock;
 919         }
 920         seq = xfs_iomap_inode_sequence(ip, IOMAP_F_SHARED);
 921         xfs_iunlock(ip, lockmode);
 922         return xfs_bmbt_to_iomap(ip, iomap, &cmap, flags, IOMAP_F_SHARED, seq);
 923
 924 out_unlock:
 925         if (lockmode)
 926                 xfs_iunlock(ip, lockmode);
 927         return error;
 928 }
 929
 930 const struct iomap_ops xfs_direct_write_iomap_ops = {
 931         .iomap_begin            = xfs_direct_write_iomap_begin,
 932 };
 933
 934 static int
 935 xfs_dax_write_iomap_end(
 936         struct inode            *inode,
 937         loff_t                  pos,
 938         loff_t                  length,
 939         ssize_t                 written,
 940         unsigned                flags,
 941         struct iomap            *iomap)
 942 {
 943         struct xfs_inode        *ip = XFS_I(inode);
 944
 945         if (!xfs_is_cow_inode(ip))
 946                 return 0;
 947
 948         if (!written) {
 949                 xfs_reflink_cancel_cow_range(ip, pos, length, true);
 950                 return 0;
 951         }
 952
 953         return xfs_reflink_end_cow(ip, pos, written);
 954 }
 955
 956 const struct iomap_ops xfs_dax_write_iomap_ops = {
 957         .iomap_begin    = xfs_direct_write_iomap_begin,
 958         .iomap_end      = xfs_dax_write_iomap_end,
 959 };
 960
 961 static int
 962 xfs_buffered_write_iomap_begin(
 963         struct inode            *inode,
 964         loff_t                  offset,
 965         loff_t                  count,
 966         unsigned                flags,
 967         struct iomap            *iomap,
 968         struct iomap            *srcmap)
 969 {
 970         struct xfs_inode        *ip = XFS_I(inode);
 971         struct xfs_mount        *mp = ip->i_mount;
 972         xfs_fileoff_t           offset_fsb = XFS_B_TO_FSBT(mp, offset);
 973         xfs_fileoff_t           end_fsb = xfs_iomap_end_fsb(mp, offset, count);
 974         struct xfs_bmbt_irec    imap, cmap;
 975         struct xfs_iext_cursor  icur, ccur;
 976         xfs_fsblock_t           prealloc_blocks = 0;
 977         bool                    eof = false, cow_eof = false, shared = false;
 978         int                     allocfork = XFS_DATA_FORK;
 979         int                     error = 0;
 980         unsigned int            lockmode = XFS_ILOCK_EXCL;
 981         u64                     seq;
 982
 983         if (xfs_is_shutdown(mp))
 984                 return -EIO;
 985
 986         /* we can't use delayed allocations when using extent size hints */
 987         if (xfs_get_extsz_hint(ip))
 988                 return xfs_direct_write_iomap_begin(inode, offset, count,
 989                                 flags, iomap, srcmap);
 990
 991         error = xfs_qm_dqattach(ip);
 992         if (error)
 993                 return error;
 994
 995         error = xfs_ilock_for_iomap(ip, flags, &lockmode);
 996         if (error)
 997                 return error;
 998
 999         if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(&ip->i_df)) ||
1000             XFS_TEST_ERROR(false, mp, XFS_ERRTAG_BMAPIFORMAT)) {
1001                 xfs_bmap_mark_sick(ip, XFS_DATA_FORK);
1002                 error = -EFSCORRUPTED;
1003                 goto out_unlock;
1004         }
1005
1006         XFS_STATS_INC(mp, xs_blk_mapw);
1007
1008         error = xfs_iread_extents(NULL, ip, XFS_DATA_FORK);
1009         if (error)
1010                 goto out_unlock;
1011
1012         /*
1013          * Search the data fork first to look up our source mapping.  We
1014          * always need the data fork map, as we have to return it to the
1015          * iomap code so that the higher level write code can read data in to
1016          * perform read-modify-write cycles for unaligned writes.
1017          */
1018         eof = !xfs_iext_lookup_extent(ip, &ip->i_df, offset_fsb, &icur, &imap);
1019         if (eof)
1020                 imap.br_startoff = end_fsb; /* fake hole until the end */
1021
1022         /* We never need to allocate blocks for zeroing or unsharing a hole. */
1023         if ((flags & (IOMAP_UNSHARE | IOMAP_ZERO)) &&
1024             imap.br_startoff > offset_fsb) {
1025                 xfs_hole_to_iomap(ip, iomap, offset_fsb, imap.br_startoff);
1026                 goto out_unlock;
1027         }
1028
1029         /*
1030          * For zeroing, trim a delalloc extent that extends beyond the EOF
1031          * block.  If it starts beyond the EOF block, convert it to an
1032          * unwritten extent.
1033          */
1034         if ((flags & IOMAP_ZERO) && imap.br_startoff <= offset_fsb &&
1035             isnullstartblock(imap.br_startblock)) {
1036                 xfs_fileoff_t eof_fsb = XFS_B_TO_FSB(mp, XFS_ISIZE(ip));
1037
1038                 if (offset_fsb >= eof_fsb)
1039                         goto convert_delay;
1040                 if (end_fsb > eof_fsb) {
1041                         end_fsb = eof_fsb;
1042                         xfs_trim_extent(&imap, offset_fsb,
1043                                         end_fsb - offset_fsb);
1044                 }
1045         }
1046
1047         /*
1048          * Search the COW fork extent list even if we did not find a data fork
1049          * extent.  This serves two purposes: first this implements the
1050          * speculative preallocation using cowextsize, so that we also unshare
1051          * block adjacent to shared blocks instead of just the shared blocks
1052          * themselves.  Second the lookup in the extent list is generally faster
1053          * than going out to the shared extent tree.
1054          */
1055         if (xfs_is_cow_inode(ip)) {
1056                 if (!ip->i_cowfp) {
1057                         ASSERT(!xfs_is_reflink_inode(ip));
1058                         xfs_ifork_init_cow(ip);
1059                 }
1060                 cow_eof = !xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb,
1061                                 &ccur, &cmap);
1062                 if (!cow_eof && cmap.br_startoff <= offset_fsb) {
1063                         trace_xfs_reflink_cow_found(ip, &cmap);
1064                         goto found_cow;
1065                 }
1066         }
1067
1068         if (imap.br_startoff <= offset_fsb) {
1069                 /*
1070                  * For reflink files we may need a delalloc reservation when
1071                  * overwriting shared extents.   This includes zeroing of
1072                  * existing extents that contain data.
1073                  */
1074                 if (!xfs_is_cow_inode(ip) ||
1075                     ((flags & IOMAP_ZERO) && imap.br_state != XFS_EXT_NORM)) {
1076                         trace_xfs_iomap_found(ip, offset, count, XFS_DATA_FORK,
1077                                         &imap);
1078                         goto found_imap;
1079                 }
1080
1081                 xfs_trim_extent(&imap, offset_fsb, end_fsb - offset_fsb);
1082
1083                 /* Trim the mapping to the nearest shared extent boundary. */
1084                 error = xfs_bmap_trim_cow(ip, &imap, &shared);
1085                 if (error)
1086                         goto out_unlock;
1087
1088                 /* Not shared?  Just report the (potentially capped) extent. */
1089                 if (!shared) {
1090                         trace_xfs_iomap_found(ip, offset, count, XFS_DATA_FORK,
1091                                         &imap);
1092                         goto found_imap;
1093                 }
1094
1095                 /*
1096                  * Fork all the shared blocks from our write offset until the
1097                  * end of the extent.
1098                  */
1099                 allocfork = XFS_COW_FORK;
1100                 end_fsb = imap.br_startoff + imap.br_blockcount;
1101         } else {
1102                 /*
1103                  * We cap the maximum length we map here to MAX_WRITEBACK_PAGES
1104                  * pages to keep the chunks of work done where somewhat
1105                  * symmetric with the work writeback does.  This is a completely
1106                  * arbitrary number pulled out of thin air.
1107                  *
1108                  * Note that the values needs to be less than 32-bits wide until
1109                  * the lower level functions are updated.
1110                  */
1111                 count = min_t(loff_t, count, 1024 * PAGE_SIZE);
1112                 end_fsb = xfs_iomap_end_fsb(mp, offset, count);
1113
1114                 if (xfs_is_always_cow_inode(ip))
1115                         allocfork = XFS_COW_FORK;
1116         }
1117
1118         if (eof && offset + count > XFS_ISIZE(ip)) {
1119                 /*
1120                  * Determine the initial size of the preallocation.
1121                  * We clean up any extra preallocation when the file is closed.
1122                  */
1123                 if (xfs_has_allocsize(mp))
1124                         prealloc_blocks = mp->m_allocsize_blocks;
1125                 else if (allocfork == XFS_DATA_FORK)
1126                         prealloc_blocks = xfs_iomap_prealloc_size(ip, allocfork,
1127                                                 offset, count, &icur);
1128                 else
1129                         prealloc_blocks = xfs_iomap_prealloc_size(ip, allocfork,
1130                                                 offset, count, &ccur);
1131                 if (prealloc_blocks) {
1132                         xfs_extlen_t    align;
1133                         xfs_off_t       end_offset;
1134                         xfs_fileoff_t   p_end_fsb;
1135
1136                         end_offset = XFS_ALLOC_ALIGN(mp, offset + count - 1);
1137                         p_end_fsb = XFS_B_TO_FSBT(mp, end_offset) +
1138                                         prealloc_blocks;
1139
1140                         align = xfs_eof_alignment(ip);
1141                         if (align)
1142                                 p_end_fsb = roundup_64(p_end_fsb, align);
1143
1144                         p_end_fsb = min(p_end_fsb,
1145                                 XFS_B_TO_FSB(mp, mp->m_super->s_maxbytes));
1146                         ASSERT(p_end_fsb > offset_fsb);
1147                         prealloc_blocks = p_end_fsb - end_fsb;
1148                 }
1149         }
1150
1151 retry:
1152         error = xfs_bmapi_reserve_delalloc(ip, allocfork, offset_fsb,
1153                         end_fsb - offset_fsb, prealloc_blocks,
1154                         allocfork == XFS_DATA_FORK ? &imap : &cmap,
1155                         allocfork == XFS_DATA_FORK ? &icur : &ccur,
1156                         allocfork == XFS_DATA_FORK ? eof : cow_eof);
1157         switch (error) {
1158         case 0:
1159                 break;
1160         case -ENOSPC:
1161         case -EDQUOT:
1162                 /* retry without any preallocation */
1163                 trace_xfs_delalloc_enospc(ip, offset, count);
1164                 if (prealloc_blocks) {
1165                         prealloc_blocks = 0;
1166                         goto retry;
1167                 }
1168                 fallthrough;
1169         default:
1170                 goto out_unlock;
1171         }
1172
1173         if (allocfork == XFS_COW_FORK) {
1174                 trace_xfs_iomap_alloc(ip, offset, count, allocfork, &cmap);
1175                 goto found_cow;
1176         }
1177
1178         /*
1179          * Flag newly allocated delalloc blocks with IOMAP_F_NEW so we punch
1180          * them out if the write happens to fail.
1181          */
1182         seq = xfs_iomap_inode_sequence(ip, IOMAP_F_NEW);
1183         xfs_iunlock(ip, lockmode);
1184         trace_xfs_iomap_alloc(ip, offset, count, allocfork, &imap);
1185         return xfs_bmbt_to_iomap(ip, iomap, &imap, flags, IOMAP_F_NEW, seq);
1186
1187 found_imap:
1188         seq = xfs_iomap_inode_sequence(ip, 0);
1189         xfs_iunlock(ip, lockmode);
1190         return xfs_bmbt_to_iomap(ip, iomap, &imap, flags, 0, seq);
1191
1192 convert_delay:
1193         xfs_iunlock(ip, lockmode);
1194         truncate_pagecache(inode, offset);
1195         error = xfs_bmapi_convert_delalloc(ip, XFS_DATA_FORK, offset,
1196                                            iomap, NULL);
1197         if (error)
1198                 return error;
1199
1200         trace_xfs_iomap_alloc(ip, offset, count, XFS_DATA_FORK, &imap);
1201         return 0;
1202
1203 found_cow:
1204         seq = xfs_iomap_inode_sequence(ip, 0);
1205         if (imap.br_startoff <= offset_fsb) {
1206                 error = xfs_bmbt_to_iomap(ip, srcmap, &imap, flags, 0, seq);
1207                 if (error)
1208                         goto out_unlock;
1209                 seq = xfs_iomap_inode_sequence(ip, IOMAP_F_SHARED);
1210                 xfs_iunlock(ip, lockmode);
1211                 return xfs_bmbt_to_iomap(ip, iomap, &cmap, flags,
1212                                          IOMAP_F_SHARED, seq);
1213         }
1214
1215         xfs_trim_extent(&cmap, offset_fsb, imap.br_startoff - offset_fsb);
1216         xfs_iunlock(ip, lockmode);
1217         return xfs_bmbt_to_iomap(ip, iomap, &cmap, flags, 0, seq);
1218
1219 out_unlock:
1220         xfs_iunlock(ip, lockmode);
1221         return error;
1222 }
1223
1224 static int
1225 xfs_buffered_write_delalloc_punch(
1226         struct inode            *inode,
1227         loff_t                  offset,
1228         loff_t                  length)
1229 {
1230         xfs_bmap_punch_delalloc_range(XFS_I(inode), offset, offset + length);
1231         return 0;
1232 }
1233
1234 static int
1235 xfs_buffered_write_iomap_end(
1236         struct inode            *inode,
1237         loff_t                  offset,
1238         loff_t                  length,
1239         ssize_t                 written,
1240         unsigned                flags,
1241         struct iomap            *iomap)
1242 {
1243
1244         struct xfs_mount        *mp = XFS_M(inode->i_sb);
1245         int                     error;
1246
1247         error = iomap_file_buffered_write_punch_delalloc(inode, iomap, offset,
1248                         length, written, &xfs_buffered_write_delalloc_punch);
1249         if (error && !xfs_is_shutdown(mp)) {
1250                 xfs_alert(mp, "%s: unable to clean up ino 0x%llx",
1251                         __func__, XFS_I(inode)->i_ino);
1252                 return error;
1253         }
1254         return 0;
1255 }
1256
1257 const struct iomap_ops xfs_buffered_write_iomap_ops = {
1258         .iomap_begin            = xfs_buffered_write_iomap_begin,
1259         .iomap_end              = xfs_buffered_write_iomap_end,
1260 };
1261
1262 /*
1263  * iomap_page_mkwrite() will never fail in a way that requires delalloc extents
1264  * that it allocated to be revoked. Hence we do not need an .iomap_end method
1265  * for this operation.
1266  */
1267 const struct iomap_ops xfs_page_mkwrite_iomap_ops = {
1268         .iomap_begin            = xfs_buffered_write_iomap_begin,
1269 };
1270
1271 static int
1272 xfs_read_iomap_begin(
1273         struct inode            *inode,
1274         loff_t                  offset,
1275         loff_t                  length,
1276         unsigned                flags,
1277         struct iomap            *iomap,
1278         struct iomap            *srcmap)
1279 {
1280         struct xfs_inode        *ip = XFS_I(inode);
1281         struct xfs_mount        *mp = ip->i_mount;
1282         struct xfs_bmbt_irec    imap;
1283         xfs_fileoff_t           offset_fsb = XFS_B_TO_FSBT(mp, offset);
1284         xfs_fileoff_t           end_fsb = xfs_iomap_end_fsb(mp, offset, length);
1285         int                     nimaps = 1, error = 0;
1286         bool                    shared = false;
1287         unsigned int            lockmode = XFS_ILOCK_SHARED;
1288         u64                     seq;
1289
1290         ASSERT(!(flags & (IOMAP_WRITE | IOMAP_ZERO)));
1291
1292         if (xfs_is_shutdown(mp))
1293                 return -EIO;
1294
1295         error = xfs_ilock_for_iomap(ip, flags, &lockmode);
1296         if (error)
1297                 return error;
1298         error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb, &imap,
1299                                &nimaps, 0);
1300         if (!error && ((flags & IOMAP_REPORT) || IS_DAX(inode)))
1301                 error = xfs_reflink_trim_around_shared(ip, &imap, &shared);
1302         seq = xfs_iomap_inode_sequence(ip, shared ? IOMAP_F_SHARED : 0);
1303         xfs_iunlock(ip, lockmode);
1304
1305         if (error)
1306                 return error;
1307         trace_xfs_iomap_found(ip, offset, length, XFS_DATA_FORK, &imap);
1308         return xfs_bmbt_to_iomap(ip, iomap, &imap, flags,
1309                                  shared ? IOMAP_F_SHARED : 0, seq);
1310 }
1311
1312 const struct iomap_ops xfs_read_iomap_ops = {
1313         .iomap_begin            = xfs_read_iomap_begin,
1314 };
1315
1316 static int
1317 xfs_seek_iomap_begin(
1318         struct inode            *inode,
1319         loff_t                  offset,
1320         loff_t                  length,
1321         unsigned                flags,
1322         struct iomap            *iomap,
1323         struct iomap            *srcmap)
1324 {
1325         struct xfs_inode        *ip = XFS_I(inode);
1326         struct xfs_mount        *mp = ip->i_mount;
1327         xfs_fileoff_t           offset_fsb = XFS_B_TO_FSBT(mp, offset);
1328         xfs_fileoff_t           end_fsb = XFS_B_TO_FSB(mp, offset + length);
1329         xfs_fileoff_t           cow_fsb = NULLFILEOFF, data_fsb = NULLFILEOFF;
1330         struct xfs_iext_cursor  icur;
1331         struct xfs_bmbt_irec    imap, cmap;
1332         int                     error = 0;
1333         unsigned                lockmode;
1334         u64                     seq;
1335
1336         if (xfs_is_shutdown(mp))
1337                 return -EIO;
1338
1339         lockmode = xfs_ilock_data_map_shared(ip);
1340         error = xfs_iread_extents(NULL, ip, XFS_DATA_FORK);
1341         if (error)
1342                 goto out_unlock;
1343
1344         if (xfs_iext_lookup_extent(ip, &ip->i_df, offset_fsb, &icur, &imap)) {
1345                 /*
1346                  * If we found a data extent we are done.
1347                  */
1348                 if (imap.br_startoff <= offset_fsb)
1349                         goto done;
1350                 data_fsb = imap.br_startoff;
1351         } else {
1352                 /*
1353                  * Fake a hole until the end of the file.
1354                  */
1355                 data_fsb = xfs_iomap_end_fsb(mp, offset, length);
1356         }
1357
1358         /*
1359          * If a COW fork extent covers the hole, report it - capped to the next
1360          * data fork extent:
1361          */
1362         if (xfs_inode_has_cow_data(ip) &&
1363             xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &cmap))
1364                 cow_fsb = cmap.br_startoff;
1365         if (cow_fsb != NULLFILEOFF && cow_fsb <= offset_fsb) {
1366                 if (data_fsb < cow_fsb + cmap.br_blockcount)
1367                         end_fsb = min(end_fsb, data_fsb);
1368                 xfs_trim_extent(&cmap, offset_fsb, end_fsb - offset_fsb);
1369                 seq = xfs_iomap_inode_sequence(ip, IOMAP_F_SHARED);
1370                 error = xfs_bmbt_to_iomap(ip, iomap, &cmap, flags,
1371                                 IOMAP_F_SHARED, seq);
1372                 /*
1373                  * This is a COW extent, so we must probe the page cache
1374                  * because there could be dirty page cache being backed
1375                  * by this extent.
1376                  */
1377                 iomap->type = IOMAP_UNWRITTEN;
1378                 goto out_unlock;
1379         }
1380
1381         /*
1382          * Else report a hole, capped to the next found data or COW extent.
1383          */
1384         if (cow_fsb != NULLFILEOFF && cow_fsb < data_fsb)
1385                 imap.br_blockcount = cow_fsb - offset_fsb;
1386         else
1387                 imap.br_blockcount = data_fsb - offset_fsb;
1388         imap.br_startoff = offset_fsb;
1389         imap.br_startblock = HOLESTARTBLOCK;
1390         imap.br_state = XFS_EXT_NORM;
1391 done:
1392         seq = xfs_iomap_inode_sequence(ip, 0);
1393         xfs_trim_extent(&imap, offset_fsb, end_fsb - offset_fsb);
1394         error = xfs_bmbt_to_iomap(ip, iomap, &imap, flags, 0, seq);
1395 out_unlock:
1396         xfs_iunlock(ip, lockmode);
1397         return error;
1398 }
1399
1400 const struct iomap_ops xfs_seek_iomap_ops = {
1401         .iomap_begin            = xfs_seek_iomap_begin,
1402 };
1403
1404 static int
1405 xfs_xattr_iomap_begin(
1406         struct inode            *inode,
1407         loff_t                  offset,
1408         loff_t                  length,
1409         unsigned                flags,
1410         struct iomap            *iomap,
1411         struct iomap            *srcmap)
1412 {
1413         struct xfs_inode        *ip = XFS_I(inode);
1414         struct xfs_mount        *mp = ip->i_mount;
1415         xfs_fileoff_t           offset_fsb = XFS_B_TO_FSBT(mp, offset);
1416         xfs_fileoff_t           end_fsb = XFS_B_TO_FSB(mp, offset + length);
1417         struct xfs_bmbt_irec    imap;
1418         int                     nimaps = 1, error = 0;
1419         unsigned                lockmode;
1420         int                     seq;
1421
1422         if (xfs_is_shutdown(mp))
1423                 return -EIO;
1424
1425         lockmode = xfs_ilock_attr_map_shared(ip);
1426
1427         /* if there are no attribute fork or extents, return ENOENT */
1428         if (!xfs_inode_has_attr_fork(ip) || !ip->i_af.if_nextents) {
1429                 error = -ENOENT;
1430                 goto out_unlock;
1431         }
1432
1433         ASSERT(ip->i_af.if_format != XFS_DINODE_FMT_LOCAL);
1434         error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb, &imap,
1435                                &nimaps, XFS_BMAPI_ATTRFORK);
1436 out_unlock:
1437
1438         seq = xfs_iomap_inode_sequence(ip, IOMAP_F_XATTR);
1439         xfs_iunlock(ip, lockmode);
1440
1441         if (error)
1442                 return error;
1443         ASSERT(nimaps);
1444         return xfs_bmbt_to_iomap(ip, iomap, &imap, flags, IOMAP_F_XATTR, seq);
1445 }
1446
1447 const struct iomap_ops xfs_xattr_iomap_ops = {
1448         .iomap_begin            = xfs_xattr_iomap_begin,
1449 };
1450
1451 int
1452 xfs_zero_range(
1453         struct xfs_inode        *ip,
1454         loff_t                  pos,
1455         loff_t                  len,
1456         bool                    *did_zero)
1457 {
1458         struct inode            *inode = VFS_I(ip);
1459
1460         if (IS_DAX(inode))
1461                 return dax_zero_range(inode, pos, len, did_zero,
1462                                       &xfs_dax_write_iomap_ops);
1463         return iomap_zero_range(inode, pos, len, did_zero,
1464                                 &xfs_buffered_write_iomap_ops);
1465 }
1466
1467 int
1468 xfs_truncate_page(
1469         struct xfs_inode        *ip,
1470         loff_t                  pos,
1471         bool                    *did_zero)
1472 {
1473         struct inode            *inode = VFS_I(ip);
1474
1475         if (IS_DAX(inode))
1476                 return dax_truncate_page(inode, pos, did_zero,
1477                                         &xfs_dax_write_iomap_ops);
1478         return iomap_truncate_page(inode, pos, did_zero,
1479                                    &xfs_buffered_write_iomap_ops);
1480 }