dm: allow dm_accept_partial_bio() for dm_io without duplicate bios

[linux.git] / drivers / md / dm.c

diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index ad2e0bbeb559a584434a1b90da504ab0b7708c71..fc1f9583a2710639f4fa080a2830814fbdd60f41 100644 (file)
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -892,13 +892,19 @@ static void dm_io_complete(struct dm_io *io)
        if (unlikely(wq_has_sleeper(&md->wait)))
                wake_up(&md->wait);
 
-       if (io_error == BLK_STS_DM_REQUEUE) {
-               /*
-                * Upper layer won't help us poll split bio, io->orig_bio
-                * may only reflect a subset of the pre-split original,
-                * so clear REQ_POLLED in case of requeue
-                */
-               bio->bi_opf &= ~REQ_POLLED;
+       if (io_error == BLK_STS_DM_REQUEUE || io_error == BLK_STS_AGAIN) {
+               if (bio->bi_opf & REQ_POLLED) {
+                       /*
+                        * Upper layer won't help us poll split bio (io->orig_bio
+                        * may only reflect a subset of the pre-split original)
+                        * so clear REQ_POLLED in case of requeue.
+                        */
+                       bio->bi_opf &= ~REQ_POLLED;
+                       if (io_error == BLK_STS_AGAIN) {
+                               /* io_uring doesn't handle BLK_STS_AGAIN (yet) */
+                               queue_io(md, bio);
+                       }
+               }
                return;
        }
 
@@ -1317,8 +1323,7 @@ static void __map_bio(struct bio *clone)
 }
 
 static void alloc_multiple_bios(struct bio_list *blist, struct clone_info *ci,
-                               struct dm_target *ti, unsigned num_bios,
-                               unsigned *len)
+                               struct dm_target *ti, unsigned num_bios)
 {
        struct bio *bio;
        int try;
@@ -1329,7 +1334,7 @@ static void alloc_multiple_bios(struct bio_list *blist, struct clone_info *ci,
                if (try)
                        mutex_lock(&ci->io->md->table_devices_lock);
                for (bio_nr = 0; bio_nr < num_bios; bio_nr++) {
-                       bio = alloc_tio(ci, ti, bio_nr, len,
+                       bio = alloc_tio(ci, ti, bio_nr, NULL,
                                        try ? GFP_NOIO : GFP_NOWAIT);
                        if (!bio)
                                break;
@@ -1357,11 +1362,11 @@ static void __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti,
                break;
        case 1:
                clone = alloc_tio(ci, ti, 0, len, GFP_NOIO);
-               dm_tio_set_flag(clone_to_tio(clone), DM_TIO_IS_DUPLICATE_BIO);
                __map_bio(clone);
                break;
        default:
-               alloc_multiple_bios(&blist, ci, ti, num_bios, len);
+               /* dm_accept_partial_bio() is not supported with shared tio->len_ptr */
+               alloc_multiple_bios(&blist, ci, ti, num_bios);
                while ((clone = bio_list_pop(&blist))) {
                        dm_tio_set_flag(clone_to_tio(clone), DM_TIO_IS_DUPLICATE_BIO);
                        __map_bio(clone);
@@ -1401,14 +1406,10 @@ static void __send_changing_extent_only(struct clone_info *ci, struct dm_target
        len = min_t(sector_t, ci->sector_count,
                    max_io_len_target_boundary(ti, dm_target_offset(ti, ci->sector)));
 
-       /*
-        * dm_accept_partial_bio cannot be used with duplicate bios,
-        * so update clone_info cursor before __send_duplicate_bios().
-        */
+       __send_duplicate_bios(ci, ti, num_bios, &len);
+
        ci->sector += len;
        ci->sector_count -= len;
-
-       __send_duplicate_bios(ci, ti, num_bios, &len);
 }
 
 static bool is_abnormal_io(struct bio *bio)