dm: conditionally enable BIOSET_PERCPU_CACHE for dm_io bioset

A bioset's per-cpu alloc cache may have broader utility in the future
but for now constrain it to being tightly coupled to QUEUE_FLAG_POLL.

Also change dm_io_complete() to use bio_clear_polled() so that it
properly clears all associated bio state on requeue.

This commit improves DM's hipri bio polling (REQ_POLLED) perf by
7 - 20% depending on the system.

Signed-off-by: Mike Snitzer <[email protected]>

diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c
index e7d42f6335a2af2c869b88e1a52cd48d04c886c8..73ed15d8cfc65d92a522651e6fc7ab6a8145520a 100644 (file)
--- a/drivers/md/dm-table.c
+++ b/drivers/md/dm-table.c
@@ -1002,6 +1002,8 @@ bool dm_table_request_based(struct dm_table *t)
        return __table_type_request_based(dm_table_get_type(t));
 }
 
+static int dm_table_supports_poll(struct dm_table *t);
+
 static int dm_table_alloc_md_mempools(struct dm_table *t, struct mapped_device *md)
 {
        enum dm_queue_mode type = dm_table_get_type(t);
@@ -1009,21 +1011,24 @@ static int dm_table_alloc_md_mempools(struct dm_table *t, struct mapped_device *
        unsigned min_pool_size = 0;
        struct dm_target *ti;
        unsigned i;
+       bool poll_supported = false;
 
        if (unlikely(type == DM_TYPE_NONE)) {
                DMWARN("no table type is set, can't allocate mempools");
                return -EINVAL;
        }
 
-       if (__table_type_bio_based(type))
+       if (__table_type_bio_based(type)) {
                for (i = 0; i < t->num_targets; i++) {
                        ti = t->targets + i;
                        per_io_data_size = max(per_io_data_size, ti->per_io_data_size);
                        min_pool_size = max(min_pool_size, ti->num_flush_bios);
                }
+               poll_supported = !!dm_table_supports_poll(t);
+       }
 
-       t->mempools = dm_alloc_md_mempools(md, type, t->integrity_supported,
-                                          per_io_data_size, min_pool_size);
+       t->mempools = dm_alloc_md_mempools(md, type, per_io_data_size, min_pool_size,
+                                          t->integrity_supported, poll_supported);
        if (!t->mempools)
                return -ENOMEM;
 

diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index 39081338ca6162e4e6c337ed827f90ffde389ba2..cdf35c6f8b0b7942e86f090d1c318fd1078f795d 100644 (file)
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -899,7 +899,7 @@ static void dm_io_complete(struct dm_io *io)
                         * may only reflect a subset of the pre-split original)
                         * so clear REQ_POLLED in case of requeue.
                         */
-                       bio->bi_opf &= ~REQ_POLLED;
+                       bio_clear_polled(bio);
                        if (io_error == BLK_STS_AGAIN) {
                                /* io_uring doesn't handle BLK_STS_AGAIN (yet) */
                                queue_io(md, bio);
@@ -2901,8 +2901,8 @@ int dm_noflush_suspending(struct dm_target *ti)
 EXPORT_SYMBOL_GPL(dm_noflush_suspending);
 
 struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, enum dm_queue_mode type,
-                                           unsigned integrity, unsigned per_io_data_size,
-                                           unsigned min_pool_size)
+                                           unsigned per_io_data_size, unsigned min_pool_size,
+                                           bool integrity, bool poll)
 {
        struct dm_md_mempools *pools = kzalloc_node(sizeof(*pools), GFP_KERNEL, md->numa_node_id);
        unsigned int pool_size = 0;
@@ -2918,7 +2918,7 @@ struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, enum dm_qu
                pool_size = max(dm_get_reserved_bio_based_ios(), min_pool_size);
                front_pad = roundup(per_io_data_size, __alignof__(struct dm_target_io)) + DM_TARGET_IO_BIO_OFFSET;
                io_front_pad = roundup(per_io_data_size,  __alignof__(struct dm_io)) + DM_IO_BIO_OFFSET;
-               ret = bioset_init(&pools->io_bs, pool_size, io_front_pad, 0);
+               ret = bioset_init(&pools->io_bs, pool_size, io_front_pad, poll ? BIOSET_PERCPU_CACHE : 0);
                if (ret)
                        goto out;
                if (integrity && bioset_integrity_create(&pools->io_bs, pool_size))

diff --git a/drivers/md/dm.h b/drivers/md/dm.h
index 9013dc1a7b0028cc98c2e99742ec5be487a753ba..3f89664fea010121f793bc404c5d8870c68594b6 100644 (file)
--- a/drivers/md/dm.h
+++ b/drivers/md/dm.h
@@ -221,8 +221,8 @@ void dm_kcopyd_exit(void);
  * Mempool operations
  */
 struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, enum dm_queue_mode type,
-                                           unsigned integrity, unsigned per_bio_data_size,
-                                           unsigned min_pool_size);
+                                           unsigned per_io_data_size, unsigned min_pool_size,
+                                           bool integrity, bool poll);
 void dm_free_md_mempools(struct dm_md_mempools *pools);
 
 /*