From 754c4bf3e231b28c9fe22612b78cf5ea7d7203c5 Mon Sep 17 00:00:00 2001 From: Ameer Hamza Date: Mon, 8 Jul 2024 15:38:09 +0500 Subject: [PATCH] Add support for two bdevs in blkdev_copy_offload Signed-off-by: Ameer Hamza --- block/blk-lib.c | 51 +++++++++++++++++++++++++----------------- block/fops.c | 6 ++--- include/linux/blkdev.h | 25 ++++++++++++++++++--- 3 files changed, 56 insertions(+), 26 deletions(-) diff --git a/block/blk-lib.c b/block/blk-lib.c index 50b60990ba3f..82f9c58b111a 100644 --- a/block/blk-lib.c +++ b/block/blk-lib.c @@ -10,22 +10,6 @@ #include "blk.h" -/* Keeps track of all outstanding copy IO */ -struct blkdev_copy_io { - atomic_t refcount; - ssize_t copied; - int status; - struct task_struct *waiter; - void (*endio)(void *private, int status, ssize_t copied); - void *private; -}; - -/* Keeps track of single outstanding copy offload IO */ -struct blkdev_copy_offload_io { - struct blkdev_copy_io *cio; - loff_t offset; -}; - static sector_t bio_discard_limit(struct block_device *bdev, sector_t sector) { unsigned int discard_granularity = bdev_discard_granularity(bdev); @@ -180,6 +164,9 @@ static void blkdev_copy_offload_src_endio(struct bio *bio) cio->status = blk_status_to_errno(bio->bi_status); } bio_put(bio); + if (offload_io->dst_bio) + bio_put(offload_io->dst_bio); + kfree(offload_io); if (atomic_dec_and_test(&cio->refcount)) @@ -216,7 +203,7 @@ static void blkdev_copy_offload_src_endio(struct bio *bio) ssize_t blkdev_copy_offload(struct block_device *bdev, loff_t pos_in, loff_t pos_out, size_t len, void (*endio)(void *, int, ssize_t), - void *private, gfp_t gfp) + void *private, gfp_t gfp, struct block_device *bdev_out) { struct blkdev_copy_io *cio; struct blkdev_copy_offload_io *offload_io; @@ -225,11 +212,25 @@ ssize_t blkdev_copy_offload(struct block_device *bdev, loff_t pos_in, size_t max_copy_bytes = bdev_max_copy_sectors(bdev) << SECTOR_SHIFT; ssize_t ret; struct blk_plug plug; + int is_mq = 0; if (!max_copy_bytes) return -EOPNOTSUPP; + if (queue_is_mq(bdev->bd_queue)) { + if (bdev->bd_queue->mq_ops != bdev_out->bd_queue->mq_ops) + return -EOPNOTSUPP; + is_mq = 1; + } else if (!bdev->bd_disk->fops->submit_bio || + bdev->bd_disk->fops->submit_bio != bdev_out->bd_disk->fops->submit_bio) { + return -EOPNOTSUPP; + } + ret = blkdev_copy_sanity_check(bdev, pos_in, bdev, pos_out, len); + + if (ret == 0) + ret = blkdev_copy_sanity_check(bdev_out, pos_in, bdev, pos_out, len); + if (ret) return ret; @@ -258,6 +259,8 @@ ssize_t blkdev_copy_offload(struct block_device *bdev, loff_t pos_in, * successful copy length */ offload_io->offset = len - rem; + if (bdev_out) + offload_io->driver_private = bdev_out->bd_queue->queuedata; dst_bio = bio_alloc(bdev, 0, REQ_OP_COPY_DST, gfp); if (!dst_bio) @@ -265,18 +268,24 @@ ssize_t blkdev_copy_offload(struct block_device *bdev, loff_t pos_in, dst_bio->bi_iter.bi_size = chunk; dst_bio->bi_iter.bi_sector = pos_out >> SECTOR_SHIFT; - blk_start_plug(&plug); - src_bio = blk_next_bio(dst_bio, bdev, 0, REQ_OP_COPY_SRC, gfp); + if (is_mq) { + blk_start_plug(&plug); + src_bio = blk_next_bio(dst_bio, bdev, 0, REQ_OP_COPY_SRC, gfp); + } else { + src_bio = bio_alloc(bdev, 0, REQ_OP_COPY_SRC, gfp); + } if (!src_bio) goto err_free_dst_bio; src_bio->bi_iter.bi_size = chunk; src_bio->bi_iter.bi_sector = pos_in >> SECTOR_SHIFT; src_bio->bi_end_io = blkdev_copy_offload_src_endio; src_bio->bi_private = offload_io; + offload_io->dst_bio = (is_mq) ? NULL : dst_bio; atomic_inc(&cio->refcount); submit_bio(src_bio); - blk_finish_plug(&plug); + if (is_mq) + blk_finish_plug(&plug); pos_in += chunk; pos_out += chunk; } @@ -289,6 +298,8 @@ ssize_t blkdev_copy_offload(struct block_device *bdev, loff_t pos_in, return blkdev_copy_wait_for_completion_io(cio); err_free_dst_bio: + if (is_mq) + blk_finish_plug(&plug); bio_put(dst_bio); err_free_offload_io: kfree(offload_io); diff --git a/block/fops.c b/block/fops.c index e3e1a525aea2..145f67dd3e99 100644 --- a/block/fops.c +++ b/block/fops.c @@ -779,12 +779,12 @@ static ssize_t blkdev_copy_file_range(struct file *file_in, loff_t pos_in, struct block_device *out_bdev = I_BDEV(bdev_file_inode(file_out)); ssize_t copied = 0; - if ((in_bdev == out_bdev) && bdev_max_copy_sectors(in_bdev) && + if (bdev_max_copy_sectors(in_bdev) && (file_in->f_iocb_flags & IOCB_DIRECT) && (file_out->f_iocb_flags & IOCB_DIRECT)) { copied = blkdev_copy_offload(in_bdev, pos_in, pos_out, len, - NULL, NULL, GFP_KERNEL); - if (copied < 0) + NULL, NULL, GFP_KERNEL, out_bdev); + if (copied != -EOPNOTSUPP && copied < 0) copied = 0; } else { copied = splice_copy_file_range(file_in, pos_in + copied, diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 168d0a612321..cece26dc91f7 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -408,6 +408,24 @@ struct queue_limits { struct blk_integrity integrity; }; +/* Keeps track of all outstanding copy IO */ +struct blkdev_copy_io { + atomic_t refcount; + ssize_t copied; + int status; + struct task_struct *waiter; + void (*endio)(void *private, int status, ssize_t copied); + void *private; +}; + +/* Keeps track of single outstanding copy offload IO */ +struct blkdev_copy_offload_io { + struct bio *dst_bio; + void *driver_private; + struct blkdev_copy_io *cio; + loff_t offset; +}; + typedef int (*report_zones_cb)(struct blk_zone *zone, unsigned int idx, void *data); @@ -1101,7 +1119,8 @@ int blkdev_issue_secure_erase(struct block_device *bdev, sector_t sector, ssize_t blkdev_copy_offload(struct block_device *bdev, loff_t pos_in, loff_t pos_out, size_t len, void (*endio)(void *, int, ssize_t), - void *private, gfp_t gfp_mask); + void *private, gfp_t gfp_mask, + struct block_device *bdev_out); #define BLKDEV_ZERO_NOUNMAP (1 << 0) /* do not free blocks */ #define BLKDEV_ZERO_NOFALLBACK (1 << 1) /* don't write explicit zeroes */ @@ -1297,8 +1316,8 @@ static inline unsigned int bdev_discard_granularity(struct block_device *bdev) return bdev_get_queue(bdev)->limits.discard_granularity; } -/* maximum copy offload length, this is set to 128MB based on current testing */ -#define BLK_COPY_MAX_BYTES (1 << 27) +/* Tested till 2GB with zvol testing */ +#define BLK_COPY_MAX_BYTES (1 << 31) static inline unsigned int bdev_max_copy_sectors(struct block_device *bdev) {