diff --git a/configure.d/1_bd_first_part.conf b/configure.d/1_bd_first_part.conf index ea70d937a..ac15f92fd 100644 --- a/configure.d/1_bd_first_part.conf +++ b/configure.d/1_bd_first_part.conf @@ -10,16 +10,19 @@ check() { cur_name=$(basename $2) config_file_path=$1 - if compile_module $cur_name "struct gendisk *disk = NULL; struct xarray xa; xa = disk->part_tbl;" "linux/genhd.h" || - compile_module $cur_name "struct gendisk *disk = NULL; struct xarray xa; xa = disk->part_tbl;" "linux/blkdev.h" + if compile_module $cur_name "struct block_device bd; bdev_partno;" "linux/blkdev.h" then echo $cur_name "1" >> $config_file_path - elif compile_module $cur_name "struct block_device bd; bd = *disk_part_iter_next(NULL);" "linux/blk_types.h" "linux/genhd.h" + elif compile_module $cur_name "struct gendisk *disk = NULL; struct xarray xa; xa = disk->part_tbl;" "linux/genhd.h" || + compile_module $cur_name "struct gendisk *disk = NULL; struct xarray xa; xa = disk->part_tbl;" "linux/blkdev.h" then echo $cur_name "2" >> $config_file_path - elif compile_module $cur_name "struct hd_struct hd; hd = *disk_part_iter_next(NULL);" "linux/genhd.h" + elif compile_module $cur_name "struct block_device bd; bd = *disk_part_iter_next(NULL);" "linux/blk_types.h" "linux/genhd.h" then echo $cur_name "3" >> $config_file_path + elif compile_module $cur_name "struct hd_struct hd; hd = *disk_part_iter_next(NULL);" "linux/genhd.h" + then + echo $cur_name "4" >> $config_file_path else echo $cur_name "X" >> $config_file_path fi @@ -37,7 +40,7 @@ apply() { unsigned long idx; xa_for_each(&disk->part_tbl, idx, part) { - if ((part_no = part->bd_partno)) { + if ((part_no = bdev_partno(part))) { break; } } @@ -47,6 +50,23 @@ apply() { "2") add_function " static inline int cas_bd_get_next_part(struct block_device *bd) + { + int part_no = 0; + struct gendisk *disk = bd->bd_disk; + struct block_device *part; + unsigned long idx; + + xa_for_each(&disk->part_tbl, idx, part) { + if ((part_no = part->bd_partno)) { + break; + } + } + + return part_no; + }" ;; + "3") + add_function " + static inline int cas_bd_get_next_part(struct block_device *bd) { int part_no = 0; struct gendisk *disk = bd->bd_disk; @@ -66,7 +86,7 @@ apply() { return part_no; }" ;; - "3") + "4") add_function " static inline int cas_bd_get_next_part(struct block_device *bd) { diff --git a/configure.d/1_queue_discard.conf b/configure.d/1_queue_discard.conf new file mode 100644 index 000000000..713013096 --- /dev/null +++ b/configure.d/1_queue_discard.conf @@ -0,0 +1,48 @@ +#!/bin/bash +# +# Copyright(c) 2012-2022 Intel Corporation +# Copyright(c) 2024 Huawei Technologies +# SPDX-License-Identifier: BSD-3-Clause +# + +. $(dirname $3)/conf_framework.sh + +check() { + cur_name=$(basename $2) + config_file_path=$1 + if compile_module $cur_name "blk_queue_max_discard_sectors(NULL, 0);" "linux/blkdev.h" + then + echo $cur_name "1" >> $config_file_path + else + echo $cur_name "2" >> $config_file_path + fi +} + +apply() { + case "$1" in + "1") + add_function " + static inline void cas_queue_max_discard_sectors( + struct request_queue *q, + unsigned int max_discard_sectors) + { + blk_queue_max_discard_sectors(q, max_discard_sectors); + }" ;; + "2") + add_function " + static inline void cas_queue_max_discard_sectors( + struct request_queue *q, + unsigned int max_discard_sectors) + { + struct queue_limits *lim = &q->limits; + + lim->max_hw_discard_sectors = max_discard_sectors; + lim->max_discard_sectors = + min(max_discard_sectors, lim->max_user_discard_sectors); + }" ;; + *) + exit 1 + esac +} + +conf_run $@ diff --git a/configure.d/1_queue_limits.conf b/configure.d/1_queue_limits.conf index 1f9149b77..ac89af3fd 100644 --- a/configure.d/1_queue_limits.conf +++ b/configure.d/1_queue_limits.conf @@ -12,20 +12,25 @@ check() { cur_name=$(basename $2) config_file_path=$1 - if compile_module $cur_name "struct queue_limits q; q.limits_aux;" "linux/blkdev.h" + if compile_module $cur_name "struct queue_limits q; q.max_write_zeroes_sectors;" "linux/blkdev.h" then - echo $cur_name "1" >> $config_file_path - elif compile_module $cur_name "struct queue_limits q; q.max_write_zeroes_sectors;" "linux/blkdev.h" - then - if compile_module $cur_name "struct queue_limits q; q.max_write_same_sectors;" "linux/blkdev.h" + if compile_module $cur_name "struct queue_limits q; q.misaligned;" "linux/blkdev.h" then - echo $cur_name "2" >> $config_file_path + if compile_module $cur_name "struct queue_limits q; q.max_write_same_sectors;" "linux/blkdev.h" + then + echo $cur_name "1" >> $config_file_path + else + echo $cur_name "2" >> $config_file_path + fi else echo $cur_name "3" >> $config_file_path fi elif compile_module $cur_name "struct queue_limits q; q.max_write_same_sectors;" "linux/blkdev.h" then echo $cur_name "4" >> $config_file_path + elif compile_module $cur_name "struct queue_limits q; q.limits_aux;" "linux/blkdev.h" + then + echo $cur_name "5" >> $config_file_path else echo $cur_name "X" >> $config_file_path fi @@ -38,30 +43,22 @@ apply() { static inline void cas_copy_queue_limits(struct request_queue *exp_q, struct queue_limits *cache_q_limits, struct request_queue *core_q) { - struct queue_limits_aux *l_aux = exp_q->limits.limits_aux; exp_q->limits = *cache_q_limits; - exp_q->limits.limits_aux = l_aux; - if (exp_q->limits.limits_aux && cache_q_limits->limits_aux) - *exp_q->limits.limits_aux = *cache_q_limits->limits_aux; exp_q->limits.max_sectors = core_q->limits.max_sectors; exp_q->limits.max_hw_sectors = core_q->limits.max_hw_sectors; exp_q->limits.max_segments = core_q->limits.max_segments; exp_q->limits.max_write_same_sectors = 0; + exp_q->limits.max_write_zeroes_sectors = 0; }" - # A workaround for RHEL/CentOS 7.3 bug in kernel. - # Merging implementation on blk-mq does not respect virt boundary - # restriction and front merges bios with non-zero offsets. - # This leads to request with gaps between bios and in consequence - # triggers BUG_ON() in nvme driver or silently corrupts data. - # To prevent this, disable merging on cache queue if there are - # requirements regarding virt boundary (marking bios with REQ_NOMERGE - # does not solve this problem). add_function " static inline void cas_cache_set_no_merges_flag(struct request_queue *cache_q) { - if (queue_virt_boundary(cache_q)) - queue_flag_set(QUEUE_FLAG_NOMERGES, cache_q); + } + static inline bool cas_queue_limits_is_misaligned( + struct queue_limits *lim) + { + return lim->misaligned; }" ;; "2") add_function " @@ -72,13 +69,17 @@ apply() { exp_q->limits.max_sectors = core_q->limits.max_sectors; exp_q->limits.max_hw_sectors = core_q->limits.max_hw_sectors; exp_q->limits.max_segments = core_q->limits.max_segments; - exp_q->limits.max_write_same_sectors = 0; exp_q->limits.max_write_zeroes_sectors = 0; }" add_function " static inline void cas_cache_set_no_merges_flag(struct request_queue *cache_q) { + } + static inline bool cas_queue_limits_is_misaligned( + struct queue_limits *lim) + { + return lim->misaligned; }" ;; "3") add_function " @@ -95,6 +96,11 @@ apply() { add_function " static inline void cas_cache_set_no_merges_flag(struct request_queue *cache_q) { + } + static inline bool cas_queue_limits_is_misaligned( + struct queue_limits *lim) + { + return lim->features & BLK_FLAG_MISALIGNED; }" ;; "4") add_function " @@ -111,6 +117,46 @@ apply() { add_function " static inline void cas_cache_set_no_merges_flag(struct request_queue *cache_q) { + } + static inline bool cas_queue_limits_is_misaligned( + struct queue_limits *lim) + { + return lim->misaligned; + }" ;; + "5") + add_function " + static inline void cas_copy_queue_limits(struct request_queue *exp_q, + struct queue_limits *cache_q_limits, struct request_queue *core_q) + { + struct queue_limits_aux *l_aux = exp_q->limits.limits_aux; + exp_q->limits = *cache_q_limits; + exp_q->limits.limits_aux = l_aux; + if (exp_q->limits.limits_aux && cache_q_limits->limits_aux) + *exp_q->limits.limits_aux = *cache_q_limits->limits_aux; + exp_q->limits.max_sectors = core_q->limits.max_sectors; + exp_q->limits.max_hw_sectors = core_q->limits.max_hw_sectors; + exp_q->limits.max_segments = core_q->limits.max_segments; + exp_q->limits.max_write_same_sectors = 0; + }" + + # A workaround for RHEL/CentOS 7.3 bug in kernel. + # Merging implementation on blk-mq does not respect virt boundary + # restriction and front merges bios with non-zero offsets. + # This leads to request with gaps between bios and in consequence + # triggers BUG_ON() in nvme driver or silently corrupts data. + # To prevent this, disable merging on cache queue if there are + # requirements regarding virt boundary (marking bios with REQ_NOMERGE + # does not solve this problem). + add_function " + static inline void cas_cache_set_no_merges_flag(struct request_queue *cache_q) + { + if (queue_virt_boundary(cache_q)) + queue_flag_set(QUEUE_FLAG_NOMERGES, cache_q); + } + static inline bool cas_queue_limits_is_misaligned( + struct queue_limits *lim) + { + return lim->misaligned; }" ;; diff --git a/configure.d/1_queue_nonrot.conf b/configure.d/1_queue_nonrot.conf new file mode 100644 index 000000000..0fa5f95f3 --- /dev/null +++ b/configure.d/1_queue_nonrot.conf @@ -0,0 +1,39 @@ +#!/bin/bash +# +# Copyright(c) 2012-2022 Intel Corporation +# Copyright(c) 2024 Huawei Technologies +# SPDX-License-Identifier: BSD-3-Clause +# + +. $(dirname $3)/conf_framework.sh + +check() { + cur_name=$(basename $2) + config_file_path=$1 + if compile_module $cur_name "(int)QUEUE_FLAG_NONROT;" "linux/blkdev.h" + then + echo $cur_name "1" >> $config_file_path + else + echo $cur_name "2" >> $config_file_path + fi +} + +apply() { + case "$1" in + "1") + add_function " + static inline void cas_queue_set_nonrot(struct request_queue *q) + { + q->queue_flags |= (1 << QUEUE_FLAG_NONROT); + }" ;; + "2") + add_function " + static inline void cas_queue_set_nonrot(struct request_queue *q) + { + }" ;; + *) + exit 1 + esac +} + +conf_run $@ diff --git a/configure.d/2_alloc_disk.conf b/configure.d/2_alloc_disk.conf index 068635791..4b14faea9 100644 --- a/configure.d/2_alloc_disk.conf +++ b/configure.d/2_alloc_disk.conf @@ -28,11 +28,12 @@ check() { apply() { case "$1" in "1") + add_typedef "struct queue_limits cas_queue_limits_t;" add_function " static inline int cas_alloc_mq_disk(struct gendisk **gd, struct request_queue **queue, - struct blk_mq_tag_set *tag_set) + struct blk_mq_tag_set *tag_set, cas_queue_limits_t *lim) { - *gd = blk_mq_alloc_disk(tag_set, NULL, NULL); + *gd = blk_mq_alloc_disk(tag_set, lim, NULL); if (!(*gd)) return -ENOMEM; @@ -48,9 +49,10 @@ apply() { ;; "2") + add_typedef "void* cas_queue_limits_t;" add_function " static inline int cas_alloc_mq_disk(struct gendisk **gd, struct request_queue **queue, - struct blk_mq_tag_set *tag_set) + struct blk_mq_tag_set *tag_set, cas_queue_limits_t *lim) { *gd = blk_mq_alloc_disk(tag_set, NULL); if (!(*gd)) @@ -68,10 +70,10 @@ apply() { ;; "3") - + add_typedef "void* cas_queue_limits_t;" add_function " static inline int cas_alloc_mq_disk(struct gendisk **gd, struct request_queue **queue, - struct blk_mq_tag_set *tag_set) + struct blk_mq_tag_set *tag_set, cas_queue_limits_t *lim) { *gd = alloc_disk(1); if (!(*gd)) diff --git a/configure.d/2_queue_write.conf b/configure.d/2_queue_write.conf index d7a29c957..8e18dc6c0 100644 --- a/configure.d/2_queue_write.conf +++ b/configure.d/2_queue_write.conf @@ -1,6 +1,7 @@ #!/bin/bash # # Copyright(c) 2012-2022 Intel Corporation +# Copyright(c) 2024 Huawei Technologies # SPDX-License-Identifier: BSD-3-Clause # @@ -9,12 +10,15 @@ check() { cur_name=$(basename $2) config_file_path=$1 - if compile_module $cur_name "blk_queue_write_cache(NULL, 0, 0);" "linux/blkdev.h" + if compile_module $cur_name "BLK_FEAT_WRITE_CACHE;" "linux/blk-mq.h" then echo $cur_name "1" >> $config_file_path - elif compile_module $cur_name "struct request_queue rq; rq.flush_flags;" "linux/blkdev.h" + elif compile_module $cur_name "blk_queue_write_cache(NULL, 0, 0);" "linux/blkdev.h" then echo $cur_name "2" >> $config_file_path + elif compile_module $cur_name "struct request_queue rq; rq.flush_flags;" "linux/blkdev.h" + then + echo $cur_name "3" >> $config_file_path else echo $cur_name "X" >> $config_file_path fi @@ -23,21 +27,39 @@ check() { apply() { case "$1" in "1") + add_define "CAS_CHECK_QUEUE_FLUSH(q) \\ + (q->limits.features & BLK_FEAT_WRITE_CACHE)" + add_define "CAS_CHECK_QUEUE_FUA(q) \\ + (q->limits.features & BLK_FEAT_FUA)" + add_define "CAS_BLK_FEAT_WRITE_CACHE BLK_FEAT_WRITE_CACHE" + add_define "CAS_BLK_FEAT_FUA BLK_FEAT_FUA" + add_define "CAS_SET_QUEUE_LIMIT(lim, flag) \\ + ({ lim->features |= flag; })" + add_function " + static inline void cas_set_queue_flush_fua(struct request_queue *q, + bool flush, bool fua) {}" ;; + "2") add_define "CAS_CHECK_QUEUE_FLUSH(q) \\ test_bit(QUEUE_FLAG_WC, &(q)->queue_flags)" add_define "CAS_CHECK_QUEUE_FUA(q) \\ test_bit(QUEUE_FLAG_FUA, &(q)->queue_flags)" + add_define "CAS_BLK_FEAT_WRITE_CACHE 0" + add_define "CAS_BLK_FEAT_FUA 0" + add_define "CAS_SET_QUEUE_LIMIT(lim, flag)" add_function " static inline void cas_set_queue_flush_fua(struct request_queue *q, bool flush, bool fua) { blk_queue_write_cache(q, flush, fua); }" ;; - "2") + "3") add_define "CAS_CHECK_QUEUE_FLUSH(q) \\ CAS_IS_SET_FLUSH((q)->flush_flags)" add_define "CAS_CHECK_QUEUE_FUA(q) \\ ((q)->flush_flags & REQ_FUA)" + add_define "CAS_BLK_FEAT_WRITE_CACHE 0" + add_define "CAS_BLK_FEAT_FUA 0" + add_define "CAS_SET_QUEUE_LIMIT(lim, flag)" add_function "static inline void cas_set_queue_flush_fua(struct request_queue *q, bool flush, bool fua) { diff --git a/modules/cas_cache/classifier.c b/modules/cas_cache/classifier.c index ca1c3af63..3f8ee8bed 100644 --- a/modules/cas_cache/classifier.c +++ b/modules/cas_cache/classifier.c @@ -29,8 +29,8 @@ trace_printk(format, ##__VA_ARGS__) #else -#define CAS_CLS_DEBUG_MSG(format, ...) -#define CAS_CLS_DEBUG_TRACE(format, ...) +#define CAS_CLS_DEBUG_MSG(format, ...) ({}) +#define CAS_CLS_DEBUG_TRACE(format, ...) ({}) #endif /* Done condition test - always accepts and stops evaluation */ diff --git a/modules/cas_cache/exp_obj.c b/modules/cas_cache/exp_obj.c index e44ae6ba9..d7a5be619 100644 --- a/modules/cas_cache/exp_obj.c +++ b/modules/cas_cache/exp_obj.c @@ -417,6 +417,7 @@ int cas_exp_obj_create(struct cas_disk *dsk, const char *dev_name, struct cas_exp_obj *exp_obj; struct request_queue *queue; struct gendisk *gd; + cas_queue_limits_t queue_limits; int result = 0; BUG_ON(!owner); @@ -465,7 +466,15 @@ int cas_exp_obj_create(struct cas_disk *dsk, const char *dev_name, goto error_init_tag_set; } - result = cas_alloc_mq_disk(&gd, &queue, &exp_obj->tag_set); + if (exp_obj->ops->set_queue_limits) { + result = exp_obj->ops->set_queue_limits(dsk, priv, + &queue_limits); + if (result) + goto error_set_queue_limits; + } + + result = cas_alloc_mq_disk(&gd, &queue, &exp_obj->tag_set, + &queue_limits); if (result) { goto error_alloc_mq_disk; } @@ -521,6 +530,7 @@ int cas_exp_obj_create(struct cas_disk *dsk, const char *dev_name, cas_cleanup_mq_disk(gd); exp_obj->gd = NULL; error_alloc_mq_disk: +error_set_queue_limits: blk_mq_free_tag_set(&exp_obj->tag_set); error_init_tag_set: module_put(owner); diff --git a/modules/cas_cache/exp_obj.h b/modules/cas_cache/exp_obj.h index be52f5a93..85447e48d 100644 --- a/modules/cas_cache/exp_obj.h +++ b/modules/cas_cache/exp_obj.h @@ -6,6 +6,7 @@ #ifndef __CASDISK_EXP_OBJ_H__ #define __CASDISK_EXP_OBJ_H__ +#include "linux_kernel_version.h" #include struct cas_disk; @@ -17,6 +18,12 @@ struct cas_exp_obj_ops { */ int (*set_geometry)(struct cas_disk *dsk, void *private); + /** + * @brief Set queue limits of exported object (top) block device. + */ + int (*set_queue_limits)(struct cas_disk *dsk, void *private, + cas_queue_limits_t *lim); + /** * @brief submit_bio of exported object (top) block device. * diff --git a/modules/cas_cache/layer_cache_management.c b/modules/cas_cache/layer_cache_management.c index 7c11d3a8f..d8e28812f 100644 --- a/modules/cas_cache/layer_cache_management.c +++ b/modules/cas_cache/layer_cache_management.c @@ -2405,7 +2405,8 @@ static int cache_mngt_check_bdev(struct ocf_mngt_cache_device_config *cfg, printk(KERN_WARNING "New cache device block properties " "differ from the previous one.\n"); } - if (tmp_limits.misaligned) { + + if (cas_queue_limits_is_misaligned(&tmp_limits)) { reattach_properties_diff = true; printk(KERN_WARNING "New cache device block interval " "doesn't line up with the previous one.\n"); diff --git a/modules/cas_cache/linux_kernel_version.h b/modules/cas_cache/linux_kernel_version.h index 41eda5b88..6c3c9bd01 100644 --- a/modules/cas_cache/linux_kernel_version.h +++ b/modules/cas_cache/linux_kernel_version.h @@ -40,7 +40,6 @@ #include #include #include -#include "exp_obj.h" #include "generated_defines.h" diff --git a/modules/cas_cache/volume/vol_block_dev_top.c b/modules/cas_cache/volume/vol_block_dev_top.c index be79dc05a..9f09cef20 100644 --- a/modules/cas_cache/volume/vol_block_dev_top.c +++ b/modules/cas_cache/volume/vol_block_dev_top.c @@ -63,13 +63,13 @@ static void blkdev_set_discard_properties(ocf_cache_t cache, CAS_SET_DISCARD_ZEROES_DATA(exp_q->limits, 0); if (core_q && cas_has_discard_support(core_bd)) { - blk_queue_max_discard_sectors(exp_q, core_q->limits.max_discard_sectors); + cas_queue_max_discard_sectors(exp_q, core_q->limits.max_discard_sectors); exp_q->limits.discard_alignment = bdev_discard_alignment(core_bd); exp_q->limits.discard_granularity = core_q->limits.discard_granularity; } else { - blk_queue_max_discard_sectors(exp_q, + cas_queue_max_discard_sectors(exp_q, min((uint64_t)core_sectors, (uint64_t)UINT_MAX)); exp_q->limits.discard_granularity = ocf_cache_get_line_size(cache); exp_q->limits.discard_alignment = 0; @@ -129,7 +129,34 @@ static int blkdev_core_set_geometry(struct cas_disk *dsk, void *private) blkdev_set_discard_properties(cache, exp_q, core_bd, sectors); - exp_q->queue_flags |= (1 << QUEUE_FLAG_NONROT); + cas_queue_set_nonrot(exp_q); + + return 0; +} + +static int blkdev_core_set_queue_limits(struct cas_disk *dsk, void *private, + cas_queue_limits_t *lim) +{ + ocf_core_t core = private; + ocf_cache_t cache = ocf_core_get_cache(core); + ocf_volume_t core_vol = ocf_core_get_volume(core); + struct bd_object *bd_core_vol; + struct request_queue *core_q; + bool flush, fua; + struct cache_priv *cache_priv = ocf_cache_get_priv(cache); + bd_core_vol = bd_object(core_vol); + + core_q = cas_disk_get_queue(bd_core_vol->dsk); + + flush = (CAS_CHECK_QUEUE_FLUSH(core_q) || + cache_priv->device_properties.flush); + fua = (CAS_CHECK_QUEUE_FUA(core_q) || cache_priv->device_properties.fua); + + if (flush) + CAS_SET_QUEUE_LIMIT(lim, CAS_BLK_FEAT_WRITE_CACHE); + + if (fua) + CAS_SET_QUEUE_LIMIT(lim, CAS_BLK_FEAT_FUA); return 0; } @@ -428,6 +455,7 @@ static void blkdev_core_submit_bio(struct cas_disk *dsk, static struct cas_exp_obj_ops kcas_core_exp_obj_ops = { .set_geometry = blkdev_core_set_geometry, + .set_queue_limits = blkdev_core_set_queue_limits, .submit_bio = blkdev_core_submit_bio, }; @@ -470,6 +498,35 @@ static int blkdev_cache_set_geometry(struct cas_disk *dsk, void *private) return 0; } +static int blkdev_cache_set_queue_limits(struct cas_disk *dsk, void *private, + cas_queue_limits_t *lim) +{ + ocf_cache_t cache; + ocf_volume_t volume; + struct bd_object *bvol; + struct request_queue *cache_q; + struct block_device *bd; + + BUG_ON(!private); + cache = private; + volume = ocf_cache_get_volume(cache); + + bvol = bd_object(volume); + + bd = cas_disk_get_blkdev(bvol->dsk); + BUG_ON(!bd); + + cache_q = bd->bd_disk->queue; + + if (CAS_CHECK_QUEUE_FLUSH(cache_q)) + CAS_SET_QUEUE_LIMIT(lim, CAS_BLK_FEAT_WRITE_CACHE); + + if (CAS_CHECK_QUEUE_FUA(cache_q)) + CAS_SET_QUEUE_LIMIT(lim, CAS_BLK_FEAT_FUA); + + return 0; +} + static void blkdev_cache_submit_bio(struct cas_disk *dsk, struct bio *bio, void *private) { @@ -485,6 +542,7 @@ static void blkdev_cache_submit_bio(struct cas_disk *dsk, static struct cas_exp_obj_ops kcas_cache_exp_obj_ops = { .set_geometry = blkdev_cache_set_geometry, + .set_queue_limits = blkdev_cache_set_queue_limits, .submit_bio = blkdev_cache_submit_bio, };