dm-verity: Convert from tasklet to BH workqueue
The only generic interface to execute asynchronously in the BH context is
tasklet; however, it's marked deprecated and has some design flaws. To
replace tasklets, BH workqueue support was recently added. A BH workqueue
behaves similarly to regular workqueues except that the queued work items
are executed in the BH context.
This commit converts dm-verity from tasklet to BH workqueue. It
backfills tasklet code that was removed with commit 0a9bab391e
("dm-crypt, dm-verity: disable tasklets") and tweaks to use BH
workqueue (and does some renaming).
This is a minimal conversion which doesn't rename the related names
including the "try_verify_in_tasklet" option. If this patch is applied, a
follow-up patch would be necessary. I couldn't decide whether the option
name would need to be updated too.
Signed-off-by: Tejun Heo <tj@kernel.org>
[snitzer: rename 'use_tasklet' to 'use_bh_wq' and 'in_tasklet' to 'in_bh']
Signed-off-by: Mike Snitzer <snitzer@kernel.org>
This commit is contained in:
parent
fb6ad4aec1
commit
c375b22333
2 changed files with 45 additions and 24 deletions
|
@ -46,7 +46,7 @@ static unsigned int dm_verity_prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE
|
|||
|
||||
module_param_named(prefetch_cluster, dm_verity_prefetch_cluster, uint, 0644);
|
||||
|
||||
static DEFINE_STATIC_KEY_FALSE(use_tasklet_enabled);
|
||||
static DEFINE_STATIC_KEY_FALSE(use_bh_wq_enabled);
|
||||
|
||||
struct dm_verity_prefetch_work {
|
||||
struct work_struct work;
|
||||
|
@ -299,7 +299,7 @@ static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io,
|
|||
|
||||
verity_hash_at_level(v, block, level, &hash_block, &offset);
|
||||
|
||||
if (static_branch_unlikely(&use_tasklet_enabled) && io->in_tasklet) {
|
||||
if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) {
|
||||
data = dm_bufio_get(v->bufio, hash_block, &buf);
|
||||
if (data == NULL) {
|
||||
/*
|
||||
|
@ -327,15 +327,14 @@ static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io,
|
|||
|
||||
r = verity_hash(v, verity_io_hash_req(v, io),
|
||||
data, 1 << v->hash_dev_block_bits,
|
||||
verity_io_real_digest(v, io), !io->in_tasklet);
|
||||
verity_io_real_digest(v, io), !io->in_bh);
|
||||
if (unlikely(r < 0))
|
||||
goto release_ret_r;
|
||||
|
||||
if (likely(memcmp(verity_io_real_digest(v, io), want_digest,
|
||||
v->digest_size) == 0))
|
||||
aux->hash_verified = 1;
|
||||
else if (static_branch_unlikely(&use_tasklet_enabled) &&
|
||||
io->in_tasklet) {
|
||||
else if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) {
|
||||
/*
|
||||
* Error handling code (FEC included) cannot be run in a
|
||||
* tasklet since it may sleep, so fallback to work-queue.
|
||||
|
@ -576,7 +575,7 @@ static int verity_verify_io(struct dm_verity_io *io)
|
|||
struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size);
|
||||
unsigned int b;
|
||||
|
||||
if (static_branch_unlikely(&use_tasklet_enabled) && io->in_tasklet) {
|
||||
if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) {
|
||||
/*
|
||||
* Copy the iterator in case we need to restart
|
||||
* verification in a work-queue.
|
||||
|
@ -616,7 +615,7 @@ static int verity_verify_io(struct dm_verity_io *io)
|
|||
continue;
|
||||
}
|
||||
|
||||
r = verity_hash_init(v, req, &wait, !io->in_tasklet);
|
||||
r = verity_hash_init(v, req, &wait, !io->in_bh);
|
||||
if (unlikely(r < 0))
|
||||
return r;
|
||||
|
||||
|
@ -635,8 +634,7 @@ static int verity_verify_io(struct dm_verity_io *io)
|
|||
if (v->validated_blocks)
|
||||
set_bit(cur_block, v->validated_blocks);
|
||||
continue;
|
||||
} else if (static_branch_unlikely(&use_tasklet_enabled) &&
|
||||
io->in_tasklet) {
|
||||
} else if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) {
|
||||
/*
|
||||
* Error handling code (FEC included) cannot be run in a
|
||||
* tasklet since it may sleep, so fallback to work-queue.
|
||||
|
@ -690,7 +688,7 @@ static void verity_finish_io(struct dm_verity_io *io, blk_status_t status)
|
|||
bio->bi_end_io = io->orig_bi_end_io;
|
||||
bio->bi_status = status;
|
||||
|
||||
if (!static_branch_unlikely(&use_tasklet_enabled) || !io->in_tasklet)
|
||||
if (!static_branch_unlikely(&use_bh_wq_enabled) || !io->in_bh)
|
||||
verity_fec_finish_io(io);
|
||||
|
||||
bio_endio(bio);
|
||||
|
@ -700,11 +698,28 @@ static void verity_work(struct work_struct *w)
|
|||
{
|
||||
struct dm_verity_io *io = container_of(w, struct dm_verity_io, work);
|
||||
|
||||
io->in_tasklet = false;
|
||||
io->in_bh = false;
|
||||
|
||||
verity_finish_io(io, errno_to_blk_status(verity_verify_io(io)));
|
||||
}
|
||||
|
||||
static void verity_bh_work(struct work_struct *w)
|
||||
{
|
||||
struct dm_verity_io *io = container_of(w, struct dm_verity_io, bh_work);
|
||||
int err;
|
||||
|
||||
io->in_bh = true;
|
||||
err = verity_verify_io(io);
|
||||
if (err == -EAGAIN || err == -ENOMEM) {
|
||||
/* fallback to retrying with work-queue */
|
||||
INIT_WORK(&io->work, verity_work);
|
||||
queue_work(io->v->verify_wq, &io->work);
|
||||
return;
|
||||
}
|
||||
|
||||
verity_finish_io(io, errno_to_blk_status(err));
|
||||
}
|
||||
|
||||
static void verity_end_io(struct bio *bio)
|
||||
{
|
||||
struct dm_verity_io *io = bio->bi_private;
|
||||
|
@ -717,8 +732,13 @@ static void verity_end_io(struct bio *bio)
|
|||
return;
|
||||
}
|
||||
|
||||
INIT_WORK(&io->work, verity_work);
|
||||
queue_work(io->v->verify_wq, &io->work);
|
||||
if (static_branch_unlikely(&use_bh_wq_enabled) && io->v->use_bh_wq) {
|
||||
INIT_WORK(&io->bh_work, verity_bh_work);
|
||||
queue_work(system_bh_wq, &io->bh_work);
|
||||
} else {
|
||||
INIT_WORK(&io->work, verity_work);
|
||||
queue_work(io->v->verify_wq, &io->work);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
|
@ -885,7 +905,7 @@ static void verity_status(struct dm_target *ti, status_type_t type,
|
|||
args++;
|
||||
if (v->validated_blocks)
|
||||
args++;
|
||||
if (v->use_tasklet)
|
||||
if (v->use_bh_wq)
|
||||
args++;
|
||||
if (v->signature_key_desc)
|
||||
args += DM_VERITY_ROOT_HASH_VERIFICATION_OPTS;
|
||||
|
@ -912,7 +932,7 @@ static void verity_status(struct dm_target *ti, status_type_t type,
|
|||
DMEMIT(" " DM_VERITY_OPT_IGN_ZEROES);
|
||||
if (v->validated_blocks)
|
||||
DMEMIT(" " DM_VERITY_OPT_AT_MOST_ONCE);
|
||||
if (v->use_tasklet)
|
||||
if (v->use_bh_wq)
|
||||
DMEMIT(" " DM_VERITY_OPT_TASKLET_VERIFY);
|
||||
sz = verity_fec_status_table(v, sz, result, maxlen);
|
||||
if (v->signature_key_desc)
|
||||
|
@ -1031,8 +1051,8 @@ static void verity_dtr(struct dm_target *ti)
|
|||
|
||||
kfree(v->signature_key_desc);
|
||||
|
||||
if (v->use_tasklet)
|
||||
static_branch_dec(&use_tasklet_enabled);
|
||||
if (v->use_bh_wq)
|
||||
static_branch_dec(&use_bh_wq_enabled);
|
||||
|
||||
kfree(v);
|
||||
|
||||
|
@ -1166,8 +1186,8 @@ static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v,
|
|||
continue;
|
||||
|
||||
} else if (!strcasecmp(arg_name, DM_VERITY_OPT_TASKLET_VERIFY)) {
|
||||
v->use_tasklet = true;
|
||||
static_branch_inc(&use_tasklet_enabled);
|
||||
v->use_bh_wq = true;
|
||||
static_branch_inc(&use_bh_wq_enabled);
|
||||
continue;
|
||||
|
||||
} else if (verity_is_fec_opt_arg(arg_name)) {
|
||||
|
@ -1338,7 +1358,7 @@ static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv)
|
|||
}
|
||||
|
||||
v->tfm = crypto_alloc_ahash(v->alg_name, 0,
|
||||
v->use_tasklet ? CRYPTO_ALG_ASYNC : 0);
|
||||
v->use_bh_wq ? CRYPTO_ALG_ASYNC : 0);
|
||||
if (IS_ERR(v->tfm)) {
|
||||
ti->error = "Cannot initialize hash function";
|
||||
r = PTR_ERR(v->tfm);
|
||||
|
@ -1463,7 +1483,7 @@ static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv)
|
|||
v->bufio = dm_bufio_client_create(v->hash_dev->bdev,
|
||||
1 << v->hash_dev_block_bits, 1, sizeof(struct buffer_aux),
|
||||
dm_bufio_alloc_callback, NULL,
|
||||
v->use_tasklet ? DM_BUFIO_CLIENT_NO_SLEEP : 0);
|
||||
v->use_bh_wq ? DM_BUFIO_CLIENT_NO_SLEEP : 0);
|
||||
if (IS_ERR(v->bufio)) {
|
||||
ti->error = "Cannot initialize dm-bufio";
|
||||
r = PTR_ERR(v->bufio);
|
||||
|
@ -1482,7 +1502,7 @@ static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv)
|
|||
* reducing wait times when reading from a dm-verity device.
|
||||
*
|
||||
* Also as required for the "try_verify_in_tasklet" feature: WQ_HIGHPRI
|
||||
* allows verify_wq to preempt softirq since verification in tasklet
|
||||
* allows verify_wq to preempt softirq since verification in BH workqueue
|
||||
* will fall-back to using it for error handling (or if the bufio cache
|
||||
* doesn't have required hashes).
|
||||
*/
|
||||
|
|
|
@ -54,7 +54,7 @@ struct dm_verity {
|
|||
unsigned char levels; /* the number of tree levels */
|
||||
unsigned char version;
|
||||
bool hash_failed:1; /* set if hash of any block failed */
|
||||
bool use_tasklet:1; /* try to verify in tasklet before work-queue */
|
||||
bool use_bh_wq:1; /* try to verify in BH wq before normal work-queue */
|
||||
unsigned int digest_size; /* digest size for the current hash algorithm */
|
||||
unsigned int ahash_reqsize;/* the size of temporary space for crypto */
|
||||
enum verity_mode mode; /* mode for handling verification errors */
|
||||
|
@ -84,9 +84,10 @@ struct dm_verity_io {
|
|||
|
||||
sector_t block;
|
||||
unsigned int n_blocks;
|
||||
bool in_tasklet;
|
||||
bool in_bh;
|
||||
|
||||
struct work_struct work;
|
||||
struct work_struct bh_work;
|
||||
|
||||
char *recheck_buffer;
|
||||
|
||||
|
|
Loading…
Reference in a new issue