diff options
Diffstat (limited to 'fs/f2fs/segment.c')
| -rw-r--r-- | fs/f2fs/segment.c | 298 | 
1 files changed, 238 insertions, 60 deletions
diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index f964b68718c1..c695ff462ee6 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -17,10 +17,12 @@  #include <linux/swap.h>  #include <linux/timer.h>  #include <linux/freezer.h> +#include <linux/sched/signal.h>  #include "f2fs.h"  #include "segment.h"  #include "node.h" +#include "gc.h"  #include "trace.h"  #include <trace/events/f2fs.h> @@ -167,6 +169,21 @@ found:  	return result - size + __reverse_ffz(tmp);  } +bool need_SSR(struct f2fs_sb_info *sbi) +{ +	int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES); +	int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS); +	int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA); + +	if (test_opt(sbi, LFS)) +		return false; +	if (sbi->gc_thread && sbi->gc_thread->gc_urgent) +		return true; + +	return free_sections(sbi) <= (node_secs + 2 * dent_secs + imeta_secs + +						2 * reserved_sections(sbi)); +} +  void register_inmem_page(struct inode *inode, struct page *page)  {  	struct f2fs_inode_info *fi = F2FS_I(inode); @@ -213,9 +230,15 @@ static int __revoke_inmem_pages(struct inode *inode,  			struct node_info ni;  			trace_f2fs_commit_inmem_page(page, INMEM_REVOKE); - +retry:  			set_new_dnode(&dn, inode, NULL, NULL, 0); -			if (get_dnode_of_data(&dn, page->index, LOOKUP_NODE)) { +			err = get_dnode_of_data(&dn, page->index, LOOKUP_NODE); +			if (err) { +				if (err == -ENOMEM) { +					congestion_wait(BLK_RW_ASYNC, HZ/50); +					cond_resched(); +					goto retry; +				}  				err = -EAGAIN;  				goto next;  			} @@ -248,6 +271,7 @@ void drop_inmem_pages(struct inode *inode)  	mutex_unlock(&fi->inmem_lock);  	clear_inode_flag(inode, FI_ATOMIC_FILE); +	clear_inode_flag(inode, FI_HOT_DATA);  	stat_dec_atomic_write(inode);  } @@ -292,6 +316,7 @@ static int __commit_inmem_pages(struct inode *inode,  		.type = DATA,  		.op = REQ_OP_WRITE,  		.op_flags = REQ_SYNC | REQ_PRIO, +		.io_type = FS_DATA_IO,  	};  	pgoff_t last_idx = ULONG_MAX;  	int err = 0; @@ -309,17 +334,21 @@ static int __commit_inmem_pages(struct inode *inode,  				inode_dec_dirty_pages(inode);  				remove_dirty_inode(inode);  			} - +retry:  			fio.page = page;  			fio.old_blkaddr = NULL_ADDR;  			fio.encrypted_page = NULL;  			fio.need_lock = LOCK_DONE;  			err = do_write_data_page(&fio);  			if (err) { +				if (err == -ENOMEM) { +					congestion_wait(BLK_RW_ASYNC, HZ/50); +					cond_resched(); +					goto retry; +				}  				unlock_page(page);  				break;  			} -  			/* record old blkaddr for revoking */  			cur->old_addr = fio.old_blkaddr;  			last_idx = page->index; @@ -447,7 +476,7 @@ static int __submit_flush_wait(struct f2fs_sb_info *sbi,  	int ret;  	bio->bi_opf = REQ_OP_WRITE | REQ_SYNC | REQ_PREFLUSH; -	bio->bi_bdev = bdev; +	bio_set_dev(bio, bdev);  	ret = submit_bio_wait(bio);  	bio_put(bio); @@ -481,6 +510,8 @@ repeat:  	if (kthread_should_stop())  		return 0; +	sb_start_intwrite(sbi->sb); +  	if (!llist_empty(&fcc->issue_list)) {  		struct flush_cmd *cmd, *next;  		int ret; @@ -499,6 +530,8 @@ repeat:  		fcc->dispatch_list = NULL;  	} +	sb_end_intwrite(sbi->sb); +  	wait_event_interruptible(*q,  		kthread_should_stop() || !llist_empty(&fcc->issue_list));  	goto repeat; @@ -519,8 +552,7 @@ int f2fs_issue_flush(struct f2fs_sb_info *sbi)  		return ret;  	} -	if (!atomic_read(&fcc->issing_flush)) { -		atomic_inc(&fcc->issing_flush); +	if (atomic_inc_return(&fcc->issing_flush) == 1) {  		ret = submit_flush_wait(sbi);  		atomic_dec(&fcc->issing_flush); @@ -530,18 +562,39 @@ int f2fs_issue_flush(struct f2fs_sb_info *sbi)  	init_completion(&cmd.wait); -	atomic_inc(&fcc->issing_flush);  	llist_add(&cmd.llnode, &fcc->issue_list); -	if (!fcc->dispatch_list) +	/* update issue_list before we wake up issue_flush thread */ +	smp_mb(); + +	if (waitqueue_active(&fcc->flush_wait_queue))  		wake_up(&fcc->flush_wait_queue);  	if (fcc->f2fs_issue_flush) {  		wait_for_completion(&cmd.wait);  		atomic_dec(&fcc->issing_flush);  	} else { -		llist_del_all(&fcc->issue_list); -		atomic_set(&fcc->issing_flush, 0); +		struct llist_node *list; + +		list = llist_del_all(&fcc->issue_list); +		if (!list) { +			wait_for_completion(&cmd.wait); +			atomic_dec(&fcc->issing_flush); +		} else { +			struct flush_cmd *tmp, *next; + +			ret = submit_flush_wait(sbi); + +			llist_for_each_entry_safe(tmp, next, list, llnode) { +				if (tmp == &cmd) { +					cmd.ret = ret; +					atomic_dec(&fcc->issing_flush); +					continue; +				} +				tmp->ret = ret; +				complete(&tmp->wait); +			} +		}  	}  	return cmd.ret; @@ -778,11 +831,14 @@ void __check_sit_bitmap(struct f2fs_sb_info *sbi,  		sentry = get_seg_entry(sbi, segno);  		offset = GET_BLKOFF_FROM_SEG0(sbi, blk); -		size = min((unsigned long)(end - blk), max_blocks); +		if (end < START_BLOCK(sbi, segno + 1)) +			size = GET_BLKOFF_FROM_SEG0(sbi, end); +		else +			size = max_blocks;  		map = (unsigned long *)(sentry->cur_valid_map);  		offset = __find_rev_next_bit(map, size, offset);  		f2fs_bug_on(sbi, offset != size); -		blk += size; +		blk = START_BLOCK(sbi, segno + 1);  	}  #endif  } @@ -815,6 +871,8 @@ static void __submit_discard_cmd(struct f2fs_sb_info *sbi,  			submit_bio(bio);  			list_move_tail(&dc->list, &dcc->wait_list);  			__check_sit_bitmap(sbi, dc->start, dc->start + dc->len); + +			f2fs_update_iostat(sbi, FS_DISCARD, 1);  		}  	} else {  		__remove_discard_cmd(sbi, dc); @@ -996,32 +1054,81 @@ static int __queue_discard_cmd(struct f2fs_sb_info *sbi,  	return 0;  } -static void __issue_discard_cmd(struct f2fs_sb_info *sbi, bool issue_cond) +static int __issue_discard_cmd(struct f2fs_sb_info *sbi, bool issue_cond)  {  	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  	struct list_head *pend_list;  	struct discard_cmd *dc, *tmp;  	struct blk_plug plug; -	int i, iter = 0; +	int iter = 0, issued = 0; +	int i; +	bool io_interrupted = false;  	mutex_lock(&dcc->cmd_lock);  	f2fs_bug_on(sbi,  		!__check_rb_tree_consistence(sbi, &dcc->root));  	blk_start_plug(&plug); -	for (i = MAX_PLIST_NUM - 1; i >= 0; i--) { +	for (i = MAX_PLIST_NUM - 1; +			i >= 0 && plist_issue(dcc->pend_list_tag[i]); i--) {  		pend_list = &dcc->pend_list[i];  		list_for_each_entry_safe(dc, tmp, pend_list, list) {  			f2fs_bug_on(sbi, dc->state != D_PREP); -			if (!issue_cond || is_idle(sbi)) +			/* Hurry up to finish fstrim */ +			if (dcc->pend_list_tag[i] & P_TRIM) { +				__submit_discard_cmd(sbi, dc); +				issued++; + +				if (fatal_signal_pending(current)) +					break; +				continue; +			} + +			if (!issue_cond) {  				__submit_discard_cmd(sbi, dc); -			if (issue_cond && iter++ > DISCARD_ISSUE_RATE) +				issued++; +				continue; +			} + +			if (is_idle(sbi)) { +				__submit_discard_cmd(sbi, dc); +				issued++; +			} else { +				io_interrupted = true; +			} + +			if (++iter >= DISCARD_ISSUE_RATE)  				goto out;  		} +		if (list_empty(pend_list) && dcc->pend_list_tag[i] & P_TRIM) +			dcc->pend_list_tag[i] &= (~P_TRIM);  	}  out:  	blk_finish_plug(&plug);  	mutex_unlock(&dcc->cmd_lock); + +	if (!issued && io_interrupted) +		issued = -1; + +	return issued; +} + +static void __drop_discard_cmd(struct f2fs_sb_info *sbi) +{ +	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; +	struct list_head *pend_list; +	struct discard_cmd *dc, *tmp; +	int i; + +	mutex_lock(&dcc->cmd_lock); +	for (i = MAX_PLIST_NUM - 1; i >= 0; i--) { +		pend_list = &dcc->pend_list[i]; +		list_for_each_entry_safe(dc, tmp, pend_list, list) { +			f2fs_bug_on(sbi, dc->state != D_PREP); +			__remove_discard_cmd(sbi, dc); +		} +	} +	mutex_unlock(&dcc->cmd_lock);  }  static void __wait_one_discard_bio(struct f2fs_sb_info *sbi, @@ -1102,11 +1209,23 @@ void stop_discard_thread(struct f2fs_sb_info *sbi)  	}  } -/* This comes from f2fs_put_super */ -void f2fs_wait_discard_bios(struct f2fs_sb_info *sbi) +/* This comes from f2fs_put_super and f2fs_trim_fs */ +void f2fs_wait_discard_bios(struct f2fs_sb_info *sbi, bool umount)  {  	__issue_discard_cmd(sbi, false); -	__wait_discard_cmd(sbi, false); +	__drop_discard_cmd(sbi); +	__wait_discard_cmd(sbi, !umount); +} + +static void mark_discard_range_all(struct f2fs_sb_info *sbi) +{ +	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; +	int i; + +	mutex_lock(&dcc->cmd_lock); +	for (i = 0; i < MAX_PLIST_NUM; i++) +		dcc->pend_list_tag[i] |= P_TRIM; +	mutex_unlock(&dcc->cmd_lock);  }  static int issue_discard_thread(void *data) @@ -1114,22 +1233,39 @@ static int issue_discard_thread(void *data)  	struct f2fs_sb_info *sbi = data;  	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;  	wait_queue_head_t *q = &dcc->discard_wait_queue; +	unsigned int wait_ms = DEF_MIN_DISCARD_ISSUE_TIME; +	int issued;  	set_freezable();  	do { -		wait_event_interruptible(*q, kthread_should_stop() || -					freezing(current) || -					atomic_read(&dcc->discard_cmd_cnt)); +		wait_event_interruptible_timeout(*q, +				kthread_should_stop() || freezing(current) || +				dcc->discard_wake, +				msecs_to_jiffies(wait_ms));  		if (try_to_freeze())  			continue;  		if (kthread_should_stop())  			return 0; -		__issue_discard_cmd(sbi, true); -		__wait_discard_cmd(sbi, true); +		if (dcc->discard_wake) { +			dcc->discard_wake = 0; +			if (sbi->gc_thread && sbi->gc_thread->gc_urgent) +				mark_discard_range_all(sbi); +		} + +		sb_start_intwrite(sbi->sb); + +		issued = __issue_discard_cmd(sbi, true); +		if (issued) { +			__wait_discard_cmd(sbi, true); +			wait_ms = DEF_MIN_DISCARD_ISSUE_TIME; +		} else { +			wait_ms = DEF_MAX_DISCARD_ISSUE_TIME; +		} + +		sb_end_intwrite(sbi->sb); -		congestion_wait(BLK_RW_SYNC, HZ/50);  	} while (!kthread_should_stop());  	return 0;  } @@ -1320,7 +1456,8 @@ static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi)  void clear_prefree_segments(struct f2fs_sb_info *sbi, struct cp_control *cpc)  { -	struct list_head *head = &(SM_I(sbi)->dcc_info->entry_list); +	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; +	struct list_head *head = &dcc->entry_list;  	struct discard_entry *entry, *this;  	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  	unsigned long *prefree_map = dirty_i->dirty_segmap[PRE]; @@ -1402,11 +1539,11 @@ skip:  			goto find_next;  		list_del(&entry->list); -		SM_I(sbi)->dcc_info->nr_discards -= total_len; +		dcc->nr_discards -= total_len;  		kmem_cache_free(discard_entry_slab, entry);  	} -	wake_up(&SM_I(sbi)->dcc_info->discard_wait_queue); +	wake_up_discard_thread(sbi, false);  }  static int create_discard_cmd_control(struct f2fs_sb_info *sbi) @@ -1424,9 +1561,13 @@ static int create_discard_cmd_control(struct f2fs_sb_info *sbi)  	if (!dcc)  		return -ENOMEM; +	dcc->discard_granularity = DEFAULT_DISCARD_GRANULARITY;  	INIT_LIST_HEAD(&dcc->entry_list); -	for (i = 0; i < MAX_PLIST_NUM; i++) +	for (i = 0; i < MAX_PLIST_NUM; i++) {  		INIT_LIST_HEAD(&dcc->pend_list[i]); +		if (i >= dcc->discard_granularity - 1) +			dcc->pend_list_tag[i] |= P_ACTIVE; +	}  	INIT_LIST_HEAD(&dcc->wait_list);  	mutex_init(&dcc->cmd_lock);  	atomic_set(&dcc->issued_discard, 0); @@ -1491,6 +1632,10 @@ static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)  	struct seg_entry *se;  	unsigned int segno, offset;  	long int new_vblocks; +	bool exist; +#ifdef CONFIG_F2FS_CHECK_FS +	bool mir_exist; +#endif  	segno = GET_SEGNO(sbi, blkaddr); @@ -1507,17 +1652,25 @@ static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)  	/* Update valid block bitmap */  	if (del > 0) { -		if (f2fs_test_and_set_bit(offset, se->cur_valid_map)) { +		exist = f2fs_test_and_set_bit(offset, se->cur_valid_map);  #ifdef CONFIG_F2FS_CHECK_FS -			if (f2fs_test_and_set_bit(offset, -						se->cur_valid_map_mir)) -				f2fs_bug_on(sbi, 1); -			else -				WARN_ON(1); -#else +		mir_exist = f2fs_test_and_set_bit(offset, +						se->cur_valid_map_mir); +		if (unlikely(exist != mir_exist)) { +			f2fs_msg(sbi->sb, KERN_ERR, "Inconsistent error " +				"when setting bitmap, blk:%u, old bit:%d", +				blkaddr, exist);  			f2fs_bug_on(sbi, 1); +		}  #endif +		if (unlikely(exist)) { +			f2fs_msg(sbi->sb, KERN_ERR, +				"Bitmap was wrongly set, blk:%u", blkaddr); +			f2fs_bug_on(sbi, 1); +			se->valid_blocks--; +			del = 0;  		} +  		if (f2fs_discard_en(sbi) &&  			!f2fs_test_and_set_bit(offset, se->discard_map))  			sbi->discard_blks--; @@ -1528,17 +1681,25 @@ static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)  				se->ckpt_valid_blocks++;  		}  	} else { -		if (!f2fs_test_and_clear_bit(offset, se->cur_valid_map)) { +		exist = f2fs_test_and_clear_bit(offset, se->cur_valid_map);  #ifdef CONFIG_F2FS_CHECK_FS -			if (!f2fs_test_and_clear_bit(offset, -						se->cur_valid_map_mir)) -				f2fs_bug_on(sbi, 1); -			else -				WARN_ON(1); -#else +		mir_exist = f2fs_test_and_clear_bit(offset, +						se->cur_valid_map_mir); +		if (unlikely(exist != mir_exist)) { +			f2fs_msg(sbi->sb, KERN_ERR, "Inconsistent error " +				"when clearing bitmap, blk:%u, old bit:%d", +				blkaddr, exist);  			f2fs_bug_on(sbi, 1); +		}  #endif +		if (unlikely(!exist)) { +			f2fs_msg(sbi->sb, KERN_ERR, +				"Bitmap was wrongly cleared, blk:%u", blkaddr); +			f2fs_bug_on(sbi, 1); +			se->valid_blocks++; +			del = 0;  		} +  		if (f2fs_discard_en(sbi) &&  			f2fs_test_and_clear_bit(offset, se->discard_map))  			sbi->discard_blks++; @@ -1900,7 +2061,7 @@ static void __refresh_next_blkoff(struct f2fs_sb_info *sbi,   * This function always allocates a used segment(from dirty seglist) by SSR   * manner, so it should recover the existing segment information of valid blocks   */ -static void change_curseg(struct f2fs_sb_info *sbi, int type, bool reuse) +static void change_curseg(struct f2fs_sb_info *sbi, int type)  {  	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);  	struct curseg_info *curseg = CURSEG_I(sbi, type); @@ -1921,12 +2082,10 @@ static void change_curseg(struct f2fs_sb_info *sbi, int type, bool reuse)  	curseg->alloc_type = SSR;  	__next_free_blkoff(sbi, curseg, 0); -	if (reuse) { -		sum_page = get_sum_page(sbi, new_segno); -		sum_node = (struct f2fs_summary_block *)page_address(sum_page); -		memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE); -		f2fs_put_page(sum_page, 1); -	} +	sum_page = get_sum_page(sbi, new_segno); +	sum_node = (struct f2fs_summary_block *)page_address(sum_page); +	memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE); +	f2fs_put_page(sum_page, 1);  }  static int get_ssr_segment(struct f2fs_sb_info *sbi, int type) @@ -1990,7 +2149,7 @@ static void allocate_segment_by_default(struct f2fs_sb_info *sbi,  	else if (curseg->alloc_type == LFS && is_next_segment_free(sbi, type))  		new_curseg(sbi, type, false);  	else if (need_SSR(sbi) && get_ssr_segment(sbi, type)) -		change_curseg(sbi, type, true); +		change_curseg(sbi, type);  	else  		new_curseg(sbi, type, false); @@ -2083,6 +2242,9 @@ int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range)  		schedule();  	} +	/* It's time to issue all the filed discards */ +	mark_discard_range_all(sbi); +	f2fs_wait_discard_bios(sbi, false);  out:  	range->len = F2FS_BLK_TO_BYTES(cpc.trimmed);  	return err; @@ -2202,9 +2364,12 @@ void allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,  	mutex_unlock(&sit_i->sentry_lock); -	if (page && IS_NODESEG(type)) +	if (page && IS_NODESEG(type)) {  		fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg)); +		f2fs_inode_chksum_set(sbi, page); +	} +  	if (add_list) {  		struct f2fs_bio_info *io; @@ -2236,7 +2401,8 @@ reallocate:  	}  } -void write_meta_page(struct f2fs_sb_info *sbi, struct page *page) +void write_meta_page(struct f2fs_sb_info *sbi, struct page *page, +					enum iostat_type io_type)  {  	struct f2fs_io_info fio = {  		.sbi = sbi, @@ -2255,6 +2421,8 @@ void write_meta_page(struct f2fs_sb_info *sbi, struct page *page)  	set_page_writeback(page);  	f2fs_submit_page_write(&fio); + +	f2fs_update_iostat(sbi, io_type, F2FS_BLKSIZE);  }  void write_node_page(unsigned int nid, struct f2fs_io_info *fio) @@ -2263,6 +2431,8 @@ void write_node_page(unsigned int nid, struct f2fs_io_info *fio)  	set_summary(&sum, nid, 0, 0);  	do_write_page(&sum, fio); + +	f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE);  }  void write_data_page(struct dnode_of_data *dn, struct f2fs_io_info *fio) @@ -2276,13 +2446,22 @@ void write_data_page(struct dnode_of_data *dn, struct f2fs_io_info *fio)  	set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);  	do_write_page(&sum, fio);  	f2fs_update_data_blkaddr(dn, fio->new_blkaddr); + +	f2fs_update_iostat(sbi, fio->io_type, F2FS_BLKSIZE);  }  int rewrite_data_page(struct f2fs_io_info *fio)  { +	int err; +  	fio->new_blkaddr = fio->old_blkaddr;  	stat_inc_inplace_blocks(fio->sbi); -	return f2fs_submit_page_bio(fio); + +	err = f2fs_submit_page_bio(fio); + +	f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE); + +	return err;  }  void __f2fs_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, @@ -2324,7 +2503,7 @@ void __f2fs_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,  	/* change the current segment */  	if (segno != curseg->segno) {  		curseg->next_segno = segno; -		change_curseg(sbi, type, true); +		change_curseg(sbi, type);  	}  	curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr); @@ -2343,7 +2522,7 @@ void __f2fs_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,  	if (recover_curseg) {  		if (old_cursegno != curseg->segno) {  			curseg->next_segno = old_cursegno; -			change_curseg(sbi, type, true); +			change_curseg(sbi, type);  		}  		curseg->next_blkoff = old_blkoff;  	} @@ -2382,8 +2561,7 @@ void f2fs_wait_on_page_writeback(struct page *page,  	}  } -void f2fs_wait_on_encrypted_page_writeback(struct f2fs_sb_info *sbi, -							block_t blkaddr) +void f2fs_wait_on_block_writeback(struct f2fs_sb_info *sbi, block_t blkaddr)  {  	struct page *cpage;  |