diff options
Diffstat (limited to 'mm')
| -rw-r--r-- | mm/filemap.c | 24 | ||||
| -rw-r--r-- | mm/hugetlb.c | 3 | ||||
| -rw-r--r-- | mm/page_alloc.c | 4 | ||||
| -rw-r--r-- | mm/sparse.c | 2 | ||||
| -rw-r--r-- | mm/vmalloc.c | 18 | ||||
| -rw-r--r-- | mm/vmscan.c | 63 | ||||
| -rw-r--r-- | mm/vmstat.c | 2 |
7 files changed, 68 insertions, 48 deletions
diff --git a/mm/filemap.c b/mm/filemap.c index cb26e33fd0ff..7b84dc814347 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -467,25 +467,15 @@ int add_to_page_cache_lru(struct page *page, struct address_space *mapping, } #ifdef CONFIG_NUMA -struct page *page_cache_alloc(struct address_space *x) +struct page *__page_cache_alloc(gfp_t gfp) { if (cpuset_do_page_mem_spread()) { int n = cpuset_mem_spread_node(); - return alloc_pages_node(n, mapping_gfp_mask(x), 0); + return alloc_pages_node(n, gfp, 0); } - return alloc_pages(mapping_gfp_mask(x), 0); + return alloc_pages(gfp, 0); } -EXPORT_SYMBOL(page_cache_alloc); - -struct page *page_cache_alloc_cold(struct address_space *x) -{ - if (cpuset_do_page_mem_spread()) { - int n = cpuset_mem_spread_node(); - return alloc_pages_node(n, mapping_gfp_mask(x)|__GFP_COLD, 0); - } - return alloc_pages(mapping_gfp_mask(x)|__GFP_COLD, 0); -} -EXPORT_SYMBOL(page_cache_alloc_cold); +EXPORT_SYMBOL(__page_cache_alloc); #endif static int __sleep_on_page_lock(void *word) @@ -826,7 +816,6 @@ struct page * grab_cache_page_nowait(struct address_space *mapping, unsigned long index) { struct page *page = find_get_page(mapping, index); - gfp_t gfp_mask; if (page) { if (!TestSetPageLocked(page)) @@ -834,9 +823,8 @@ grab_cache_page_nowait(struct address_space *mapping, unsigned long index) page_cache_release(page); return NULL; } - gfp_mask = mapping_gfp_mask(mapping) & ~__GFP_FS; - page = alloc_pages(gfp_mask, 0); - if (page && add_to_page_cache_lru(page, mapping, index, gfp_mask)) { + page = __page_cache_alloc(mapping_gfp_mask(mapping) & ~__GFP_FS); + if (page && add_to_page_cache_lru(page, mapping, index, GFP_KERNEL)) { page_cache_release(page); page = NULL; } diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 2dbec90dc3ba..a088f593a807 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -478,6 +478,9 @@ int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma, retry: page = find_lock_page(mapping, idx); if (!page) { + size = i_size_read(mapping->host) >> HPAGE_SHIFT; + if (idx >= size) + goto out; if (hugetlb_get_quota(mapping)) goto out; page = alloc_huge_page(vma, address); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index f5fc45472d5c..b55bb358b832 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -2261,7 +2261,7 @@ unsigned long __init __absent_pages_in_range(int nid, /* Account for ranges past physical memory on this node */ if (range_end_pfn > prev_end_pfn) - hole_pages = range_end_pfn - + hole_pages += range_end_pfn - max(range_start_pfn, prev_end_pfn); return hole_pages; @@ -2407,7 +2407,7 @@ static void __meminit free_area_init_core(struct pglist_data *pgdat, zone->zone_pgdat = pgdat; zone->free_pages = 0; - zone->temp_priority = zone->prev_priority = DEF_PRIORITY; + zone->prev_priority = DEF_PRIORITY; zone_pcp_init(zone); INIT_LIST_HEAD(&zone->active_list); diff --git a/mm/sparse.c b/mm/sparse.c index 86c52ab80878..b3c82ba30012 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -211,7 +211,7 @@ static struct page *__kmalloc_section_memmap(unsigned long nr_pages) struct page *page, *ret; unsigned long memmap_size = sizeof(struct page) * nr_pages; - page = alloc_pages(GFP_KERNEL, get_order(memmap_size)); + page = alloc_pages(GFP_KERNEL|__GFP_NOWARN, get_order(memmap_size)); if (page) goto got_map_page; diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 1133dd3aafcf..46606c133e82 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -160,13 +160,15 @@ int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page ***pages) return err; } -struct vm_struct *__get_vm_area_node(unsigned long size, unsigned long flags, - unsigned long start, unsigned long end, int node) +static struct vm_struct *__get_vm_area_node(unsigned long size, unsigned long flags, + unsigned long start, unsigned long end, + int node, gfp_t gfp_mask) { struct vm_struct **p, *tmp, *area; unsigned long align = 1; unsigned long addr; + BUG_ON(in_interrupt()); if (flags & VM_IOREMAP) { int bit = fls(size); @@ -180,7 +182,7 @@ struct vm_struct *__get_vm_area_node(unsigned long size, unsigned long flags, addr = ALIGN(start, align); size = PAGE_ALIGN(size); - area = kmalloc_node(sizeof(*area), GFP_KERNEL, node); + area = kmalloc_node(sizeof(*area), gfp_mask & GFP_LEVEL_MASK, node); if (unlikely(!area)) return NULL; @@ -236,7 +238,7 @@ out: struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags, unsigned long start, unsigned long end) { - return __get_vm_area_node(size, flags, start, end, -1); + return __get_vm_area_node(size, flags, start, end, -1, GFP_KERNEL); } /** @@ -253,9 +255,11 @@ struct vm_struct *get_vm_area(unsigned long size, unsigned long flags) return __get_vm_area(size, flags, VMALLOC_START, VMALLOC_END); } -struct vm_struct *get_vm_area_node(unsigned long size, unsigned long flags, int node) +struct vm_struct *get_vm_area_node(unsigned long size, unsigned long flags, + int node, gfp_t gfp_mask) { - return __get_vm_area_node(size, flags, VMALLOC_START, VMALLOC_END, node); + return __get_vm_area_node(size, flags, VMALLOC_START, VMALLOC_END, node, + gfp_mask); } /* Caller must hold vmlist_lock */ @@ -487,7 +491,7 @@ static void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot, if (!size || (size >> PAGE_SHIFT) > num_physpages) return NULL; - area = get_vm_area_node(size, VM_ALLOC, node); + area = get_vm_area_node(size, VM_ALLOC, node, gfp_mask); if (!area) return NULL; diff --git a/mm/vmscan.c b/mm/vmscan.c index f05527bf792b..518540a4a2a6 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -723,6 +723,20 @@ done: return nr_reclaimed; } +/* + * We are about to scan this zone at a certain priority level. If that priority + * level is smaller (ie: more urgent) than the previous priority, then note + * that priority level within the zone. This is done so that when the next + * process comes in to scan this zone, it will immediately start out at this + * priority level rather than having to build up its own scanning priority. + * Here, this priority affects only the reclaim-mapped threshold. + */ +static inline void note_zone_scanning_priority(struct zone *zone, int priority) +{ + if (priority < zone->prev_priority) + zone->prev_priority = priority; +} + static inline int zone_is_near_oom(struct zone *zone) { return zone->pages_scanned >= (zone->nr_active + zone->nr_inactive)*3; @@ -746,7 +760,7 @@ static inline int zone_is_near_oom(struct zone *zone) * But we had to alter page->flags anyway. */ static void shrink_active_list(unsigned long nr_pages, struct zone *zone, - struct scan_control *sc) + struct scan_control *sc, int priority) { unsigned long pgmoved; int pgdeactivate = 0; @@ -770,7 +784,7 @@ static void shrink_active_list(unsigned long nr_pages, struct zone *zone, * `distress' is a measure of how much trouble we're having * reclaiming pages. 0 -> no problems. 100 -> great trouble. */ - distress = 100 >> zone->prev_priority; + distress = 100 >> min(zone->prev_priority, priority); /* * The point of this algorithm is to decide when to start @@ -922,7 +936,7 @@ static unsigned long shrink_zone(int priority, struct zone *zone, nr_to_scan = min(nr_active, (unsigned long)sc->swap_cluster_max); nr_active -= nr_to_scan; - shrink_active_list(nr_to_scan, zone, sc); + shrink_active_list(nr_to_scan, zone, sc, priority); } if (nr_inactive) { @@ -972,9 +986,7 @@ static unsigned long shrink_zones(int priority, struct zone **zones, if (!cpuset_zone_allowed(zone, __GFP_HARDWALL)) continue; - zone->temp_priority = priority; - if (zone->prev_priority > priority) - zone->prev_priority = priority; + note_zone_scanning_priority(zone, priority); if (zone->all_unreclaimable && priority != DEF_PRIORITY) continue; /* Let kswapd poll it */ @@ -1024,7 +1036,6 @@ unsigned long try_to_free_pages(struct zone **zones, gfp_t gfp_mask) if (!cpuset_zone_allowed(zone, __GFP_HARDWALL)) continue; - zone->temp_priority = DEF_PRIORITY; lru_pages += zone->nr_active + zone->nr_inactive; } @@ -1065,13 +1076,22 @@ unsigned long try_to_free_pages(struct zone **zones, gfp_t gfp_mask) if (!sc.all_unreclaimable) ret = 1; out: + /* + * Now that we've scanned all the zones at this priority level, note + * that level within the zone so that the next thread which performs + * scanning of this zone will immediately start out at this priority + * level. This affects only the decision whether or not to bring + * mapped pages onto the inactive list. + */ + if (priority < 0) + priority = 0; for (i = 0; zones[i] != 0; i++) { struct zone *zone = zones[i]; if (!cpuset_zone_allowed(zone, __GFP_HARDWALL)) continue; - zone->prev_priority = zone->temp_priority; + zone->prev_priority = priority; } return ret; } @@ -1111,6 +1131,11 @@ static unsigned long balance_pgdat(pg_data_t *pgdat, int order) .swap_cluster_max = SWAP_CLUSTER_MAX, .swappiness = vm_swappiness, }; + /* + * temp_priority is used to remember the scanning priority at which + * this zone was successfully refilled to free_pages == pages_high. + */ + int temp_priority[MAX_NR_ZONES]; loop_again: total_scanned = 0; @@ -1118,11 +1143,8 @@ loop_again: sc.may_writepage = !laptop_mode; count_vm_event(PAGEOUTRUN); - for (i = 0; i < pgdat->nr_zones; i++) { - struct zone *zone = pgdat->node_zones + i; - - zone->temp_priority = DEF_PRIORITY; - } + for (i = 0; i < pgdat->nr_zones; i++) + temp_priority[i] = DEF_PRIORITY; for (priority = DEF_PRIORITY; priority >= 0; priority--) { int end_zone = 0; /* Inclusive. 0 = ZONE_DMA */ @@ -1183,10 +1205,9 @@ scan: if (!zone_watermark_ok(zone, order, zone->pages_high, end_zone, 0)) all_zones_ok = 0; - zone->temp_priority = priority; - if (zone->prev_priority > priority) - zone->prev_priority = priority; + temp_priority[i] = priority; sc.nr_scanned = 0; + note_zone_scanning_priority(zone, priority); nr_reclaimed += shrink_zone(priority, zone, &sc); reclaim_state->reclaimed_slab = 0; nr_slab = shrink_slab(sc.nr_scanned, GFP_KERNEL, @@ -1226,10 +1247,15 @@ scan: break; } out: + /* + * Note within each zone the priority level at which this zone was + * brought into a happy state. So that the next thread which scans this + * zone will start out at that priority level. + */ for (i = 0; i < pgdat->nr_zones; i++) { struct zone *zone = pgdat->node_zones + i; - zone->prev_priority = zone->temp_priority; + zone->prev_priority = temp_priority[i]; } if (!all_zones_ok) { cond_resched(); @@ -1358,7 +1384,7 @@ static unsigned long shrink_all_zones(unsigned long nr_pages, int pass, if (zone->nr_scan_active >= nr_pages || pass > 3) { zone->nr_scan_active = 0; nr_to_scan = min(nr_pages, zone->nr_active); - shrink_active_list(nr_to_scan, zone, sc); + shrink_active_list(nr_to_scan, zone, sc, prio); } } @@ -1614,6 +1640,7 @@ static int __zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order) */ priority = ZONE_RECLAIM_PRIORITY; do { + note_zone_scanning_priority(zone, priority); nr_reclaimed += shrink_zone(priority, zone, &sc); priority--; } while (priority >= 0 && nr_reclaimed < nr_pages); diff --git a/mm/vmstat.c b/mm/vmstat.c index 45b124e012f5..8614e8f6743b 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -587,11 +587,9 @@ static int zoneinfo_show(struct seq_file *m, void *arg) seq_printf(m, "\n all_unreclaimable: %u" "\n prev_priority: %i" - "\n temp_priority: %i" "\n start_pfn: %lu", zone->all_unreclaimable, zone->prev_priority, - zone->temp_priority, zone->zone_start_pfn); spin_unlock_irqrestore(&zone->lock, flags); seq_putc(m, '\n'); |