diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index ee16ba08cf51..babc344d6259 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -99,6 +99,7 @@ static inline int get_pageblock_migratetype(struct page *page) struct free_area { struct list_head free_list[MIGRATE_TYPES]; unsigned long nr_free; + unsigned long nr_free_cma; }; struct pglist_data; diff --git a/include/linux/page-isolation.h b/include/linux/page-isolation.h index 2dc1e1697b45..9b57ec668f96 100644 --- a/include/linux/page-isolation.h +++ b/include/linux/page-isolation.h @@ -33,10 +33,10 @@ bool has_unmovable_pages(struct zone *zone, struct page *page, int count, bool skip_hwpoisoned_pages); void set_pageblock_migratetype(struct page *page, int migratetype); int move_freepages_block(struct zone *zone, struct page *page, - int migratetype); + int migratetype, int old_mt); int move_freepages(struct zone *zone, struct page *start_page, struct page *end_page, - int migratetype); + int migratetype, int old_mt); /* * Changes migrate type in [start_pfn, end_pfn) to be MIGRATE_ISOLATE. diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 2d343a8c546e..650cfbe39ea2 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -595,6 +595,8 @@ continue_merging: } else { list_del(&buddy->lru); zone->free_area[order].nr_free--; + if (is_migrate_cma(migratetype)) + zone->free_area[order].nr_free_cma--; rmv_page_order(buddy); } combined_idx = buddy_idx & page_idx; @@ -655,6 +657,8 @@ done_merging: list_add(&page->lru, &zone->free_area[order].free_list[migratetype]); out: zone->free_area[order].nr_free++; + if (is_migrate_cma(migratetype)) + zone->free_area[order].nr_free_cma++; } static inline int free_pages_check(struct page *page) @@ -897,6 +901,8 @@ static inline void expand(struct zone *zone, struct page *page, #endif list_add(&page[size].lru, &area->free_list[migratetype]); area->nr_free++; + if (is_migrate_cma(migratetype)) + area->nr_free_cma++; set_page_order(&page[size], high); } } @@ -966,6 +972,8 @@ struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, list_del(&page->lru); rmv_page_order(page); area->nr_free--; + if (is_migrate_cma(migratetype)) + area->nr_free_cma--; expand(zone, page, order, current_order, area, migratetype); return page; } @@ -1005,7 +1013,7 @@ int *get_migratetype_fallbacks(int mtype) */ int move_freepages(struct zone *zone, struct page *start_page, struct page *end_page, - int migratetype) + int migratetype, int old_mt) { struct page *page; unsigned long order; @@ -1039,6 +1047,12 @@ int move_freepages(struct zone *zone, order = page_order(page); list_move(&page->lru, &zone->free_area[order].free_list[migratetype]); + + if (is_migrate_cma(migratetype)) + zone->free_area[order].nr_free_cma++; + else if (is_migrate_cma(old_mt)) + zone->free_area[order].nr_free_cma--; + set_freepage_migratetype(page, migratetype); page += 1 << order; pages_moved += 1 << order; @@ -1048,7 +1062,7 @@ int move_freepages(struct zone *zone, } int move_freepages_block(struct zone *zone, struct page *page, - int migratetype) + int migratetype, int old_mt) { unsigned long start_pfn, end_pfn; struct page *start_page, *end_page; @@ -1065,7 +1079,7 @@ int move_freepages_block(struct zone *zone, struct page *page, if (!zone_spans_pfn(zone, end_pfn)) return 0; - return move_freepages(zone, start_page, end_page, migratetype); + return move_freepages(zone, start_page, end_page, migratetype, old_mt); } static void change_pageblock_range(struct page *pageblock_page, @@ -1105,6 +1119,8 @@ __rmqueue_fallback(struct zone *zone, int order, int start_migratetype) page = list_entry(area->free_list[migratetype].next, struct page, lru); area->nr_free--; + if (is_migrate_cma(migratetype)) + area->nr_free_cma--; /* * If breaking a large block of pages, move all free @@ -1126,7 +1142,7 @@ __rmqueue_fallback(struct zone *zone, int order, int start_migratetype) page_group_by_mobility_disabled)) { int pages; pages = move_freepages_block(zone, page, - start_migratetype); + start_migratetype,0); /* Claim the whole block if over half of it is free */ if (pages >= (1 << (pageblock_order-1)) || @@ -1543,6 +1559,8 @@ int __isolate_free_page(struct page *page, unsigned int order) /* Remove page from free list */ list_del(&page->lru); zone->free_area[order].nr_free--; + if (is_migrate_cma(mt)) + zone->free_area[order].nr_free_cma--; rmv_page_order(page); /* Set the pageblock if the isolated page is at least a pageblock */ @@ -4084,7 +4102,7 @@ static void setup_zone_migrate_reserve(struct zone *zone) set_pageblock_migratetype(page, MIGRATE_RESERVE); move_freepages_block(zone, page, - MIGRATE_RESERVE); + MIGRATE_RESERVE, 0); reserve--; continue; } @@ -4096,7 +4114,8 @@ static void setup_zone_migrate_reserve(struct zone *zone) */ if (block_migratetype == MIGRATE_RESERVE) { set_pageblock_migratetype(page, MIGRATE_MOVABLE); - move_freepages_block(zone, page, MIGRATE_MOVABLE); + move_freepages_block(zone, page, + MIGRATE_MOVABLE, 0); } } } @@ -4174,6 +4193,7 @@ static void __meminit zone_init_free_lists(struct zone *zone) for_each_migratetype_order(order, t) { INIT_LIST_HEAD(&zone->free_area[order].free_list[t]); zone->free_area[order].nr_free = 0; + zone->free_area[order].nr_free_cma = 0; } } @@ -6445,6 +6465,8 @@ __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) if (PageHighMem(page)) totalhigh_pages -= 1 << order; #endif + if (is_migrate_cma(get_pageblock_migratetype(page))) + zone->free_area[order].nr_free_cma--; for (i = 0; i < (1 << order); i++) SetPageReserved((page+i)); pfn += (1 << order); diff --git a/mm/page_isolation.c b/mm/page_isolation.c index bba78e07d16e..daa6a07de976 100644 --- a/mm/page_isolation.c +++ b/mm/page_isolation.c @@ -60,7 +60,8 @@ out: set_pageblock_migratetype(page, MIGRATE_ISOLATE); zone->nr_isolate_pageblock++; - nr_pages = move_freepages_block(zone, page, MIGRATE_ISOLATE); + nr_pages = move_freepages_block(zone, page, + MIGRATE_ISOLATE, migratetype); __mod_zone_freepage_state(zone, -nr_pages, migratetype); } @@ -115,7 +116,8 @@ void unset_migratetype_isolate(struct page *page, unsigned migratetype) * pageblock scanning for freepage moving. */ if (!isolated_page) { - nr_pages = move_freepages_block(zone, page, migratetype); + nr_pages = move_freepages_block(zone, page, + migratetype, 0); __mod_zone_freepage_state(zone, nr_pages, migratetype); } set_pageblock_migratetype(page, migratetype); @@ -234,7 +236,7 @@ __test_page_isolated_in_pageblock(unsigned long pfn, unsigned long end_pfn, end_page = page + (1 << page_order(page)) - 1; move_freepages(page_zone(page), page, end_page, - MIGRATE_ISOLATE); + MIGRATE_ISOLATE,0); } pfn += 1 << page_order(page); }