Piteball/android_kernel_google_msm
1
0
Fork 0
mirror of https://github.com/followmsi/android_kernel_google_msm.git synced 2024-11-06 23:17:41 +00:00
Code Issues Projects Releases Wiki Activity

cma: redirect page allocation to CMA

Browse source 
CMA pages are designed to be used as fallback for movable allocations
and cannot be used for non-movable allocations. If CMA pages are
utilized poorly, non-movable allocations may end up getting starved if
all regular movable pages are allocated and the only pages left are
CMA. Always using CMA pages first creates unacceptable performance
problems. As a midway alternative, use CMA pages for certain
userspace allocations. The userspace pages can be migrated or dropped
quickly which giving decent utilization.

Change-Id: I6165dda01b705309eebabc6dfa67146b7a95c174
CRs-Fixed: 452508
[lauraa@codeaurora.org: Missing CONFIG_CMA guards, add commit text]
Signed-off-by: Laura Abbott <lauraa@codeaurora.org>
This commit is contained in:
Heesub Shin 2013-01-07 11:10:13 +09:00 committed by Iliyan Malchev
parent 2390206ce0
commit 262de8b688
4 changed files with 73 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
include/linux/gfp.h
View file

@ -23,6 +23,7 @@ struct vm_area_struct;
#define ___GFP_REPEAT 0x400u
#define ___GFP_NOFAIL 0x800u
#define ___GFP_NORETRY 0x1000u
#define ___GFP_CMA 0x2000u
#define ___GFP_COMP 0x4000u
#define ___GFP_ZERO 0x8000u
#define ___GFP_NOMEMALLOC 0x10000u
@ -51,7 +52,9 @@ struct vm_area_struct;
#define __GFP_HIGHMEM ((__force gfp_t)___GFP_HIGHMEM)
#define __GFP_DMA32 ((__force gfp_t)___GFP_DMA32)
#define __GFP_MOVABLE ((__force gfp_t)___GFP_MOVABLE) /* Page is movable */
#define GFP_ZONEMASK (__GFP_DMA|__GFP_HIGHMEM|__GFP_DMA32|__GFP_MOVABLE)
#define __GFP_CMA ((__force gfp_t)___GFP_CMA)
#define GFP_ZONEMASK (__GFP_DMA|__GFP_HIGHMEM|__GFP_DMA32|__GFP_MOVABLE| \
__GFP_CMA)
/*
* Action modifiers - doesn't change the zoning
*
@ -124,7 +127,7 @@ struct vm_area_struct;
#endif
/* This mask makes up all the page movable related flags */
#define GFP_MOVABLE_MASK (__GFP_RECLAIMABLE|__GFP_MOVABLE)
#define GFP_MOVABLE_MASK (__GFP_RECLAIMABLE|__GFP_MOVABLE|__GFP_CMA)
/* Control page allocator reclaim behavior */
#define GFP_RECLAIM_MASK (__GFP_WAIT|__GFP_HIGH|__GFP_IO|__GFP_FS|\
@ -157,8 +160,14 @@ static inline int allocflags_to_migratetype(gfp_t gfp_flags)
return MIGRATE_UNMOVABLE;
/* Group based on mobility */
#ifndef CONFIG_CMA
return (((gfp_flags & __GFP_MOVABLE) != 0) << 1) |
((gfp_flags & __GFP_RECLAIMABLE) != 0);
#else
return (((gfp_flags & __GFP_MOVABLE) != 0) << 1) |
(((gfp_flags & __GFP_CMA) != 0) << 1) |
((gfp_flags & __GFP_RECLAIMABLE) != 0);
#endif
}
#ifdef CONFIG_HIGHMEM

15
include/linux/highmem.h
View file

@ -211,9 +211,24 @@ static inline struct page *
alloc_zeroed_user_highpage_movable(struct vm_area_struct *vma,
unsigned long vaddr)
{
#ifndef CONFIG_CMA
return __alloc_zeroed_user_highpage(__GFP_MOVABLE, vma, vaddr);
#else
return __alloc_zeroed_user_highpage(__GFP_MOVABLE|__GFP_CMA, vma,
vaddr);
#endif
}
#ifdef CONFIG_CMA
static inline struct page *
alloc_zeroed_user_highpage_movable_cma(struct vm_area_struct *vma,
unsigned long vaddr)
{
return __alloc_zeroed_user_highpage(__GFP_MOVABLE|__GFP_CMA, vma,
vaddr);
}
#endif
static inline void clear_highpage(struct page *page)
{
void *kaddr = kmap_atomic(page);

1
include/linux/mmzone.h
View file

@ -390,6 +390,7 @@ struct zone {
* process to make sure that the system is not starved.
*/
unsigned long min_cma_pages;
bool cma_alloc;
#endif
struct free_area free_area[MAX_ORDER];

50
mm/page_alloc.c
View file

@ -1121,6 +1121,37 @@ retry_reserve:
return page;
}
static struct page *__rmqueue_cma(struct zone *zone, unsigned int order,
int migratetype)
{
struct page *page = 0;
#ifdef CONFIG_CMA
if (migratetype == MIGRATE_MOVABLE && !zone->cma_alloc)
page = __rmqueue_smallest(zone, order, MIGRATE_CMA);
else
#endif
retry_reserve :
page = __rmqueue_smallest(zone, order, migratetype);
if (unlikely(!page) && migratetype != MIGRATE_RESERVE) {
page = __rmqueue_fallback(zone, order, migratetype);
/*
* Use MIGRATE_RESERVE rather than fail an allocation. goto
* is used because __rmqueue_smallest is an inline function
* and we want just one call site
*/
if (!page) {
migratetype = MIGRATE_RESERVE;
goto retry_reserve;
}
}
trace_mm_page_alloc_zone_locked(page, order, migratetype);
return page;
}
/*
* Obtain a specified number of elements from the buddy allocator, all under
* a single hold of the lock, for efficiency. Add them to the supplied list.
@ -1128,13 +1159,17 @@ retry_reserve:
*/
static int rmqueue_bulk(struct zone *zone, unsigned int order,
unsigned long count, struct list_head *list,
int migratetype, int cold)
int migratetype, int cold, int cma)
{
int mt = migratetype, i;
spin_lock(&zone->lock);
for (i = 0; i < count; ++i) {
struct page *page = __rmqueue(zone, order, migratetype);
struct page *page;
if (cma)
page = __rmqueue_cma(zone, order, migratetype);
else
page = __rmqueue(zone, order, migratetype);
if (unlikely(page == NULL))
break;
@ -1481,7 +1516,8 @@ again:
if (list_empty(list)) {
pcp->count += rmqueue_bulk(zone, 0,
pcp->batch, list,
migratetype, cold);
migratetype, cold,
gfp_flags & __GFP_CMA);
if (unlikely(list_empty(list)))
goto failed;
}
@ -1508,7 +1544,10 @@ again:
WARN_ON_ONCE(order > 1);
}
spin_lock_irqsave(&zone->lock, flags);
page = __rmqueue(zone, order, migratetype);
if (gfp_flags & __GFP_CMA)
page = __rmqueue_cma(zone, order, migratetype);
else
page = __rmqueue(zone, order, migratetype);
spin_unlock(&zone->lock);
if (!page)
goto failed;
@ -5885,6 +5924,8 @@ int alloc_contig_range(unsigned long start, unsigned long end,
if (ret)
goto done;
zone->cma_alloc = 1;
ret = __alloc_contig_migrate_range(start, end);
if (ret)
goto done;
@ -5949,6 +5990,7 @@ int alloc_contig_range(unsigned long start, unsigned long end,
done:
undo_isolate_page_range(pfn_max_align_down(start),
pfn_max_align_up(end), migratetype);
zone->cma_alloc = 0;
return ret;
}