#ifdef CONFIG_AMLOGIC_USER_FAULT
extern void show_all_pfn(struct task_struct *task, struct pt_regs *regs);
+extern void show_vma(struct mm_struct *mm, unsigned long addr);
#else
static inline void show_all_pfn(struct task_struct *task, struct pt_regs *regs)
{
}
+static inline void show_vma(struct mm_struct *mm, unsigned long addr)
+{
+}
#endif /* CONFIG_AMLOGIC_USER_FAULT */
#define UDBG_UNDEFINED (1 << 0)
extern void show_pte(struct mm_struct *mm, unsigned long addr);
#ifdef CONFIG_AMLOGIC_USER_FAULT
extern void show_all_pfn(struct task_struct *task, struct pt_regs *regs);
+extern void show_vma(struct mm_struct *mm, unsigned long addr);
#else
static inline void show_all_pfn(struct task_struct *task, struct pt_regs *regs)
{
}
+static inline void show_vma(struct mm_struct *mm, unsigned long addr)
+{
+}
#endif /* CONFIG_AMLOGIC_USER_FAULT */
extern void __show_regs(struct pt_regs *);
set_fs(fs);
}
-static void show_vma(struct mm_struct *mm, unsigned long addr)
+void show_vma(struct mm_struct *mm, unsigned long addr)
{
struct vm_area_struct *vma;
struct file *file;
#include <linux/compiler.h>
#include <linux/mm.h>
#include <linux/kernel.h>
+#include <linux/rmap.h>
#include <linux/kthread.h>
#include <linux/sched/rt.h>
#include <linux/completion.h>
#include <linux/amlogic/aml_cma.h>
#include <linux/hugetlb.h>
#include <linux/proc_fs.h>
+#include <asm/system_misc.h>
#include <trace/events/page_isolation.h>
#ifdef CONFIG_AMLOGIC_PAGE_TRACE
#include <linux/amlogic/page_trace.h>
int **resultp)
{
gfp_t gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_BDEV;
+ struct page *new = NULL;
+#ifdef CONFIG_AMLOGIC_PAGE_TRACE
+ struct page_trace *old_trace, *new_trace;
+#endif
/*
* TODO: allocate a destination hugepage from a nearest neighbor node,
* accordance with memory policy of the user process if possible. For
* now as a simple work-around, we use the next node for destination.
*/
- if (PageHuge(page))
- return alloc_huge_page_node(page_hstate(compound_head(page)),
+ if (PageHuge(page)) {
+ new = alloc_huge_page_node(page_hstate(compound_head(page)),
next_node_in(page_to_nid(page),
node_online_map));
+ #ifdef CONFIG_AMLOGIC_PAGE_TRACE
+ if (new) {
+ old_trace = find_page_base(page);
+ new_trace = find_page_base(new);
+ *new_trace = *old_trace;
+ }
+ #endif
+ return new;
+ }
if (PageHighMem(page))
gfp_mask |= __GFP_HIGHMEM;
- return alloc_page(gfp_mask);
+ new = alloc_page(gfp_mask);
+#ifdef CONFIG_AMLOGIC_PAGE_TRACE
+ if (new) {
+ old_trace = find_page_base(page);
+ new_trace = find_page_base(new);
+ *new_trace = *old_trace;
+ }
+#endif
+ return new;
}
/* [start, end) must belong to a single zone. */
}
EXPORT_SYMBOL(aml_cma_free);
+static int cma_vma_show(struct page *page, struct vm_area_struct *vma,
+ unsigned long addr, void *arg)
+{
+#ifdef CONFIG_AMLOGIC_USER_FAULT
+ struct mm_struct *mm = vma->vm_mm;
+
+ show_vma(mm, addr);
+#endif
+ return SWAP_AGAIN;
+}
+
+void rmap_walk_vma(struct page *page)
+{
+ struct rmap_walk_control rwc = {
+ .rmap_one = cma_vma_show,
+ };
+
+ pr_info("%s, show map for page:%lx,f:%lx, m:%p, p:%d\n",
+ __func__, page_to_pfn(page), page->flags,
+ page->mapping, page_count(page));
+ if (!page_mapping(page))
+ return;
+ rmap_walk(page, &rwc);
+}
+
void show_page(struct page *page)
{
unsigned long trace = 0;
page->flags & 0xffffffff,
page_mapcount(page), page_count(page),
(void *)trace);
+ if (cma_debug_level > 4)
+ rmap_walk_vma(page);
}
static int cma_debug_show(struct seq_file *m, void *arg)
PG_young,
PG_idle,
#endif
+#ifdef CONFIG_AMLOGIC_CMA
+ PG_cma_allocating, /* indicate page is under cma allocating */
+#endif
__NR_PAGEFLAGS,
/* Filesystems */
__PAGEFLAG(Slab, slab, PF_NO_TAIL)
__PAGEFLAG(SlobFree, slob_free, PF_NO_TAIL)
PAGEFLAG(Checked, checked, PF_NO_COMPOUND) /* Used by some filesystems */
+#ifdef CONFIG_AMLOGIC_CMA
+PAGEFLAG(CmaAllocating, cma_allocating, PF_ANY)
+#endif
/* Xen */
PAGEFLAG(Pinned, pinned, PF_NO_COMPOUND)
* Flags checked when a page is freed. Pages being freed should not have
* these flags set. It they are, there is a problem.
*/
+#ifdef CONFIG_AMLOGIC_CMA
+#define PAGE_FLAGS_CHECK_AT_FREE \
+ (1UL << PG_lru | 1UL << PG_locked | \
+ 1UL << PG_private | 1UL << PG_private_2 | \
+ 1UL << PG_writeback | 1UL << PG_reserved | \
+ 1UL << PG_cma_allocating | \
+ 1UL << PG_slab | 1UL << PG_swapcache | 1UL << PG_active | \
+ 1UL << PG_unevictable | __PG_MLOCKED)
+#else
#define PAGE_FLAGS_CHECK_AT_FREE \
(1UL << PG_lru | 1UL << PG_locked | \
1UL << PG_private | 1UL << PG_private_2 | \
1UL << PG_writeback | 1UL << PG_reserved | \
1UL << PG_slab | 1UL << PG_swapcache | 1UL << PG_active | \
1UL << PG_unevictable | __PG_MLOCKED)
+#endif
/*
* Flags checked when a page is prepped for return by the page allocator.
#else
#define IF_HAVE_PG_IDLE(flag,string)
#endif
+#ifdef CONFIG_AMLOGIC_CMA
+#define PG_CMA_ALLOCATING(flag, string) ,{1UL << flag, string}
+#else
+#define PG_CMA_ALLOCATING(flag, string)
+#endif
#define __def_pageflag_names \
{1UL << PG_locked, "locked" }, \
{1UL << PG_reclaim, "reclaim" }, \
{1UL << PG_swapbacked, "swapbacked" }, \
{1UL << PG_unevictable, "unevictable" } \
+PG_CMA_ALLOCATING(PG_cma_allocating, "cma_allocating") \
IF_HAVE_PG_MLOCK(PG_mlocked, "mlocked" ) \
IF_HAVE_PG_UNCACHED(PG_uncached, "uncached" ) \
IF_HAVE_PG_HWPOISON(PG_hwpoison, "hwpoison" ) \
#ifdef CONFIG_AMLOGIC_CMA
#include <asm/pgtable.h>
#include <linux/amlogic/aml_cma.h>
+#include <linux/delay.h>
#include <linux/amlogic/secmon.h>
#endif /* CONFIG_AMLOGIC_CMA */
#ifdef CONFIG_AMLOGIC_CMA
int dummy;
unsigned long long tick;
+ unsigned long long in_tick, timeout;
+
+ in_tick = sched_clock();
#endif /* CONFIG_AMLOGIC_CMA */
if (!cma || !cma->count)
tick = sched_clock();
cma_debug(0, NULL, "(cma %p, count %zu, align %d)\n",
(void *)cma, count, align);
+ in_tick = sched_clock();
+ timeout = 2ULL * 1000000 * (1 + ((count * PAGE_SIZE) >> 20));
#endif
if (!count)
return NULL;
#ifndef CONFIG_AMLOGIC_CMA
/* try again with a bit different memory target */
start = bitmap_no + mask + 1;
+ #else
+ /*
+ * CMA allocation time out, may blocked on some pages
+ * relax CPU and try later
+ */
+ if ((sched_clock() - in_tick) >= timeout)
+ usleep_range(1000, 2000);
#endif /* CONFIG_AMLOGIC_CMA */
}
#include <linux/freezer.h>
#include <linux/page_owner.h>
#include "internal.h"
+#ifdef CONFIG_AMLOGIC_PAGE_TRACE
+#include <linux/amlogic/page_trace.h>
+#endif
#ifdef CONFIG_COMPACTION
static inline void count_compact_event(enum vm_event_item item)
isolate_success:
list_add(&page->lru, &cc->migratepages);
+ #ifdef CONFIG_AMLOGIC_CMA
+ if (cc->page_type == COMPACT_CMA)
+ SetPageCmaAllocating(page);
+ #endif
cc->nr_migratepages++;
nr_isolated++;
{
struct compact_control *cc = (struct compact_control *)data;
struct page *freepage;
+#ifdef CONFIG_AMLOGIC_PAGE_TRACE
+ struct page_trace *old_trace, *new_trace;
+#endif
/*
* Isolate free pages if necessary, and if we are not aborting due to
freepage = list_entry(cc->freepages.next, struct page, lru);
list_del(&freepage->lru);
cc->nr_freepages--;
+#ifdef CONFIG_AMLOGIC_PAGE_TRACE
+ if (freepage) {
+ old_trace = find_page_base(migratepage);
+ new_trace = find_page_base(freepage);
+ *new_trace = *old_trace;
+ }
+#endif
return freepage;
}
tree_rmap_item =
unstable_tree_search_insert(rmap_item, page, &tree_page);
+#ifdef CONFIG_AMLOGIC_CMA
+ /*
+ * Now page is inserted to unstable tree, but do not
+ * let cma page to be kpage, it can be merged with other pages
+ */
+ if (cma_page(page)) {
+ if (tree_rmap_item)
+ put_page(tree_page);
+ return;
+ }
+#endif /* CONFIG_AMLOGIC_CMA */
if (tree_rmap_item) {
bool split;
goto oom;
if (is_zero_pfn(pte_pfn(orig_pte))) {
+ #ifdef CONFIG_AMLOGIC_CMA
+ gfp_t tmp_flag = __GFP_MOVABLE | __GFP_BDEV;
+
+ new_page = __alloc_zeroed_user_highpage(tmp_flag, vma,
+ fe->address);
+ #else
new_page = alloc_zeroed_user_highpage_movable(vma, fe->address);
+ #endif
if (!new_page)
goto oom;
} else {
+ #ifdef CONFIG_AMLOGIC_CMA
+ new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE | __GFP_BDEV,
+ vma, fe->address);
+ #else
new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma,
fe->address);
+ #endif
if (!new_page)
goto oom;
cow_user_page(new_page, old_page, fe->address, vma);
struct page *page2;
list_for_each_entry_safe(page, page2, l, lru) {
+ #ifdef CONFIG_AMLOGIC_CMA
+ if (PageCmaAllocating(page)) /* migrate/reclaim failed */
+ ClearPageCmaAllocating(page);
+ #endif
if (unlikely(PageHuge(page))) {
putback_active_hugepage(page);
continue;
pte_t pte;
swp_entry_t entry;
struct page *page;
+#ifdef CONFIG_AMLOGIC_CMA
+ bool need_wait = 0;
+#endif
spin_lock(ptl);
pte = *ptep;
goto out;
page = migration_entry_to_page(entry);
+#ifdef CONFIG_AMLOGIC_CMA
+ /* This page is under cma allocating, do not increase it ref */
+ if (PageCmaAllocating(page)) {
+ pr_debug("%s, Page:%lx, flags:%lx, m:%d, c:%d, map:%p\n",
+ __func__, page_to_pfn(page), page->flags,
+ page_mapcount(page), page_count(page),
+ page->mapping);
+ need_wait = 1;
+ goto out;
+ }
+#endif
/*
* Once radix-tree replacement of page migration started, page_count
return;
out:
pte_unmap_unlock(ptep, ptl);
+#ifdef CONFIG_AMLOGIC_CMA
+ if (need_wait)
+ schedule_timeout(1);
+#endif
}
void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
else
*result = page_to_nid(newpage);
}
+#ifdef CONFIG_AMLOGIC_CMA
+ if (reason == MR_CMA && rc == MIGRATEPAGE_SUCCESS)
+ ClearPageCmaAllocating(page);
+#endif
return rc;
}
*/
#ifdef CONFIG_AMLOGIC_CMA
static struct page *__rmqueue(struct zone *zone, unsigned int order,
- int migratetype, gfp_t gfp_flags)
+ int migratetype, bool cma)
#else
static struct page *__rmqueue(struct zone *zone, unsigned int order,
int migratetype)
#ifdef CONFIG_AMLOGIC_CMA
/* use CMA first */
- if (migratetype == MIGRATE_MOVABLE && can_use_cma(gfp_flags)) {
+ if (migratetype == MIGRATE_MOVABLE && cma) {
page = __rmqueue_cma_fallback(zone, order);
if (page) {
trace_mm_page_alloc_zone_locked(page, order,
#ifdef CONFIG_AMLOGIC_CMA
static int rmqueue_bulk(struct zone *zone, unsigned int order,
unsigned long count, struct list_head *list,
- int migratetype, bool cold, gfp_t flags)
+ int migratetype, bool cold, bool cma)
#else
static int rmqueue_bulk(struct zone *zone, unsigned int order,
unsigned long count, struct list_head *list,
spin_lock(&zone->lock);
for (i = 0; i < count; ++i) {
#ifdef CONFIG_AMLOGIC_CMA
- struct page *page = __rmqueue(zone, order, migratetype, flags);
+ struct page *page = __rmqueue(zone, order, migratetype, cma);
#else
struct page *page = __rmqueue(zone, order, migratetype);
#endif /* CONFIG_AMLOGIC_CMA */
unsigned long flags;
struct page *page;
bool cold = ((gfp_flags & __GFP_COLD) != 0);
+#ifdef CONFIG_AMLOGIC_CMA
+ bool cma = can_use_cma(gfp_flags);
+#endif
if (likely(order == 0)) {
struct per_cpu_pages *pcp;
pcp->count += rmqueue_bulk(zone, 0,
pcp->batch, list,
migratetype, cold,
- gfp_flags);
+ cma);
#else
pcp->count += rmqueue_bulk(zone, 0,
pcp->batch, list,
* For 2, we should replace with a cma page
* before page is deleted from PCP list.
*/
- if (!can_use_cma(gfp_flags) &&
- is_migrate_cma_page(page)) {
+ if (!cma && is_migrate_cma_page(page)) {
/* case 1 */
page = rmqueue_no_cma(zone, order, migratetype);
if (page)
goto failed;
} else if ((migratetype == MIGRATE_MOVABLE) &&
(get_pcppage_migratetype(page) != MIGRATE_CMA) &&
- can_use_cma(gfp_flags)) {
+ cma) {
struct page *tmp_page;
spin_lock(&zone->lock);
if (!page)
#ifdef CONFIG_AMLOGIC_CMA
page = __rmqueue(zone, order,
- migratetype, gfp_flags);
+ migratetype, cma);
#else
page = __rmqueue(zone, order, migratetype);
#endif /* CONFIG_AMLOGIC_CMA */
struct page *page;
shmem_pseudo_vma_init(&pvma, info, index);
+#ifdef CONFIG_AMLOGIC_CMA
+ page = swapin_readahead(swap, gfp | __GFP_BDEV, &pvma, 0);
+#else
page = swapin_readahead(swap, gfp, &pvma, 0);
+#endif
shmem_pseudo_vma_destroy(&pvma);
return page;
struct page *page;
shmem_pseudo_vma_init(&pvma, info, index);
+#ifdef CONFIG_AMLOGIC_CMA
+ page = alloc_page_vma(gfp | __GFP_BDEV, &pvma, 0);
+#else
page = alloc_page_vma(gfp, &pvma, 0);
+#endif
shmem_pseudo_vma_destroy(&pvma);
return page;
* Get a new page to read into from swap.
*/
if (!new_page) {
+ #ifdef CONFIG_AMLOGIC_CMA
+ new_page = alloc_page_vma(gfp_mask | __GFP_BDEV,
+ vma, addr);
+ #else
new_page = alloc_page_vma(gfp_mask, vma, addr);
+ #endif
if (!new_page)
break; /* Out of memory */
}
* appear not as the counts should be low
*/
list_add(&page->lru, &free_pages);
+ #ifdef CONFIG_AMLOGIC_CMA
+ if (ttu_flags & TTU_IGNORE_ACCESS)
+ ClearPageCmaAllocating(page);
+ #endif
continue;
cull_mlocked: