kevin/linux
1
Fork 0
Code

mm/gup: introduce check_and_migrate_movable_folios()

Browse Source 
This helper is the folio equivalent of check_and_migrate_movable_pages(). 
Therefore, all the rules that apply to check_and_migrate_movable_pages()
also apply to this one as well.  Currently, this helper is only used by
memfd_pin_folios().

This patch also includes changes to rename and convert the internal
functions collect_longterm_unpinnable_pages() and
migrate_longterm_unpinnable_pages() to work on folios.  As a result,
check_and_migrate_movable_pages() is now a wrapper around
check_and_migrate_movable_folios().

Link: https://lkml.kernel.org/r/20240624063952.1572359-3-vivek.kasireddy@intel.com
Signed-off-by: Vivek Kasireddy <vivek.kasireddy@intel.com>
Suggested-by: David Hildenbrand <david@redhat.com>
Acked-by: David Hildenbrand <david@redhat.com>
Acked-by: Dave Airlie <airlied@redhat.com>
Acked-by: Gerd Hoffmann <kraxel@redhat.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Jason Gunthorpe <jgg@nvidia.com>
Cc: Peter Xu <peterx@redhat.com>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Daniel Vetter <daniel.vetter@ffwll.ch>
Cc: Dongwon Kim <dongwon.kim@intel.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Junxiao Chang <junxiao.chang@intel.com>
Cc: Mike Kravetz <mike.kravetz@oracle.com>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Shuah Khan <shuah@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Vivek Kasireddy 2024-06-23 23:36:10 -07:00 committed by Andrew Morton
parent 6cc040542b
commit 53ba78de06
1 changed files with 78 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

126
mm/gup.c
View File

@ -2441,19 +2441,19 @@ struct page *get_dump_page(unsigned long addr)
#ifdef CONFIG_MIGRATION #ifdef CONFIG_MIGRATION
/* /*
* Returns the number of collected pages. Return value is always >= 0. * Returns the number of collected folios. Return value is always >= 0.
*/ */
static unsigned long collect_longterm_unpinnable_pages( static unsigned long collect_longterm_unpinnable_folios(
struct list_head *movable_page_list, struct list_head *movable_folio_list,
unsigned long nr_pages, unsigned long nr_folios,
struct page **pages) struct folio **folios)
{ {
unsigned long i, collected = 0; unsigned long i, collected = 0;
struct folio *prev_folio = NULL; struct folio *prev_folio = NULL;
bool drain_allow = true; bool drain_allow = true;
for (i = 0; i < nr_pages; i++) { for (i = 0; i < nr_folios; i++) {
struct folio *folio = page_folio(pages[i]); struct folio *folio = folios[i];
if (folio == prev_folio) if (folio == prev_folio)
continue; continue;
@ -2468,7 +2468,7 @@ static unsigned long collect_longterm_unpinnable_pages(
continue; continue;
if (folio_test_hugetlb(folio)) { if (folio_test_hugetlb(folio)) {
isolate_hugetlb(folio, movable_page_list); isolate_hugetlb(folio, movable_folio_list);
continue; continue;
} }
@ -2480,7 +2480,7 @@ static unsigned long collect_longterm_unpinnable_pages(
if (!folio_isolate_lru(folio)) if (!folio_isolate_lru(folio))
continue; continue;
list_add_tail(&folio->lru, movable_page_list); list_add_tail(&folio->lru, movable_folio_list);
node_stat_mod_folio(folio, node_stat_mod_folio(folio,
NR_ISOLATED_ANON + folio_is_file_lru(folio), NR_ISOLATED_ANON + folio_is_file_lru(folio),
folio_nr_pages(folio)); folio_nr_pages(folio));
@ -2490,27 +2490,28 @@ static unsigned long collect_longterm_unpinnable_pages(
} }
/* /*
* Unpins all pages and migrates device coherent pages and movable_page_list. * Unpins all folios and migrates device coherent folios and movable_folio_list.
* Returns -EAGAIN if all pages were successfully migrated or -errno for failure * Returns -EAGAIN if all folios were successfully migrated or -errno for
* (or partial success). * failure (or partial success).
*/ */
static int migrate_longterm_unpinnable_pages( static int migrate_longterm_unpinnable_folios(
struct list_head *movable_page_list, struct list_head *movable_folio_list,
unsigned long nr_pages, unsigned long nr_folios,
struct page **pages) struct folio **folios)
{ {
int ret; int ret;
unsigned long i; unsigned long i;
for (i = 0; i < nr_pages; i++) { for (i = 0; i < nr_folios; i++) {
struct folio *folio = page_folio(pages[i]); struct folio *folio = folios[i];
if (folio_is_device_coherent(folio)) { if (folio_is_device_coherent(folio)) {
/* /*
* Migration will fail if the page is pinned, so convert * Migration will fail if the folio is pinned, so
* the pin on the source page to a normal reference. * convert the pin on the source folio to a normal
* reference.
*/ */
pages[i] = NULL; folios[i] = NULL;
folio_get(folio); folio_get(folio);
gup_put_folio(folio, 1, FOLL_PIN); gup_put_folio(folio, 1, FOLL_PIN);
@ -2523,24 +2524,24 @@ static int migrate_longterm_unpinnable_pages(
} }
/* /*
* We can't migrate pages with unexpected references, so drop * We can't migrate folios with unexpected references, so drop
* the reference obtained by __get_user_pages_locked(). * the reference obtained by __get_user_pages_locked().
* Migrating pages have been added to movable_page_list after * Migrating folios have been added to movable_folio_list after
* calling folio_isolate_lru() which takes a reference so the * calling folio_isolate_lru() which takes a reference so the
* page won't be freed if it's migrating. * folio won't be freed if it's migrating.
*/ */
unpin_user_page(pages[i]); unpin_folio(folios[i]);
pages[i] = NULL; folios[i] = NULL;
} }
if (!list_empty(movable_page_list)) { if (!list_empty(movable_folio_list)) {
struct migration_target_control mtc = { struct migration_target_control mtc = {
.nid = NUMA_NO_NODE, .nid = NUMA_NO_NODE,
.gfp_mask = GFP_USER | __GFP_NOWARN, .gfp_mask = GFP_USER | __GFP_NOWARN,
.reason = MR_LONGTERM_PIN, .reason = MR_LONGTERM_PIN,
}; };
if (migrate_pages(movable_page_list, alloc_migration_target, if (migrate_pages(movable_folio_list, alloc_migration_target,
NULL, (unsigned long)&mtc, MIGRATE_SYNC, NULL, (unsigned long)&mtc, MIGRATE_SYNC,
MR_LONGTERM_PIN, NULL)) { MR_LONGTERM_PIN, NULL)) {
ret = -ENOMEM; ret = -ENOMEM;
@ -2548,48 +2549,71 @@ static int migrate_longterm_unpinnable_pages(
} }
} }
putback_movable_pages(movable_page_list); putback_movable_pages(movable_folio_list);
return -EAGAIN; return -EAGAIN;
err: err:
for (i = 0; i < nr_pages; i++) unpin_folios(folios, nr_folios);
if (pages[i]) putback_movable_pages(movable_folio_list);
unpin_user_page(pages[i]);
putback_movable_pages(movable_page_list);
return ret; return ret;
} }
/* /*
* Check whether all pages are *allowed* to be pinned. Rather confusingly, all * Check whether all folios are *allowed* to be pinned indefinitely (longterm).
* pages in the range are required to be pinned via FOLL_PIN, before calling * Rather confusingly, all folios in the range are required to be pinned via
* this routine. * FOLL_PIN, before calling this routine.
* *
* If any pages in the range are not allowed to be pinned, then this routine * If any folios in the range are not allowed to be pinned, then this routine
* will migrate those pages away, unpin all the pages in the range and return * will migrate those folios away, unpin all the folios in the range and return
* -EAGAIN. The caller should re-pin the entire range with FOLL_PIN and then * -EAGAIN. The caller should re-pin the entire range with FOLL_PIN and then
* call this routine again. * call this routine again.
* *
* If an error other than -EAGAIN occurs, this indicates a migration failure. * If an error other than -EAGAIN occurs, this indicates a migration failure.
* The caller should give up, and propagate the error back up the call stack. * The caller should give up, and propagate the error back up the call stack.
* *
* If everything is OK and all pages in the range are allowed to be pinned, then * If everything is OK and all folios in the range are allowed to be pinned,
* this routine leaves all pages pinned and returns zero for success. * then this routine leaves all folios pinned and returns zero for success.
*/
static long check_and_migrate_movable_folios(unsigned long nr_folios,
struct folio **folios)
{
unsigned long collected;
LIST_HEAD(movable_folio_list);
collected = collect_longterm_unpinnable_folios(&movable_folio_list,
nr_folios, folios);
if (!collected)
return 0;
return migrate_longterm_unpinnable_folios(&movable_folio_list,
nr_folios, folios);
}
/*
* This routine just converts all the pages in the @pages array to folios and
* calls check_and_migrate_movable_folios() to do the heavy lifting.
*
* Please see the check_and_migrate_movable_folios() documentation for details.
*/ */
static long check_and_migrate_movable_pages(unsigned long nr_pages, static long check_and_migrate_movable_pages(unsigned long nr_pages,
struct page **pages) struct page **pages)
{ {
unsigned long collected; struct folio **folios;
LIST_HEAD(movable_page_list); long i, ret;
collected = collect_longterm_unpinnable_pages(&movable_page_list, folios = kmalloc_array(nr_pages, sizeof(*folios), GFP_KERNEL);
nr_pages, pages); if (!folios)
if (!collected) return -ENOMEM;
return 0;
return migrate_longterm_unpinnable_pages(&movable_page_list, nr_pages, for (i = 0; i < nr_pages; i++)
pages); folios[i] = page_folio(pages[i]);
ret = check_and_migrate_movable_folios(nr_pages, folios);
kfree(folios);
return ret;
} }
#else #else
static long check_and_migrate_movable_pages(unsigned long nr_pages, static long check_and_migrate_movable_pages(unsigned long nr_pages,
@ -2597,6 +2621,12 @@ static long check_and_migrate_movable_pages(unsigned long nr_pages,
{ {
return 0; return 0;
} }
static long check_and_migrate_movable_folios(unsigned long nr_folios,
struct folio **folios)
{
return 0;
}
#endif /* CONFIG_MIGRATION */ #endif /* CONFIG_MIGRATION */
/* /*