kevin/linux
1
Fork 0
Code

hugetlb: batch TLB flushes when restoring vmemmap

Browse Source 
Update the internal hugetlb restore vmemmap code path such that TLB
flushing can be batched.  Use the existing mechanism of passing the
VMEMMAP_REMAP_NO_TLB_FLUSH flag to indicate flushing should not be
performed for individual pages.  The routine
hugetlb_vmemmap_restore_folios is the only user of this new mechanism, and
it will perform a global flush after all vmemmap is restored.

Link: https://lkml.kernel.org/r/20231019023113.345257-9-mike.kravetz@oracle.com
Signed-off-by: Joao Martins <joao.m.martins@oracle.com>
Signed-off-by: Mike Kravetz <mike.kravetz@oracle.com>
Reviewed-by: Muchun Song <songmuchun@bytedance.com>
Cc: Anshuman Khandual <anshuman.khandual@arm.com>
Cc: Barry Song <21cnbao@gmail.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: David Rientjes <rientjes@google.com>
Cc: James Houghton <jthoughton@google.com>
Cc: Konrad Dybcio <konradybcio@kernel.org>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Miaohe Lin <linmiaohe@huawei.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Naoya Horiguchi <naoya.horiguchi@linux.dev>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Sergey Senozhatsky <senozhatsky@chromium.org>
Cc: Usama Arif <usama.arif@bytedance.com>
Cc: Xiongchun Duan <duanxiongchun@bytedance.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Mike Kravetz 2023-10-18 19:31:10 -07:00 committed by Andrew Morton
parent f13b83fdd9
commit c24f188b22
1 changed files with 24 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
mm/hugetlb_vmemmap.c
View File

@ -461,18 +461,19 @@ out:
* @end: end address of the vmemmap virtual address range that we want to
* remap.
* @reuse: reuse address.
* @flags: modifications to vmemmap_remap_walk flags
*
* Return: %0 on success, negative error code otherwise.
*/
static int vmemmap_remap_alloc(unsigned long start, unsigned long end,
unsigned long reuse)
unsigned long reuse, unsigned long flags)
{
LIST_HEAD(vmemmap_pages);
struct vmemmap_remap_walk walk = {
.remap_pte = vmemmap_restore_pte,
.reuse_addr = reuse,
.vmemmap_pages = &vmemmap_pages,
.flags = 0,
.flags = flags,
};
/* See the comment in the vmemmap_remap_free(). */
@ -494,17 +495,7 @@ EXPORT_SYMBOL(hugetlb_optimize_vmemmap_key);
static bool vmemmap_optimize_enabled = IS_ENABLED(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP_DEFAULT_ON);
core_param(hugetlb_free_vmemmap, vmemmap_optimize_enabled, bool, 0);
/**
* hugetlb_vmemmap_restore - restore previously optimized (by
* hugetlb_vmemmap_optimize()) vmemmap pages which
* will be reallocated and remapped.
* @h: struct hstate.
* @head: the head page whose vmemmap pages will be restored.
*
* Return: %0 if @head's vmemmap pages have been reallocated and remapped,
* negative error code otherwise.
*/
int hugetlb_vmemmap_restore(const struct hstate *h, struct page *head)
static int __hugetlb_vmemmap_restore(const struct hstate *h, struct page *head, unsigned long flags)
{
int ret;
unsigned long vmemmap_start = (unsigned long)head, vmemmap_end;
@ -525,7 +516,7 @@ int hugetlb_vmemmap_restore(const struct hstate *h, struct page *head)
* When a HugeTLB page is freed to the buddy allocator, previously
* discarded vmemmap pages must be allocated and remapping.
*/
ret = vmemmap_remap_alloc(vmemmap_start, vmemmap_end, vmemmap_reuse);
ret = vmemmap_remap_alloc(vmemmap_start, vmemmap_end, vmemmap_reuse, flags);
if (!ret) {
ClearHPageVmemmapOptimized(head);
static_branch_dec(&hugetlb_optimize_vmemmap_key);
@ -534,6 +525,21 @@ int hugetlb_vmemmap_restore(const struct hstate *h, struct page *head)
return ret;
}
/**
* hugetlb_vmemmap_restore - restore previously optimized (by
* hugetlb_vmemmap_optimize()) vmemmap pages which
* will be reallocated and remapped.
* @h: struct hstate.
* @head: the head page whose vmemmap pages will be restored.
*
* Return: %0 if @head's vmemmap pages have been reallocated and remapped,
* negative error code otherwise.
*/
int hugetlb_vmemmap_restore(const struct hstate *h, struct page *head)
{
return __hugetlb_vmemmap_restore(h, head, 0);
}
/**
* hugetlb_vmemmap_restore_folios - restore vmemmap for every folio on the list.
* @h: hstate.
@ -557,7 +563,8 @@ long hugetlb_vmemmap_restore_folios(const struct hstate *h,
list_for_each_entry_safe(folio, t_folio, folio_list, lru) {
if (folio_test_hugetlb_vmemmap_optimized(folio)) {
ret = hugetlb_vmemmap_restore(h, &folio->page);
ret = __hugetlb_vmemmap_restore(h, &folio->page,
VMEMMAP_REMAP_NO_TLB_FLUSH);
if (ret)
break;
restored++;
@ -567,6 +574,8 @@ long hugetlb_vmemmap_restore_folios(const struct hstate *h,
list_move(&folio->lru, non_hvo_folios);
}
if (restored)
flush_tlb_all();
if (!ret)
ret = restored;
return ret;