hugetlb: perform vmemmap optimization on a list of pages

When adding hugetlb pages to the pool, we first create a list of the
allocated pages before adding to the pool.  Pass this list of pages to a
new routine hugetlb_vmemmap_optimize_folios() for vmemmap optimization.

Due to significant differences in vmemmmap initialization for bootmem
allocated hugetlb pages, a new routine prep_and_add_bootmem_folios is
created.

We also modify the routine vmemmap_should_optimize() to check for pages
that are already optimized.  There are code paths that might request
vmemmap optimization twice and we want to make sure this is not attempted.

Link: https://lkml.kernel.org/r/[email protected]
Signed-off-by: Mike Kravetz <[email protected]>
Reviewed-by: Muchun Song <[email protected]>
Cc: Anshuman Khandual <[email protected]>
Cc: Barry Song <[email protected]>
Cc: David Hildenbrand <[email protected]>
Cc: David Rientjes <[email protected]>
Cc: James Houghton <[email protected]>
Cc: Joao Martins <[email protected]>
Cc: Konrad Dybcio <[email protected]>
Cc: Matthew Wilcox (Oracle) <[email protected]>
Cc: Miaohe Lin <[email protected]>
Cc: Michal Hocko <[email protected]>
Cc: Naoya Horiguchi <[email protected]>
Cc: Oscar Salvador <[email protected]>
Cc: Sergey Senozhatsky <[email protected]>
Cc: Usama Arif <[email protected]>
Cc: Xiongchun Duan <[email protected]>
Signed-off-by: Andrew Morton <[email protected]>

mm/hugetlb.c

@@ -2282,6 +2282,9 @@ static void prep_and_add_allocated_folios(struct hstate *h,
  unsigned long flags;
  struct folio *folio, *tmp_f;
 
+ /* Send list for bulk vmemmap optimization processing */
+ hugetlb_vmemmap_optimize_folios(h, folio_list);
+
  /* Add all new pool pages to free lists in one lock cycle */
  spin_lock_irqsave(&hugetlb_lock, flags);
  list_for_each_entry_safe(folio, tmp_f, folio_list, lru) {
@@ -3344,6 +3347,35 @@ static void __init hugetlb_folio_init_vmemmap(struct folio *folio,
  prep_compound_head((struct page *)folio, huge_page_order(h));
 }
 
+static void __init prep_and_add_bootmem_folios(struct hstate *h,
+ struct list_head *folio_list)
+{
+ unsigned long flags;
+ struct folio *folio, *tmp_f;
+
+ /* Send list for bulk vmemmap optimization processing */
+ hugetlb_vmemmap_optimize_folios(h, folio_list);
+
+ /* Add all new pool pages to free lists in one lock cycle */
+ spin_lock_irqsave(&hugetlb_lock, flags);
+ list_for_each_entry_safe(folio, tmp_f, folio_list, lru) {
+ if (!folio_test_hugetlb_vmemmap_optimized(folio)) {
+ /*
+ * If HVO fails, initialize all tail struct pages
+ * We do not worry about potential long lock hold
+ * time as this is early in boot and there should
+ * be no contention.
+ */
+ hugetlb_folio_init_tail_vmemmap(folio,
+ HUGETLB_VMEMMAP_RESERVE_PAGES,
+ pages_per_huge_page(h));
+ }
+ __prep_account_new_huge_page(h, folio_nid(folio));
+ enqueue_hugetlb_folio(h, folio);
+ }
+ spin_unlock_irqrestore(&hugetlb_lock, flags);
+}
+
 /*
  * Put bootmem huge pages into the standard lists after mem_map is up.
  * Note: This only applies to gigantic (order > MAX_ORDER) pages.
@@ -3364,20 +3396,15 @@ static void __init gather_bootmem_prealloc(void)
  * in this list. If so, process each size separately.
  */
  if (h != prev_h && prev_h != NULL)
- prep_and_add_allocated_folios(prev_h, &folio_list);
+ prep_and_add_bootmem_folios(prev_h, &folio_list);
  prev_h = h;
 
  VM_BUG_ON(!hstate_is_gigantic(h));
  WARN_ON(folio_ref_count(folio) != 1);
 
  hugetlb_folio_init_vmemmap(folio, h,
  HUGETLB_VMEMMAP_RESERVE_PAGES);
- __prep_new_hugetlb_folio(h, folio);
- /* If HVO fails, initialize all tail struct pages */
- if (!HPageVmemmapOptimized(&folio->page))
- hugetlb_folio_init_tail_vmemmap(folio,
- HUGETLB_VMEMMAP_RESERVE_PAGES,
- pages_per_huge_page(h));
+ init_new_hugetlb_folio(h, folio);
  list_add(&folio->lru, &folio_list);
 
  /*
@@ -3389,7 +3416,7 @@ static void __init gather_bootmem_prealloc(void)
  cond_resched();
  }
 
- prep_and_add_allocated_folios(h, &folio_list);
+ prep_and_add_bootmem_folios(h, &folio_list);
 }
 
 static void __init hugetlb_hstate_alloc_pages_onenode(struct hstate *h, int nid)

mm/hugetlb_vmemmap.c

@@ -483,6 +483,9 @@ int hugetlb_vmemmap_restore(const struct hstate *h, struct page *head)
 /* Return true iff a HugeTLB whose vmemmap should and can be optimized. */
 static bool vmemmap_should_optimize(const struct hstate *h, const struct page *head)
 {
+ if (HPageVmemmapOptimized((struct page *)head))
+ return false;
+
  if (!READ_ONCE(vmemmap_optimize_enabled))
  return false;
 
@@ -572,6 +575,14 @@ void hugetlb_vmemmap_optimize(const struct hstate *h, struct page *head)
  SetHPageVmemmapOptimized(head);
 }
 
+void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_list)
+{
+ struct folio *folio;
+
+ list_for_each_entry(folio, folio_list, lru)
+ hugetlb_vmemmap_optimize(h, &folio->page);
+}
+
 static struct ctl_table hugetlb_vmemmap_sysctls[] = {
  {
  .procname = "hugetlb_optimize_vmemmap",

mm/hugetlb_vmemmap.h

@@ -20,6 +20,7 @@
 #ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
 int hugetlb_vmemmap_restore(const struct hstate *h, struct page *head);
 void hugetlb_vmemmap_optimize(const struct hstate *h, struct page *head);
+void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_list);
 
 static inline unsigned int hugetlb_vmemmap_size(const struct hstate *h)
 {
@@ -48,6 +49,10 @@ static inline void hugetlb_vmemmap_optimize(const struct hstate *h, struct page
 {
 }
 
+static inline void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_list)
+{
+}
+
 static inline unsigned int hugetlb_vmemmap_optimizable_size(const struct hstate *h)
 {
  return 0;