mm: page_alloc: per-migratetype pcplist for THPs

Right now, there is only one pcplist for THP allocations. However,
while most THPs are movable, the huge zero page is not. This means a
movable THP allocation can grab an unmovable block from the pcplist,
and a subsequent THP split, partial free, and reallocation of the
remainder will mix movable and unmovable pages in the block.

While this isn't a huge source of block pollution in practice, it
happens often enough to trigger debug warnings fairly quickly under
load. In the interest of tightening up pageblock hygiene, make the THP
pcplists fully migratetype-aware, just like the lower order ones.

Signed-off-by: Johannes Weiner <[email protected]>

include/linux/mmzone.h

@@ -552,13 +552,11 @@ enum zone_watermarks {
 };
 
 /*
- * One per migratetype for each PAGE_ALLOC_COSTLY_ORDER. One additional list
- * for THP which will usually be GFP_MOVABLE. Even if it is another type,
- * it should not contribute to serious fragmentation causing THP allocation
- * failures.
+ * One per migratetype for each PAGE_ALLOC_COSTLY_ORDER. One additional set
+ * for THP (usually GFP_MOVABLE, but with exception of the huge zero page.)
  */
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
-#define NR_PCP_THP 1
+#define NR_PCP_THP MIGRATE_PCPTYPES
 #else
 #define NR_PCP_THP 0
 #endif

mm/page_alloc.c

@@ -710,7 +710,7 @@ static inline unsigned int order_to_pindex(int migratetype, int order)
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 	if (order > PAGE_ALLOC_COSTLY_ORDER) {
 		VM_BUG_ON(order != pageblock_order);
-		return NR_LOWORDER_PCP_LISTS;
+		return NR_LOWORDER_PCP_LISTS + migratetype;
 	}
 #else
 	VM_BUG_ON(order > PAGE_ALLOC_COSTLY_ORDER);
@@ -724,7 +724,7 @@ static inline int pindex_to_order(unsigned int pindex)
 	int order = pindex / MIGRATE_PCPTYPES;
 
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
-	if (pindex == NR_LOWORDER_PCP_LISTS)
+	if (pindex >= NR_LOWORDER_PCP_LISTS)
 		order = pageblock_order;
 #else
 	VM_BUG_ON(order > PAGE_ALLOC_COSTLY_ORDER);