Re: improved version - Re: contig memory allocator fix (vm_contig.c)

[Matthew Dillon <dillon@xxxxxxxxxxxxxxxxxxxx>]

    I did a cleanup pass on Andrew's code and rearranged a few minor things,
    plus removed the vm_contig_[int]_pg_alloc() shim and just made the
    original vm_contig_pg_alloc() take an mflags argument, and changed
    the mflags type to an int.

    It looks like nothing else in the system used the vm_contig_pg_alloc()
    entry point anyway so I also made that static.

    Please test.

						-Matt

Index: vm/vm_contig.c
===================================================================
RCS file: /cvs/src/sys/vm/vm_contig.c,v
retrieving revision 1.12
diff -u -r1.12 vm_contig.c
--- vm/vm_contig.c	10 Nov 2004 20:19:51 -0000	1.12
+++ vm/vm_contig.c	22 Feb 2005 19:33:20 -0000
@@ -146,8 +146,8 @@
 
 	for (m = TAILQ_FIRST(&vm_page_queues[queue].pl); m != NULL; m = next) {
 		KASSERT(m->queue == queue,
-			("vm_contig_clean: page %p's queue is not %d", m, queue));
-		
+			("vm_contig_clean: page %p's queue is not %d", 
+			m, queue));
 		next = TAILQ_NEXT(m, pageq);
 		
 		if (vm_page_sleep_busy(m, TRUE, "vpctw0"))
@@ -170,15 +170,29 @@
 				return (TRUE);
 			}
 		}
-		
 		if ((m->dirty == 0) && (m->busy == 0) && (m->hold_count == 0))
 			vm_page_cache(m);
 	}
-
 	return (FALSE);
 }
 
 /*
+ * vm_contig_pg_flush:
+ * 
+ * Attempt to flush (count) pages from the given page queue.   This may or
+ * may not succeed.  Take up to <count> passes and delay 1/20 of a second
+ * between each pass.
+ */
+static void
+vm_contig_pg_flush(int queue, int count) 
+{
+	while (count > 0) {
+		if (!vm_contig_pg_clean(queue))
+			break;
+		--count;
+	}
+}
+/*
  * vm_contig_pg_alloc:
  *
  * Allocate contiguous pages from the VM.  This function does not
@@ -187,15 +201,10 @@
  *
  * Malloc()'s data structures have been used for collection of
  * statistics and for allocations of less than a page.
- *
  */
-int
-vm_contig_pg_alloc(
-	unsigned long size,
-	vm_paddr_t low,
-	vm_paddr_t high,
-	unsigned long alignment,
-	unsigned long boundary)
+static int
+vm_contig_pg_alloc(unsigned long size, vm_paddr_t low, vm_paddr_t high,
+	unsigned long alignment, unsigned long boundary, int mflags)
 {
 	int i, start, pass;
 	vm_offset_t phys;
@@ -212,13 +221,20 @@
 		panic("vm_contig_pg_alloc: boundary must be a power of 2");
 
 	start = 0;
-	for (pass = 0; pass <= 1; pass++) {
-		crit_enter();
-again:
+	crit_enter();
+
+	/*
+	 * Three passes (0, 1, 2).  Each pass scans the VM page list for
+	 * free or cached pages.  After each pass if the entire scan failed
+	 * we attempt to flush inactive pages and reset the start index back
+	 * to 0.  For passes 1 and 2 we also attempt to flush active pages.
+	 */
+	for (pass = 0; pass < 3; pass++) {
 		/*
-		 * Find first page in array that is free, within range, aligned, and
-		 * such that the boundary won't be crossed.
+		 * Find first page in array that is free, within range, 
+		 * aligned, and such that the boundary won't be crossed.
 		 */
+again:
 		for (i = start; i < vmstats.v_page_count; i++) {
 			m = &pga[i];
 			phys = VM_PAGE_TO_PHYS(m);
@@ -244,13 +260,39 @@
 		if ((i == vmstats.v_page_count) ||
 			((VM_PAGE_TO_PHYS(&pga[i]) + size) > high)) {
 
-again1:
-			if (vm_contig_pg_clean(PQ_INACTIVE))
-				goto again1;
-			if (vm_contig_pg_clean(PQ_ACTIVE))
-				goto again1;
+			/*
+			 * Best effort flush of all inactive pages.
+			 * This is quite quick, for now stall all
+			 * callers, even if they've specified M_NOWAIT.
+			 */
+			vm_contig_pg_flush(PQ_INACTIVE, 
+					    vmstats.v_inactive_count);
+
+			crit_exit(); /* give interrupts a chance */
+			crit_enter();
+
+			/*
+			 * Best effort flush of active pages.
+			 *
+			 * This is very, very slow.
+			 * Only do this if the caller has agreed to M_WAITOK.
+			 *
+			 * If enough pages are flushed, we may succeed on
+			 * next (final) pass, if not the caller, contigmalloc(),
+			 * will fail in the index < 0 case.
+			 */
+			if (pass > 0 && (mflags & M_WAITOK)) {
+				vm_contig_pg_flush (PQ_ACTIVE,
+						    vmstats.v_active_count);
+			}
 
-			crit_exit();
+			/*
+			 * We're already too high in the address space
+			 * to succeed, reset to 0 for the next iteration.
+			 */
+			start = 0;
+			crit_exit(); /* give interrupts a chance */
+			crit_enter();
 			continue;	/* next pass */
 		}
 		start = i;
@@ -423,13 +465,13 @@
 	int index;
 	void *rv;
 
-	index = vm_contig_pg_alloc(size, low, high, alignment, boundary);
+	index = vm_contig_pg_alloc(size, low, high, alignment, boundary, flags);
 	if (index < 0) {
 		printf("contigmalloc_map: failed in index < 0 case!");
 		return NULL;
 	}
 
-	rv = (void *) vm_contig_pg_kmap(index, size, map, flags);
+	rv = (void *)vm_contig_pg_kmap(index, size, map, flags);
 	if (!rv)
 		vm_contig_pg_free(index, size);
 	
Index: vm/vm_page.h
===================================================================
RCS file: /cvs/src/sys/vm/vm_page.h,v
retrieving revision 1.16
diff -u -r1.16 vm_page.h
--- vm/vm_page.h	4 Oct 2004 09:05:26 -0000	1.16
+++ vm/vm_page.h	22 Feb 2005 19:32:17 -0000
@@ -418,7 +418,6 @@
 vm_page_t vm_page_list_find(int basequeue, int index, boolean_t prefer_zero);
 void vm_page_zero_invalid(vm_page_t m, boolean_t setvalid);
 void vm_page_free_toq(vm_page_t m);
-int vm_contig_pg_alloc(u_long, vm_paddr_t, vm_paddr_t, u_long, u_long);
 vm_offset_t vm_contig_pg_kmap(int, u_long, vm_map_t, int);
 void vm_contig_pg_free(int, u_long);