runtime: release unused memory to the OS.

Periodically browse MHeap's freelists for long unused spans and release them if any.

Current hardcoded settings:
        - GC is forced if none occured over the last 2 minutes.
        - spans are handed back after 5 minutes of uselessness.

SysUnused (for Unix) is a wrapper on madvise MADV_DONTNEED on Linux and MADV_FREE on BSDs.

R=rsc, dvyukov, remyoudompheng
CC=golang-dev
https://golang.org/cl/5451057

src/pkg/runtime/malloc.h

@@ -205,6 +205,7 @@ struct MStats
  uint64 heap_sys; // bytes obtained from system
  uint64 heap_idle; // bytes in idle spans
  uint64 heap_inuse; // bytes in non-idle spans
+ uint64 heap_released; // bytes released to the OS
  uint64 heap_objects; // total number of allocated objects
 
  // Statistics about allocation of low-level fixed-size structures.
@@ -220,6 +221,7 @@ struct MStats
  // Statistics about garbage collector.
  // Protected by stopping the world during GC.
  uint64 next_gc; // next GC (in heap_alloc time)
+ uint64 last_gc; // last GC (in absolute time)
  uint64 pause_total_ns;
  uint64 pause_ns[256];
  uint32 numgc;
@@ -304,14 +306,16 @@ struct MSpan
 {
  MSpan *next; // in a span linked list
  MSpan *prev; // in a span linked list
- MSpan *allnext;  // in the list of all spans
+ MSpan *allnext; // in the list of all spans
  PageID start; // starting page number
  uintptr npages; // number of pages in span
  MLink *freelist; // list of free objects
  uint32 ref; // number of allocated objects in this span
  uint32 sizeclass; // size class
  uint32 state; // MSpanInUse etc
- byte *limit; // end of data in span
+ int64 unusedsince; // First time spotted by GC in MSpanFree state
+ uintptr npreleased; // number of pages released to the OS
+ byte *limit; // end of data in span
 };
 
 void runtime·MSpan_Init(MSpan *span, PageID start, uintptr npages);
@@ -381,6 +385,7 @@ MSpan* runtime·MHeap_LookupMaybe(MHeap *h, void *v);
 void runtime·MGetSizeClassInfo(int32 sizeclass, uintptr *size, int32 *npages, int32 *nobj);
 void* runtime·MHeap_SysAlloc(MHeap *h, uintptr n);
 void runtime·MHeap_MapBits(MHeap *h);
+void runtime·MHeap_Scavenger(void);
 
 void* runtime·mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed);
 int32 runtime·mlookup(void *v, byte **base, uintptr *size, MSpan **s);

src/pkg/runtime/mem.go

@@ -17,11 +17,12 @@ type MemStats struct {
  Frees uint64 // number of frees
 
  // Main allocation heap statistics.
- HeapAlloc uint64 // bytes allocated and still in use
- HeapSys uint64 // bytes obtained from system
- HeapIdle uint64 // bytes in idle spans
- HeapInuse uint64 // bytes in non-idle span
- HeapObjects uint64 // total number of allocated objects
+ HeapAlloc uint64 // bytes allocated and still in use
+ HeapSys uint64 // bytes obtained from system
+ HeapIdle uint64 // bytes in idle spans
+ HeapInuse uint64 // bytes in non-idle span
+ HeapReleased uint64 // bytes released to the OS
+ HeapObjects uint64 // total number of allocated objects
 
  // Low-level fixed-size structure allocator statistics.
  // Inuse is bytes used now.
@@ -35,7 +36,8 @@ type MemStats struct {
  BuckHashSys uint64 // profiling bucket hash table
 
  // Garbage collector statistics.
- NextGC uint64
+ NextGC uint64 // next run in HeapAlloc time (bytes)
+ LastGC uint64 // last run in absolute time (ns)
  PauseTotalNs uint64
  PauseNs [256]uint64 // most recent GC pause times
  NumGC uint32

src/pkg/runtime/mgc0.c

@@ -716,8 +716,10 @@ sweep(void)
  byte *p;
  MCache *c;
  byte *arena_start;
+ int64 now;
 
  arena_start = runtime·mheap.arena_start;
+ now = runtime·nanotime();
 
  for(;;) {
  s = work.spans;
@@ -726,6 +728,11 @@ sweep(void)
  if(!runtime·casp(&work.spans, s, s->allnext))
  continue;
 
+ // Stamp newly unused spans. The scavenger will use that
+ // info to potentially give back some pages to the OS.
+ if(s->state == MSpanFree && s->unusedsince == 0)
+ s->unusedsince = now;
+
  if(s->state != MSpanInUse)
  continue;
 
@@ -963,6 +970,7 @@ runtime·gc(int32 force)
  obj1 = mstats.nmalloc - mstats.nfree;
 
  t3 = runtime·nanotime();
+ mstats.last_gc = t3;
  mstats.pause_ns[mstats.numgc%nelem(mstats.pause_ns)] = t3 - t0;
  mstats.pause_total_ns += t3 - t0;
  mstats.numgc++;

src/pkg/runtime/mheap.c

@@ -103,6 +103,8 @@ MHeap_AllocLocked(MHeap *h, uintptr npage, int32 sizeclass)
  runtime·MSpanList_Remove(s);
  s->state = MSpanInUse;
  mstats.heap_idle -= s->npages<<PageShift;
+ mstats.heap_released -= s->npreleased<<PageShift;
+ s->npreleased = 0;
 
  if(s->npages > npage) {
  // Trim extra and put it back in the heap.
@@ -280,6 +282,8 @@ MHeap_FreeLocked(MHeap *h, MSpan *s)
  }
  mstats.heap_idle += s->npages<<PageShift;
  s->state = MSpanFree;
+ s->unusedsince = 0;
+ s->npreleased = 0;
  runtime·MSpanList_Remove(s);
  sp = (uintptr*)(s->start<<PageShift);
 
@@ -292,6 +296,7 @@ MHeap_FreeLocked(MHeap *h, MSpan *s)
  *tp |= *sp; // propagate "needs zeroing" mark
  s->start = t->start;
  s->npages += t->npages;
+ s->npreleased = t->npreleased; // absorb released pages
  p -= t->npages;
  h->map[p] = s;
  runtime·MSpanList_Remove(t);
@@ -304,6 +309,7 @@ MHeap_FreeLocked(MHeap *h, MSpan *s)
  tp = (uintptr*)(t->start<<PageShift);
  *sp |= *tp; // propagate "needs zeroing" mark
  s->npages += t->npages;
+ s->npreleased += t->npreleased;
  h->map[p + s->npages - 1] = s;
  runtime·MSpanList_Remove(t);
  t->state = MSpanDead;
@@ -317,8 +323,81 @@ MHeap_FreeLocked(MHeap *h, MSpan *s)
  runtime·MSpanList_Insert(&h->free[s->npages], s);
  else
  runtime·MSpanList_Insert(&h->large, s);
+}
 
- // TODO(rsc): IncrementalScavenge() to return memory to OS.
+// Release (part of) unused memory to OS.
+// Goroutine created in runtime·schedinit.
+// Loop forever.
+void
+runtime·MHeap_Scavenger(void)
+{
+ MHeap *h;
+ MSpan *s, *list;
+ uint64 tick, now, forcegc, limit;
+ uint32 k, i;
+ uintptr released, sumreleased;
+ byte *env;
+ bool trace;
+ Note note;
+
+ // If we go two minutes without a garbage collection, force one to run.
+ forcegc = 2*60*1e9;
+ // If a span goes unused for 5 minutes after a garbage collection,
+ // we hand it back to the operating system.
+ limit = 5*60*1e9;
+ // Make wake-up period small enough for the sampling to be correct.
+ tick = forcegc < limit ? forcegc/2 : limit/2;
+
+ trace = false;
+ env = runtime·getenv("GOGCTRACE");
+ if(env != nil)
+ trace = runtime·atoi(env) > 0;
+
+ h = &runtime·mheap;
+ for(k=0;; k++) {
+ runtime·noteclear(&note);
+ runtime·entersyscall();
+ runtime·notetsleep(&note, tick);
+ runtime·exitsyscall();
+
+ runtime·lock(h);
+ now = runtime·nanotime();
+ if(now - mstats.last_gc > forcegc) {
+ runtime·unlock(h);
+ runtime·gc(1);
+ runtime·lock(h);
+ now = runtime·nanotime();
+ if (trace)
+ runtime·printf("scvg%d: GC forced\n", k);
+ }
+ sumreleased = 0;
+ for(i=0; i < nelem(h->free)+1; i++) {
+ if(i < nelem(h->free))
+ list = &h->free[i];
+ else
+ list = &h->large;
+ if(runtime·MSpanList_IsEmpty(list))
+ continue;
+ for(s=list->next; s != list; s=s->next) {
+ if(s->unusedsince != 0 && (now - s->unusedsince) > limit) {
+ released = (s->npages - s->npreleased) << PageShift;
+ mstats.heap_released += released;
+ sumreleased += released;
+ s->npreleased = s->npages;
+ runtime·SysUnused((void*)(s->start << PageShift), s->npages << PageShift);
+ }
+ }
+ }
+ runtime·unlock(h);
+
+ if(trace) {
+ if(sumreleased > 0)
+ runtime·printf("scvg%d: %p MB released\n", k, sumreleased>>20);
+ runtime·printf("scvg%d: inuse: %D, idle: %D, sys: %D, released: %D, consumed: %D (MB)\n",
+ k, mstats.heap_inuse>>20, mstats.heap_idle>>20, mstats.heap_sys>>20,
+ mstats.heap_released>>20, (mstats.heap_sys - mstats.heap_released)>>20);
+ }
+ }
 }
 
 // Initialize a new span with the given start and npages.
@@ -333,6 +412,8 @@ runtime·MSpan_Init(MSpan *span, PageID start, uintptr npages)
  span->ref = 0;
  span->sizeclass = 0;
  span->state = 0;
+ span->unusedsince = 0;
+ span->npreleased = 0;
 }
 
 // Initialize an empty doubly-linked list.

src/pkg/runtime/proc.c

@@ -164,6 +164,9 @@ setmcpumax(uint32 n)
  }
 }
 
+// Keep trace of scavenger's goroutine for deadlock detection.
+static G *scvg;
+
 // The bootstrap sequence is:
 //
 // call osinit
@@ -206,6 +209,8 @@ runtime·schedinit(void)
 
  mstats.enablegc = 1;
  m->nomemprof--;
+
+ scvg = runtime·newproc1((byte*)runtime·MHeap_Scavenger, nil, 0, 0, runtime·schedinit);
 }
 
 extern void main·init(void);
@@ -582,9 +587,12 @@ nextgandunlock(void)
  mput(m);
  }
 
- v = runtime·atomicload(&runtime·sched.atomic);
- if(runtime·sched.grunning == 0)
- runtime·throw("all goroutines are asleep - deadlock!");
+ // Look for deadlock situation: one single active g which happens to be scvg.
+ if(runtime·sched.grunning == 1 && runtime·sched.gwait == 0) {
+ if(scvg->status == Grunning || scvg->status == Gsyscall)
+ runtime·throw("all goroutines are asleep - deadlock!");
+ }
+
  m->nextg = nil;
  m->waitnextg = 1;
  runtime·noteclear(&m->havenextg);
@@ -593,6 +601,7 @@ nextgandunlock(void)
  // Entersyscall might have decremented mcpu too, but if so
  // it will see the waitstop and take the slow path.
  // Exitsyscall never increments mcpu beyond mcpumax.
+ v = runtime·atomicload(&runtime·sched.atomic);
  if(atomic_waitstop(v) && atomic_mcpu(v) <= atomic_mcpumax(v)) {
  // set waitstop = 0 (known to be 1)
  runtime·xadd(&runtime·sched.atomic, -1<<waitstopShift);