diff -Naur mlton/runtime/gc/cheney-copy.c mltonp1/runtime/gc/cheney-copy.c --- mlton/runtime/gc/cheney-copy.c 2008-04-11 19:12:27.000000000 +0200 +++ mltonp1/runtime/gc/cheney-copy.c 2008-05-05 01:37:24.000000000 +0200 @@ -39,6 +39,7 @@ void swapHeapsForCheneyCopy (GC_state s) { struct GC_heap tempHeap; + copyCardMapAndCrossMap (s, &s->secondaryHeap); tempHeap = s->secondaryHeap; s->secondaryHeap = s->heap; s->heap = tempHeap; diff -Naur mlton/runtime/gc/generational.c mltonp1/runtime/gc/generational.c --- mlton/runtime/gc/generational.c 2008-04-11 19:12:27.000000000 +0200 +++ mltonp1/runtime/gc/generational.c 2008-05-05 01:37:40.000000000 +0200 @@ -119,7 +119,34 @@ s->generationalMaps.crossMapLength * CROSS_MAP_ELEM_SIZE); } -void createCardMapAndCrossMap (GC_state s) { +/* Compute the number of bytes that are needed to store the card map and + cross map at the end of a heap with the given size. */ +size_t computeCardMapAndCrossMapSize (GC_state s, size_t size) { + unless (s->mutatorMarksCards) { + return 0; + } + assert (isAligned (size, CARD_SIZE)); + + GC_cardMapIndex cardMapLength; + size_t cardMapSize; + GC_crossMapIndex crossMapLength; + size_t crossMapSize; + size_t totalMapSize; + + cardMapLength = sizeToCardMapIndex (size); + cardMapSize = align (cardMapLength * CARD_MAP_ELEM_SIZE, s->sysvals.pageSize); + cardMapLength = (GC_cardMapIndex)(cardMapSize / CARD_MAP_ELEM_SIZE); + + crossMapLength = sizeToCardMapIndex (size); + crossMapSize = align (crossMapLength * CROSS_MAP_ELEM_SIZE, s->sysvals.pageSize); + crossMapLength = (GC_crossMapIndex)(crossMapSize / CROSS_MAP_ELEM_SIZE); + + totalMapSize = cardMapSize + crossMapSize; + + return totalMapSize; +} + +static inline void initCardMapAndCrossMap (GC_state s, GC_heap h, size_t size) { unless (s->mutatorMarksCards) { s->generationalMaps.cardMapLength = 0; s->generationalMaps.cardMap = NULL; @@ -128,7 +155,7 @@ s->generationalMaps.crossMap = NULL; return; } - assert (isAligned (s->heap.size, CARD_SIZE)); + assert (isAligned (size, CARD_SIZE)); GC_cardMapIndex cardMapLength; size_t cardMapSize; @@ -136,37 +163,47 @@ size_t crossMapSize; size_t totalMapSize; - cardMapLength = sizeToCardMapIndex (s->heap.size); + cardMapLength = sizeToCardMapIndex (size); cardMapSize = align (cardMapLength * CARD_MAP_ELEM_SIZE, s->sysvals.pageSize); cardMapLength = (GC_cardMapIndex)(cardMapSize / CARD_MAP_ELEM_SIZE); s->generationalMaps.cardMapLength = cardMapLength; - crossMapLength = sizeToCardMapIndex (s->heap.size); + crossMapLength = sizeToCardMapIndex (size); crossMapSize = align (crossMapLength * CROSS_MAP_ELEM_SIZE, s->sysvals.pageSize); crossMapLength = (GC_crossMapIndex)(crossMapSize / CROSS_MAP_ELEM_SIZE); s->generationalMaps.crossMapLength = crossMapLength; totalMapSize = cardMapSize + crossMapSize; - if (DEBUG_MEM) - fprintf (stderr, "Creating card/cross map of size %s\n", - uintmaxToCommaString(totalMapSize)); + /* The card map starts at the end of the heap. */ s->generationalMaps.cardMap = - GC_mmapAnon_safe (NULL, totalMapSize); + (GC_cardMap) (h->start + size); s->generationalMaps.crossMap = - (s->generationalMaps.cardMap + (cardMapSize / CARD_MAP_ELEM_SIZE)); + (GC_crossMap) (s->generationalMaps.cardMap + (cardMapSize / CARD_MAP_ELEM_SIZE)); + if (DEBUG_MEM or s->controls.messages) + fprintf (stderr, + "[GC: Created card/cross map at "FMTPTR" of size %s bytes.]\n", + (uintptr_t)(s->generationalMaps.cardMap), + uintmaxToCommaString(totalMapSize)); if (DEBUG_CARD_MARKING) fprintf (stderr, "cardMap = "FMTPTR" crossMap = "FMTPTR"\n", (uintptr_t)s->generationalMaps.cardMap, (uintptr_t)s->generationalMaps.crossMap); setCardMapAbsolute (s); - clearCardMap (s); - clearCrossMap (s); } -void resizeCardMapAndCrossMap (GC_state s) { - if (s->mutatorMarksCards - and (s->generationalMaps.cardMapLength * CARD_MAP_ELEM_SIZE) - != align (sizeToCardMapIndex (s->heap.size), s->sysvals.pageSize)) { +void createCardMapAndCrossMap (GC_state s) { + initCardMapAndCrossMap (s, &s->heap, s->heap.size); + if (s->mutatorMarksCards) { + clearCardMap (s); + clearCrossMap (s); + } +} + +/* This function is called before the given heap becomes the new current heap + used to store the program datas. + The 2 heaps can have a different size. */ +void copyCardMapAndCrossMap (GC_state s, GC_heap h) { + if (s->mutatorMarksCards) { GC_cardMap oldCardMap; size_t oldCardMapSize; GC_crossMap oldCrossMap; @@ -176,13 +213,80 @@ oldCardMapSize = s->generationalMaps.cardMapLength * CARD_MAP_ELEM_SIZE; oldCrossMap = s->generationalMaps.crossMap; oldCrossMapSize = s->generationalMaps.crossMapLength * CROSS_MAP_ELEM_SIZE; - createCardMapAndCrossMap (s); + + initCardMapAndCrossMap (s, h, h->size); + + clearCardMap (s); + GC_memcpy ((pointer)oldCardMap, (pointer)s->generationalMaps.cardMap, + min (s->generationalMaps.cardMapLength * CARD_MAP_ELEM_SIZE, + oldCardMapSize)); + clearCrossMap (s); GC_memcpy ((pointer)oldCrossMap, (pointer)s->generationalMaps.crossMap, - min (s->generationalMaps.crossMapLength * CROSS_MAP_ELEM_SIZE, - oldCrossMapSize)); - if (DEBUG_MEM) - fprintf (stderr, "Releasing card/cross map.\n"); - GC_release (oldCardMap, oldCardMapSize + oldCrossMapSize); + min (s->generationalMaps.crossMapLength * CROSS_MAP_ELEM_SIZE, + oldCrossMapSize)); + } +} + +/* This function is called before we shrink the heap buffer which contains the + card/cross map datas. */ +void shrinkCardMapAndCrossMap (GC_state s, size_t keep) { + if (s->mutatorMarksCards) { + GC_crossMap oldCrossMap; + + oldCrossMap = s->generationalMaps.crossMap; + + initCardMapAndCrossMap (s, &s->heap, keep); + + GC_memmove ((pointer)oldCrossMap, (pointer)s->generationalMaps.crossMap, + s->generationalMaps.crossMapLength * CROSS_MAP_ELEM_SIZE); + clearCardMap (s); + } +} + +/* This function is called after we remap the heap buffer which contains the + card/cross map datas. + The remapped heap must be bigger than the original one. */ +void remapCardMapAndCrossMap (GC_state s, pointer orig) { + if (s->mutatorMarksCards) { + GC_cardMap oldCardMap; + size_t oldCardMapSize; + GC_crossMap oldCrossMap; + size_t oldCrossMapSize; + + oldCardMap = (GC_cardMap) ((pointer) s->generationalMaps.cardMap + (s->heap.start - orig)); + oldCardMapSize = s->generationalMaps.cardMapLength * CARD_MAP_ELEM_SIZE; + oldCrossMap = (GC_crossMap) (oldCardMap + (oldCardMapSize / CARD_MAP_ELEM_SIZE)); + oldCrossMapSize = s->generationalMaps.crossMapLength * CROSS_MAP_ELEM_SIZE; + + initCardMapAndCrossMap (s, &s->heap, s->heap.size); + + if (DEBUG_MEM or s->controls.messages) { + fprintf (stderr, "[GC: oldCardMap = "FMTPTR" oldCrossMap = "FMTPTR"]\n", + (uintptr_t)oldCardMap, + (uintptr_t)oldCrossMap); + fprintf (stderr, + "[GC: oldCardMapSize = %s bytes oldCrossMapSize = %s bytes]\n", + uintmaxToCommaString(oldCardMapSize), uintmaxToCommaString(oldCrossMapSize)); + fprintf (stderr, "[GC: cardMap = "FMTPTR" crossMap = "FMTPTR"]\n", + (uintptr_t)s->generationalMaps.cardMap, + (uintptr_t)s->generationalMaps.crossMap); + } + GC_memmove ((pointer)oldCrossMap, (pointer)s->generationalMaps.crossMap, + oldCrossMapSize); + if (DEBUG_MEM or s->controls.messages) { + fprintf (stderr, + "[GC: crossMapSize = %s bytes]\n", + uintmaxToCommaString(s->generationalMaps.crossMapLength * CROSS_MAP_ELEM_SIZE)); + } + memset((pointer)s->generationalMaps.crossMap + oldCrossMapSize, CROSS_MAP_EMPTY, + s->generationalMaps.crossMapLength * CROSS_MAP_ELEM_SIZE - oldCrossMapSize); + if (DEBUG_MEM or s->controls.messages) { + fprintf(stderr, "[GC: cross map OK]\n"); + } + clearCardMap (s); + if (DEBUG_MEM or s->controls.messages) { + fprintf(stderr, "[GC: card map OK]\n"); + } } } diff -Naur mlton/runtime/gc/generational.h mltonp1/runtime/gc/generational.h --- mlton/runtime/gc/generational.h 2008-04-11 19:12:27.000000000 +0200 +++ mltonp1/runtime/gc/generational.h 2008-05-05 01:37:49.000000000 +0200 @@ -76,8 +76,11 @@ static inline void clearCardMap (GC_state s); static inline void clearCrossMap (GC_state s); +static size_t computeCardMapAndCrossMapSize (GC_state s, size_t size); static void createCardMapAndCrossMap (GC_state s); -static void resizeCardMapAndCrossMap (GC_state s); +static void copyCardMapAndCrossMap (GC_state s, GC_heap h); +static void shrinkCardMapAndCrossMap (GC_state s, size_t keep); +static void remapCardMapAndCrossMap (GC_state s, pointer orig); #if ASSERT static bool isCrossMapOk (GC_state s); diff -Naur mlton/runtime/gc/heap.c mltonp1/runtime/gc/heap.c --- mlton/runtime/gc/heap.c 2008-04-11 19:12:27.000000000 +0200 +++ mltonp1/runtime/gc/heap.c 2008-05-05 01:37:57.000000000 +0200 @@ -114,7 +114,7 @@ "[GC: Releasing heap at "FMTPTR" of size %s bytes.]\n", (uintptr_t)(h->start), uintmaxToCommaString(h->size)); - GC_release (h->start, h->size); + GC_release (h->start, h->size + computeCardMapAndCrossMapSize(s, h->size)); initHeap (s, h); } @@ -126,13 +126,17 @@ } keep = align (keep, s->sysvals.pageSize); if (keep < h->size) { + size_t oldMapSize, newMapSize; if (DEBUG or s->controls.messages) fprintf (stderr, "[GC: Shrinking heap at "FMTPTR" of size %s bytes to size %s bytes.]\n", (uintptr_t)(h->start), uintmaxToCommaString(h->size), uintmaxToCommaString(keep)); - GC_decommit (h->start + keep, h->size - keep); + shrinkCardMapAndCrossMap(s, keep); + oldMapSize = computeCardMapAndCrossMapSize(s, h->size); + newMapSize = computeCardMapAndCrossMapSize(s, keep); + GC_decommit (h->start + keep + newMapSize, h->size - keep + oldMapSize - newMapSize); h->size = keep; } } @@ -157,6 +161,7 @@ assert (isHeapInit (h)); if (desiredSize < minSize) desiredSize = minSize; + minSize = align (minSize, s->sysvals.pageSize); desiredSize = align (desiredSize, s->sysvals.pageSize); assert (0 == h->size and NULL == h->start); backoff = (desiredSize - minSize) / 20; @@ -176,6 +181,8 @@ static bool direction = TRUE; unsigned int i; + size_t totalMapSize = computeCardMapAndCrossMapSize (s, h->size); + assert (isAligned (h->size, s->sysvals.pageSize)); for (i = 1; i <= count; i++) { size_t address; @@ -183,7 +190,7 @@ address = (size_t)i * step; if (direction) address = (size_t)0x0 - address; - h->start = GC_mmapAnon ((pointer)address, h->size); + h->start = GC_mmapAnon ((pointer)address, h->size + totalMapSize); if ((void*)-1 == h->start) h->start = (void*)NULL; unless ((void*)NULL == h->start) { @@ -208,6 +215,9 @@ uintmaxToCommaString (backoff), uintmaxToCommaString (minSize)); } + /* Make sure we always try to allocate at least 'minSize' bytes. */ + if (h->size > minSize && (h->size - backoff) < minSize) + backoff = h->size - minSize; } h->size = 0; return FALSE; @@ -237,6 +247,7 @@ #endif assert (minSize <= desiredSize); assert (desiredSize >= h->size); + minSize = align (minSize, s->sysvals.pageSize); desiredSize = align (desiredSize, s->sysvals.pageSize); backoff = (desiredSize - minSize) / 20; if (0 == backoff) @@ -244,16 +255,35 @@ backoff = align (backoff, s->sysvals.pageSize); for (size = desiredSize; size >= minSize; size -= backoff) { pointer new; + size_t oldMapSize = computeCardMapAndCrossMapSize (s, h->size); + size_t newMapSize = computeCardMapAndCrossMapSize (s, size); - new = GC_mremap (h->start, h->size, size); + new = GC_mremap (h->start, h->size + oldMapSize, size + newMapSize); unless ((void*)-1 == new) { h->start = new; h->size = size; if (h->size > s->cumulativeStatistics.maxHeapSizeSeen) s->cumulativeStatistics.maxHeapSizeSeen = h->size; assert (minSize <= h->size and h->size <= desiredSize); + if (DEBUG or s->controls.messages) + fprintf (stderr, + "[GC: Heap remapped at "FMTPTR" with size %s bytes.]\n", + (uintptr_t)(h->start), + uintmaxToCommaString(h->size)); return TRUE; } + if (s->controls.messages) { + fprintf (stderr, + "[GC: Remapping heap with size %s bytes cannot be satisfied,]\n", + uintmaxToCommaString (size)); + fprintf (stderr, + "[GC:\tbacking off by %s bytes with minimum size of %s bytes.]\n", + uintmaxToCommaString (backoff), + uintmaxToCommaString (minSize)); + } + /* Make sure we always try to allocate at least 'minSize' bytes. */ + if (size > minSize && (size - backoff) < minSize) + backoff = size - minSize; } return FALSE; } @@ -272,6 +302,10 @@ size_t size; assert (desiredSize >= s->heap.size); + /* Now the card/cross map is stored at the end of the heap buffer, make sure we + won't actually shrink the heap. */ + if (minSize < s->heap.size) + minSize = s->heap.size; if (DEBUG_RESIZING or s->controls.messages) { fprintf (stderr, "[GC: Growing heap at "FMTPTR" of size %s bytes,]\n", @@ -286,8 +320,10 @@ orig = curHeapp->start; size = curHeapp->oldGenSize; assert (size <= s->heap.size); - if (remapHeap (s, curHeapp, desiredSize, minSize)) + if (remapHeap (s, curHeapp, desiredSize, minSize)) { + remapCardMapAndCrossMap (s, orig); goto done; + } shrinkHeap (s, curHeapp, size); initHeap (s, &newHeap); /* Allocate a space of the desired size. */ @@ -299,21 +335,26 @@ from = curHeapp->start + size; to = newHeap.start + size; remaining = size; + copyCardMapAndCrossMap(s, &newHeap); + GC_decommit(orig + curHeapp->size, computeCardMapAndCrossMapSize(s, curHeapp->size)); copy: assert (remaining == (size_t)(from - curHeapp->start) and from >= curHeapp->start and to >= newHeap.start); if (remaining < COPY_CHUNK_SIZE) { GC_memcpy (orig, newHeap.start, remaining); + GC_release (orig, curHeapp->size); } else { + size_t keep; remaining -= COPY_CHUNK_SIZE; from -= COPY_CHUNK_SIZE; to -= COPY_CHUNK_SIZE; GC_memcpy (from, to, COPY_CHUNK_SIZE); - shrinkHeap (s, curHeapp, remaining); + keep = align (remaining, s->sysvals.pageSize); + GC_decommit (orig + keep, orig + curHeapp->size); + curHeapp->size = keep; goto copy; } - releaseHeap (s, curHeapp); newHeap.oldGenSize = size; *curHeapp = newHeap; } else { @@ -338,6 +379,10 @@ GC_diskBack_read (data, curHeapp->start, size); GC_diskBack_close (data); curHeapp->oldGenSize = size; + if (s->mutatorMarksCards) { + createCardMapAndCrossMap (s); + updateCrossMap (s); + } } else { GC_diskBack_close (data); if (s->controls.messages) @@ -363,13 +409,12 @@ uintmaxToCommaString(s->heap.size)); desiredSize = sizeofHeapDesired (s, minSize, s->heap.size); assert (minSize <= desiredSize); - if (desiredSize <= s->heap.size) + if (desiredSize <= s->heap.size) { shrinkHeap (s, &s->heap, desiredSize); - else { + } else { releaseHeap (s, &s->secondaryHeap); growHeap (s, desiredSize, minSize); } - resizeCardMapAndCrossMap (s); assert (s->heap.size >= minSize); } diff -Naur mlton/runtime/gc/virtual-memory.c mltonp1/runtime/gc/virtual-memory.c --- mlton/runtime/gc/virtual-memory.c 2008-04-11 19:12:28.000000000 +0200 +++ mltonp1/runtime/gc/virtual-memory.c 2008-05-05 01:38:05.000000000 +0200 @@ -12,7 +12,7 @@ result = GC_mmapAnon (p, length); if ((void*)-1 == result) { GC_displayMem (); - die ("Out of memory."); + die ("Out of memory (2). Unable to allocate %s bytes.\n", uintmaxToCommaString(length)); } return result; } @@ -29,3 +29,10 @@ return; memcpy (dst, src, size); } + +static inline void GC_memmove (pointer src, pointer dst, size_t size) { + if (DEBUG_DETAILED) + fprintf (stderr, "GC_memmove ("FMTPTR", "FMTPTR", %"PRIuMAX")\n", + (uintptr_t)src, (uintptr_t)dst, (uintmax_t)size); + memmove (dst, src, size); +} diff -Naur mlton/runtime/gc.h mltonp1/runtime/gc.h --- mlton/runtime/gc.h 2008-04-11 19:12:28.000000000 +0200 +++ mltonp1/runtime/gc.h 2008-05-05 01:38:16.000000000 +0200 @@ -44,8 +44,8 @@ #include "gc/int-inf.h" #include "gc/string.h" #include "gc/object-size.h" -#include "gc/generational.h" #include "gc/heap.h" +#include "gc/generational.h" #include "gc/current.h" #include "gc/foreach.h" #include "gc/translate.h"