--- mlton-r6698.patched/runtime/gc/heap.c 2008-11-03 23:45:47.000000000 +0100 +++ mlton-r6698/runtime/gc/heap.c 2008-11-05 23:53:54.000000000 +0100 @@ -39,6 +39,7 @@ size_t sizeofHeapDesired (GC_state s, size_t liveSize, size_t currentSize) { size_t liveWithMapsSize; size_t res; + size_t maxSize; float withMapsRatio; liveSize = align (liveSize, s->sysvals.pageSize); @@ -82,6 +83,9 @@ res = s->sysvals.ram; } else { /* Required live ratio. */ res = liveSize * s->controls.ratios.markCompact; + /* Avoid overflow */ + if (res < liveSize) + res = SIZE_MAX; /* If the current heap is bigger than res, then shrinking always * sounds like a good idea. However, depending on what pages the * VM keeps around, growing could be very expensive, if it @@ -104,6 +108,16 @@ die ("Out of memory with max heap size %s.", uintmaxToCommaString(s->controls.maxHeap)); } + + /* Avoid overflow. */ + maxSize = align(SIZE_MAX - s->sysvals.pageSize, s->sysvals.pageSize); + maxSize -= sizeofCardMapAndCrossMap(s, maxSize); + if (res > maxSize) + res = maxSize; + if (res < liveSize) + die ("Out of memory with system max heap size %s.", + uintmaxToCommaString(res)); + if (DEBUG_RESIZING) fprintf (stderr, "%s = sizeofHeapDesired (%s, %s)\n", uintmaxToCommaString(res),