improve rezalloc and rezalloc aligned, issue #763

Author: daanx

include/mimalloc/internal.h

@@ -390,7 +390,7 @@ static inline bool _mi_is_aligned(void* p, size_t alignment) {
 // Align upwards
 static inline uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) {
   mi_assert_internal(alignment != 0);
-  uintptr_t mask = alignment - 1;
+  const uintptr_t mask = alignment - 1;
   if ((alignment & mask) == 0) {  // power of two?
     return ((sz + mask) & ~mask);
   }
@@ -399,12 +399,27 @@ static inline uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) {
   }
 }
 
-
 // Align a pointer upwards
 static inline void* _mi_align_up_ptr(const void* p, size_t alignment) {
   return (void*)_mi_align_up((uintptr_t)p, alignment);
 }
 
+// Align down
+static inline uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) {
+  mi_assert_internal(alignment != 0);
+  const uintptr_t mask = alignment - 1;
+  if ((alignment & mask) == 0) {  // power of two?
+    return (sz & ~mask);
+  }
+  else {
+    return ((sz/alignment)*alignment);
+  }
+}
+
+// Align a pointer downwards
+static inline void* _mi_align_down_ptr(const void* p, size_t alignment) {
+  return (void*)_mi_align_down((uintptr_t)p, alignment);
+}
 
 // Divide upwards: `s <= _mi_divide_up(s,d)*d < s+d`.
 static inline uintptr_t _mi_divide_up(uintptr_t size, size_t divider) {

src/alloc-aligned.c

@@ -301,17 +301,18 @@ static void* mi_heap_realloc_zero_aligned_at(mi_heap_t* heap, void* p, size_t ne
     return p;  // reallocation still fits, is aligned and not more than 50% waste
   }
   else {
-    // note: we don't zero allocate upfront so we only zero initialize the expanded part
+    // note: we don't zero allocate upfront so we only zero initialize the expanded part (at the cost of calling mi_usable_size)
     void* const newp = mi_heap_malloc_aligned_at(heap,newsize,alignment,offset);
     if (newp != NULL) {
-      const size_t usable = mi_usable_size(newp);
+      const size_t copy_size  = (newsize > size ? size : newsize);
+      const size_t zero_start = (copy_size >= sizeof(intptr_t) ? copy_size - sizeof(intptr_t) : 0); // also set last word in the previous allocation to zero to ensure any padding is zero-initialized    
+      const size_t usable     = mi_usable_size(newp);
       mi_assert_internal(usable >= newsize); // use usable for zero'ing, issue #763
-      if (zero && usable > size) {
+      if (zero && usable > zero_start) {
         // also set last word in the previous allocation to zero to ensure any padding is zero-initialized
-        const size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0);
-        _mi_memzero((uint8_t*)newp + start, usable - start);
+        _mi_memzero((uint8_t*)newp + zero_start, usable - zero_start);
       }
-      _mi_memcpy(newp, p, (newsize > size ? size : newsize)); // cannot be aligned due to abitrary offset... (todo: require offset to be a multiple of sizeof(void*)?)
+      _mi_memcpy(newp, p, copy_size); // cannot be aligned due to abitrary offset... (todo: require offset to be a multiple of sizeof(void*)?)
       mi_free(p); // only free if successful
     }
     return newp;

src/alloc.c

@@ -310,23 +310,26 @@ void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero,
     if (usable_post!=NULL) { *usable_post = mi_page_usable_block_size(page); }
     return p;  // reallocation still fits and not more than 50% waste
   }
-  // note: we don't zero allocate upfront so we only zero initialize the expanded part
-  void* const newp = mi_heap_umalloc(heap,newsize,usable_post);
+  // note: we don't zero allocate upfront so we only zero initialize the expanded part 
+  size_t usable; // use usable for zero-ing, issue #763
+  void* const newp = mi_heap_umalloc(heap,newsize,&usable);
+  if (usable_post!=NULL) { *usable_post = usable; }
   if mi_likely(newp != NULL) {
-    const size_t usable = mi_usable_size(newp);
-    mi_assert_internal(usable >= newsize); // use usable for zero-ing, issue #763
-    if (zero && usable > size) {
-      // also set last word in the previous allocation to zero to ensure any padding is zero-initialized
-      const size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0);      
-      _mi_memzero((uint8_t*)newp + start, usable - start);
+    const size_t copy_size  = (newsize > size ? size : newsize);
+    const size_t zero_start = _mi_align_down( (copy_size >= sizeof(intptr_t) ? copy_size - sizeof(intptr_t) : 0), sizeof(intptr_t)); // also set last word in the previous allocation to zero to ensure any padding is zero-initialized
+    #if MI_PADDING
+    usable = mi_usable_size(newp); // avoid zero'ing padding
+    #endif
+    mi_assert_internal(usable >= newsize); 
+    if (zero && usable > zero_start) {      
+      _mi_memzero_aligned((uint8_t*)newp + zero_start, usable - zero_start);
     }
     else if (newsize == 0) {
       ((uint8_t*)newp)[0] = 0; // work around for applications that expect zero-reallocation to be zero initialized (issue #725)
     }
     if mi_likely(p != NULL) {
-      const size_t copysize = (newsize > size ? size : newsize);
-      mi_track_mem_defined(p,copysize);  // _mi_useable_size may be too large for byte precise memory tracking..
-      _mi_memcpy(newp, p, copysize);
+      mi_track_mem_defined(p,copy_size);  // _mi_useable_size may be too large for byte precise memory tracking..
+      _mi_memcpy_aligned(newp, p, copy_size);
       mi_free(p); // only free the original pointer if successful
     }
   }

src/os.c

@@ -91,22 +91,6 @@ void _mi_os_init(void) {
 bool _mi_os_decommit(void* addr, size_t size);
 bool _mi_os_commit(void* addr, size_t size, bool* is_zero);
 
-static inline uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) {
-  mi_assert_internal(alignment != 0);
-  uintptr_t mask = alignment - 1;
-  if ((alignment & mask) == 0) { // power of two?
-    return (sz & ~mask);
-  }
-  else {
-    return ((sz / alignment) * alignment);
-  }
-}
-
-static void* _mi_align_down_ptr(void* p, size_t alignment) {
-  return (void*)_mi_align_down((uintptr_t)p, alignment);
-}
-
-
 /* -----------------------------------------------------------
   aligned hinting
 -------------------------------------------------------------- */