#ifndef PK_MEM_H #define PK_MEM_H #include "./pkmem-types.h" /* deleteme */ #include "./pkmacros.h" /* deleteme */ #include #include #ifndef PK_MEM_DEFAULT_BUCKET_SIZE # define PK_MEM_DEFAULT_BUCKET_SIZE (1ULL * 1024ULL * 1024ULL * 256ULL) #endif size_t pk_mem_bucket_calculate_size(size_t sz, size_t reserved_block_count); struct pk_membucket* pk_mem_bucket_create(const char* description, int64_t sz, enum PK_MEMBUCKET_FLAGS flags); void pk_mem_bucket_debug_print(struct pk_membucket *bkt); void pk_mem_bucket_destroy(struct pk_membucket* bkt); void pk_mem_bucket_reset(struct pk_membucket* bkt); void pk_mem_bucket_set_client_mem_bucket(struct pk_membucket *bkt); bool pk_mem_bucket_ptr_is_in_mem_bucket(const void* ptr, const struct pk_membucket* bkt); void* pk_new_base(size_t sz, size_t alignment); void* pk_new_bkt(size_t sz, size_t alignment, struct pk_membucket* bkt); void* pk_new(size_t sz, size_t alignment, struct pk_membucket* bkt); void pk_delete_base(const void* ptr, size_t sz); void pk_delete_bkt(const void* ptr, size_t sz, struct pk_membucket* bkt); void pk_delete(const void* ptr, size_t sz, struct pk_membucket* bkt); #if defined(__cplusplus) #include #include static inline void stupid_header_warnings_cpp() { (void)std::is_const::value; } template inline T* pk_new(pk_membucket* bucket = NULL) { void* ptr = NULL; if (bucket) { ptr = pk_new_bkt(sizeof(T), alignof(T), bucket); } else { ptr = pk_new_base(sizeof(T), alignof(T)); } if IS_CONSTRUCTIBLE(T) { return new (ptr) T{}; } return reinterpret_cast(ptr); } template inline T* pk_new_arr(long count, pk_membucket* bucket = NULL) { char* ptr = NULL; if (bucket) { ptr = static_cast(pk_new_bkt(sizeof(T) * count, alignof(T), bucket)); } else { ptr = static_cast(pk_new_base(sizeof(T) * count, alignof(T))); } if (ptr == NULL) return NULL; if IS_CONSTRUCTIBLE(T) { for (long i = 0; i < count; ++i) { new (ptr + (i * sizeof(T))) T{}; } } return reinterpret_cast(ptr); } template inline void pk_delete(const T* ptr, pk_membucket* bucket = NULL) { if IS_DESTRUCTIBLE(T) { reinterpret_cast(ptr)->~T(); } if (bucket) { return pk_delete_bkt(static_cast(ptr), sizeof(T), bucket); } else { return pk_delete_base(static_cast(ptr), sizeof(T)); } } template inline void pk_delete_arr(const T* ptr, long count, pk_membucket* bucket = NULL) { if IS_DESTRUCTIBLE(T) { for (long i = 0; i < count; ++i) { reinterpret_cast(reinterpret_cast(ptr) + (i * sizeof(T)))->~T(); } } if (bucket) { return pk_delete_bkt(static_cast(ptr), sizeof(T) * count, bucket); } else { return pk_delete_base(static_cast(ptr), sizeof(T) * count); } } #endif /* __cplusplus */ #endif /* PK_MEM */ #ifdef PK_IMPL_MEM #include #include #include #include static inline void pkmem_stupid_header_warnings() { (void)stdout; } #if defined(PK_MEMORY_DEBUGGER) #endif #ifndef PK_MINIMUM_ALIGNMENT # define PK_MINIMUM_ALIGNMENT 1 #endif #ifndef PK_MEMORY_DEBUGGER_MAX_BUCKET_COUNT #define PK_MEMORY_DEBUGGER_MAX_BUCKET_COUNT 16 #endif #define EXPECTED_PK_MEMBLOCK_SIZE 128 #define pk_memblock_blocks_idx(bkt, idx) ((bkt->block_capacity-1)-(idx)) #define pk_bkt_data(bkt) ((char*)bkt + EXPECTED_PK_MEMBLOCK_SIZE) #define pk_bkt_head(bkt) ((&bkt->data[0]) + bkt->head) #define pk_bkt_data_sz(bkt) (size_t)((char*)&bkt->blocks[0] - &bkt->data[0]) struct pk_memblock { union { char* data; void* ptr; }; size_t size; }; struct pk_membucket { // 00 mtx_t mtx; // 40 // the total size of the bucket, struct+data size_t size; // 48 // the current head of the bucket: byte offset from `data`. // All currently alloc'd data is before this offset size_t head; // 56 uint32_t block_capacity; // 60 uint32_t block_head_l; // 64 // this should ALWAYS point to the last block containing unalloced space in bkt uint32_t block_head_r; // 68 // the number of active allocations from this bucket // -should correlate to blocks that have a sz > 0 uint32_t alloc_count; // 72 struct pk_memblock *blocks; // 80 enum PK_MEMBUCKET_FLAGS flags; // 88 const char *description; // 96 #ifdef PK_MEMORY_DEBUGGER struct pk_memblock *debug_blocks; // 104 uint32_t debug_head_l; // 108 uint32_t debug_head_r; // 112 uint32_t debug_block_capacity; // 116 char padding[(8*1)+4]; #else char padding[(8*4)]; #endif // 128 // starting point for alloc'd data // data[] is illegal in c++ (though it works in gcc/clang, but so does this) char data[1]; }; static struct pk_membucket *client_bucket = NULL; size_t pk_mem_bucket_calculate_size(size_t sz, size_t reserved_block_count) { size_t base_size = EXPECTED_PK_MEMBLOCK_SIZE + sz + (sizeof(struct pk_memblock) * reserved_block_count); // This trick ensures that our array of pk_memblocks at the end is mem-aligned. // We do, however, still have to do the math when setting the ptr. // Why? the user may have strict memory requirements and didn't call this function. return base_size + (64 - (base_size % 64)); } bool pk_mem_bucket_ptr_is_in_mem_bucket(const void* ptr, const struct pk_membucket* bkt) { return (ptr >= (void*)bkt && ptr < (void*)pk_bkt_head(bkt)); } void pk_mem_bucket_debug_print(struct pk_membucket *bkt) { PK_LOG_INF("pk_membucket details:\n"); PK_LOGV_INF("\tbkt: %p\n", (void *)bkt); PK_LOGV_INF("\tdescription: %s\n", bkt->description); PK_LOGV_INF("\tsize: %lu\n", bkt->size); PK_LOGV_INF("\thead: %lu\n", bkt->head); PK_LOGV_INF("\tallocs: %u\n", bkt->alloc_count); PK_LOGV_INF("\tblock head_l: %u\n", bkt->block_head_l); PK_LOGV_INF("\tblock head_r: %u\n", bkt->block_head_r); PK_LOGV_INF("\tflags: %lu\n", bkt->flags); #ifdef PK_MEMORY_DEBUGGER PK_LOGV_INF("\tdebug alloc head_l: %u\n", bkt->debug_head_l); PK_LOGV_INF("\tdebug alloc head_r: %u\n", bkt->debug_head_r); PK_LOGV_INF("\tdebug cappacity: %u\n", bkt->debug_block_capacity); #endif } struct pk_membucket* pk_mem_bucket_create(const char* description, int64_t sz, enum PK_MEMBUCKET_FLAGS flags) { // 512 example: // [000-127] pk_membucket // [128-191] 64 bytes of data LOL // [192-511] 20 pk_memblocks (20 is worst-case, start 16, 4 per 64 bytes) if ((sz % 64) > 0) { sz += 64 - (sz % 64); } assert(sz >= 512 && "[pkmem.h] bucket too small to track allocation data"); struct pk_membucket* bkt = (struct pk_membucket*)aligned_alloc(64, sz); if (bkt == NULL) return NULL; mtx_init(&bkt->mtx, mtx_plain); bkt->size = sz; bkt->head = 0; bkt->block_capacity = 16; bkt->block_head_l = 0; bkt->block_head_r = 0; bkt->alloc_count = 0; bkt->flags = flags; bkt->description = description; char* blocks_addr = (char*)bkt + sz - (sizeof(struct pk_memblock) * bkt->block_capacity); blocks_addr -= (size_t)blocks_addr % 64; bkt->blocks = (struct pk_memblock*)blocks_addr; bkt->block_capacity = (size_t)(((char*)bkt + sz) - blocks_addr) / sizeof(struct pk_memblock); bkt->blocks[pk_memblock_blocks_idx(bkt,0)].size = pk_bkt_data_sz(bkt); bkt->blocks[pk_memblock_blocks_idx(bkt,0)].ptr = pk_bkt_data(bkt); #ifdef PK_MEMORY_DEBUGGER bkt->debug_head_l = 0; bkt->debug_head_r = 0; bkt->debug_block_capacity = 128; bkt->debug_blocks = (struct pk_memblock*)aligned_alloc(alignof(struct pk_memblock), sizeof(struct pk_memblock) * 128); bkt->debug_blocks[0].ptr = NULL; bkt->debug_blocks[0].size = 0; #endif return bkt; } void pk_mem_bucket_destroy(struct pk_membucket* bkt) { assert(bkt != NULL); #ifdef PK_MEMORY_DEBUGGER if (bkt->debug_blocks != NULL) free(bkt->debug_blocks); #endif free(bkt); } void pk_mem_bucket_reset(struct pk_membucket* bkt) { if (PK_HAS_FLAG(bkt->flags, PK_MEMBUCKET_FLAG_TRANSIENT) == false) { PK_LOG_ERR("WARNING: pk_bucket_reset called on non-transient pk_membucket\n"); } bkt->head = 0; bkt->block_capacity = 16; char* blocks_addr = (char*)bkt + bkt->size - (sizeof(struct pk_memblock) * bkt->block_capacity); blocks_addr -= (size_t)blocks_addr % 64; bkt->blocks = (struct pk_memblock*)blocks_addr; bkt->block_capacity = (size_t)(((char*)bkt + bkt->size) - blocks_addr) / sizeof(struct pk_memblock); bkt->block_head_l = 0; bkt->block_head_r = 0; bkt->alloc_count = 0; bkt->blocks[pk_memblock_blocks_idx(bkt,0)].size = pk_bkt_data_sz(bkt); bkt->blocks[pk_memblock_blocks_idx(bkt,0)].ptr = pk_bkt_data(bkt); #ifdef PK_MEMORY_DEBUGGER bkt->debug_head_l = 0; bkt->debug_head_r = 0; bkt->debug_blocks[0].ptr = NULL; bkt->debug_blocks[0].size = 0; #endif } void pk_mem_bucket_set_client_mem_bucket(struct pk_membucket *bkt) { client_bucket = bkt; } void pk_bucket_insert_block(struct pk_membucket* bkt, const struct pk_memblock* block) { // 2025-06-03 JCB // Note that this function should only be called if we're INSERTING. // This means that the block will never go at the END of the list - that would be an append. // It can, however, be placed at the beginning, in which case the entire array shifts. struct pk_memblock* new_block = NULL; struct pk_memblock* old_block = NULL; size_t i, k; // 1. resize if needed if (bkt->block_head_r+1 == bkt->block_capacity) { if (bkt->blocks[pk_memblock_blocks_idx(bkt, bkt->block_head_r)].size < sizeof(struct pk_memblock)) { PK_LOG_ERR("[pkmem.h] bkt out of memory when expanding memory blocks."); exit(1); } // this is all that needs done, arr can just grow like this bkt->blocks[pk_memblock_blocks_idx(bkt, bkt->block_head_r)].size -= sizeof(struct pk_memblock); bkt->block_capacity += 1; bkt->blocks -= 1; } // 2. move all blocks forward until we pass the pointer // reminder that these blocks are in REVERSE order for (i = bkt->block_head_r+1; i > 0; --i) { k = pk_memblock_blocks_idx(bkt, i); new_block = &bkt->blocks[k]; old_block = new_block+1; *new_block = *old_block; if (old_block->data < block->data) { break; } } assert(old_block != NULL); if (i == 0 && old_block != NULL) { *old_block = *block; } else { *new_block = *block; } bkt->block_head_r += 1; } void pk_bucket_collapse_blocks(struct pk_membucket* bkt) { // 1. loop through from (rev_idx)0 to head_r, shifting any blocks that have size 0 struct pk_memblock* new_block; struct pk_memblock* old_block; size_t i, ii, bhr; // there's an off by one annoynce here // start with ii = 0 // if we start with ii = 1, we might subtract from block_head_r when nothing was shifted for (i = 0, ii = 0, bhr = bkt->block_head_r; (i + ii) <= bhr; ++i) { new_block = &bkt->blocks[pk_memblock_blocks_idx(bkt, i)]; if (new_block->size > 0) continue; do { old_block = new_block - ii; if (old_block->size == 0) { ii+=1; } else { break; } } while (i + ii <= bhr); *new_block = *old_block; old_block->size = 0; old_block->ptr = NULL; } bkt->block_head_r -= ii; } void* pk_new_bkt(size_t sz, size_t alignment, struct pk_membucket* bkt) { #ifdef PK_MEMORY_FORCE_MALLOC return malloc(sz); #endif if (sz == 0) return NULL; if (bkt == NULL) return NULL; // TODO some type of error handling if ((bkt->size - bkt->head) < (sz + alignment - 1)) return NULL; size_t i, k; size_t calculatedAlignment = alignment < PK_MINIMUM_ALIGNMENT ? PK_MINIMUM_ALIGNMENT : alignment; size_t misalignment = 0; struct pk_memblock tmp_blk; struct pk_memblock* block = NULL; void* data = NULL; mtx_lock(&bkt->mtx); // find block for (i = 0; i <= bkt->block_head_r; ++i) { k = pk_memblock_blocks_idx(bkt, i); tmp_blk = bkt->blocks[k]; misalignment = (size_t)(tmp_blk.data) % calculatedAlignment; misalignment = (calculatedAlignment - misalignment) % calculatedAlignment; if (tmp_blk.size < sz + misalignment) { continue; } block = &bkt->blocks[k]; break; } if (block == NULL) { mtx_unlock(&bkt->mtx); assert(block != NULL && "memory corruption: not enough space in chosen bkt"); } data = block->data + misalignment; #ifdef PK_MEMORY_DEBUGGER size_t ii; if (PK_HAS_FLAG(bkt->flags, PK_MEMBUCKET_FLAG_TRANSIENT) == false) { for (i = 0; i < bkt->debug_head_r; ++i) { assert((bkt->debug_blocks[i].size == 0 || (void*)(bkt->debug_blocks[i].data) != data) && "mem address alloc'd twice!"); } i = bkt->debug_head_l; if (bkt->debug_head_l == bkt->debug_head_r) { bkt->debug_head_l++; bkt->debug_head_r++; if (bkt->debug_head_r == bkt->debug_block_capacity) { struct pk_memblock *debug_blocks; debug_blocks = (struct pk_memblock*)aligned_alloc(alignof(struct pk_memblock), sizeof(struct pk_memblock) * (bkt->debug_block_capacity + 128)); assert(debug_blocks != NULL); memcpy(debug_blocks, bkt->debug_blocks, sizeof(struct pk_memblock) * bkt->debug_block_capacity); free(bkt->debug_blocks); bkt->debug_blocks = debug_blocks; bkt->debug_block_capacity += 128; } bkt->debug_blocks[bkt->debug_head_r].ptr = NULL; bkt->debug_blocks[bkt->debug_head_r].size = 0; } else { // 2025-06-05 JCB // This intentionally looks at debug_head_r, which could potentially // be uninitialized. I added some logic elsewhere to ensure that // whenever debug_head_r is incremented, we set the related block // to NULL/0 so that this will catch size==0. // I was experiencing an issue where in testing it was initialized to // NULL/0, but then in a client application it was garbage data. for (ii = bkt->debug_head_l+1; ii <= bkt->debug_head_r; ++ii) { if (bkt->debug_blocks[ii].size == 0) { bkt->debug_head_l = ii; break; } } assert(ii != bkt->debug_head_r+1); } assert(bkt->debug_head_l <= bkt->debug_head_r); bkt->debug_blocks[i].data = (char*)data; bkt->debug_blocks[i].size = sz; } #endif if (block->data == pk_bkt_head(bkt)) { bkt->head += (sz + misalignment); } tmp_blk.data = block->data; tmp_blk.size = misalignment; block->data += misalignment + sz; block->size -= misalignment + sz; if (tmp_blk.size > 0) { pk_bucket_insert_block(bkt, &tmp_blk); } pk_bucket_collapse_blocks(bkt); bkt->alloc_count++; assert(data >= (void*)pk_bkt_data(bkt) && "allocated data is before bucket data"); assert((char*)data <= pk_bkt_data(bkt) + bkt->size && "allocated data is after bucket data"); #ifdef PK_MEMORY_DEBUGGER if (PK_HAS_FLAG(bkt->flags, PK_MEMBUCKET_FLAG_TRANSIENT) == false) { size_t k; int64_t debug_tracked_alloc_size = 0; int64_t debug_bucket_alloc_size = pk_bkt_data_sz(bkt); for (i = 0; i < bkt->debug_head_r; ++i) { debug_tracked_alloc_size += bkt->debug_blocks[i].size; } for (i = 0; i <= bkt->block_head_r; ++i) { k = pk_memblock_blocks_idx(bkt, i); debug_bucket_alloc_size -= bkt->blocks[k].size; } assert(debug_tracked_alloc_size == debug_bucket_alloc_size && "allocation size mismatch!"); } #endif mtx_unlock(&bkt->mtx); memset(data, 0, sz); return data; } void* pk_new_base(size_t sz, size_t alignment) { if (client_bucket == NULL) return NULL; return pk_new_bkt(sz, alignment, client_bucket); } void* pk_new(size_t sz, size_t alignment, struct pk_membucket* bkt) { if (bkt != NULL) return pk_new_bkt(sz, alignment, bkt); return pk_new_base(sz, alignment); } void pk_delete_bkt(const void* ptr, size_t sz, struct pk_membucket* bkt) { #ifdef PK_MEMORY_FORCE_MALLOC #if defined(__cplusplus) std::free(const_cast(ptr)); #else free((void*)ptr); #endif return; #endif size_t i, k; mtx_lock(&bkt->mtx); assert(bkt->alloc_count > 0); assert(pk_mem_bucket_ptr_is_in_mem_bucket(ptr, bkt) && "pointer not in memory bucket range"); assert(sz > 0 && "attempted to free pointer of size 0"); #ifdef PK_MEMORY_DEBUGGER bool found = PK_HAS_FLAG(bkt->flags, PK_MEMBUCKET_FLAG_TRANSIENT); struct pk_memblock *debug_memblocks = bkt->debug_blocks; struct pk_memblock *mb; if (found == false) { for (i = bkt->debug_head_r+1; i > 0; --i) { mb = &debug_memblocks[i-1]; if (mb->size == 0) continue; if ((void*)(mb->ptr) == ptr) { assert(mb->size == sz && "[pkmem.h] incorrect free size"); mb->ptr = NULL; mb->size = 0; found = true; if (i <= bkt->debug_head_l) { bkt->debug_head_l = i-1; } if (i == bkt->debug_head_r+1) { if (bkt->debug_head_l == bkt->debug_head_r) { bkt->debug_head_l--; } bkt->debug_head_r--; } assert(bkt->debug_head_l <= bkt->debug_head_r); break; } } } assert(found && "[pkmem.h] double free or invalid ptr"); #endif bkt->alloc_count--; if (bkt->alloc_count == 0) { bkt->head = 0; bkt->block_head_l = 0; bkt->block_head_r = 0; bkt->blocks[pk_memblock_blocks_idx(bkt, 0)].data = pk_bkt_data(bkt); bkt->blocks[pk_memblock_blocks_idx(bkt, 0)].size = pk_bkt_data_sz(bkt); #ifdef PK_MEMORY_DEBUGGER bkt->debug_head_l = 0; bkt->debug_head_r = 0; bkt->debug_blocks[0].data = NULL; bkt->debug_blocks[0].size = 0; #endif mtx_unlock(&bkt->mtx); return; } char* afterPtr = ((char*)(ptr))+sz; struct pk_memblock* tmp_blk = NULL; struct pk_memblock* beforeBlk = NULL; struct pk_memblock* afterBlk = NULL; for (i = bkt->block_head_r+1; i > 0 ; --i) { k = pk_memblock_blocks_idx(bkt, (i-1)); tmp_blk = &bkt->blocks[k]; if (tmp_blk->data + tmp_blk->size == ptr) { beforeBlk = tmp_blk; break; } if (i <= bkt->block_head_r+1 && tmp_blk->data == afterPtr) { afterBlk = tmp_blk; continue; } if (tmp_blk->data < (char*)ptr) { break; } } if (ptr == &bkt->data[0] && afterBlk == NULL && bkt->blocks[pk_memblock_blocks_idx(bkt, 0)].data == afterPtr) { afterBlk = &bkt->blocks[pk_memblock_blocks_idx(bkt, 0)]; } if (afterBlk != NULL && afterBlk->data == pk_bkt_head(bkt)) { bkt->head -= sz; if (beforeBlk != NULL) { bkt->head -= beforeBlk->size; } } if (beforeBlk == NULL && afterBlk == NULL) { struct pk_memblock newBlock; memset(&newBlock, 0, sizeof(struct pk_memblock)); newBlock.data = (char*)ptr; newBlock.size = sz; pk_bucket_insert_block(bkt, &newBlock); } else if (beforeBlk != NULL && afterBlk != NULL) { beforeBlk->size += sz + afterBlk->size; if (beforeBlk->data == pk_bkt_head(bkt)) { bkt->block_head_r--; } afterBlk->size = 0; afterBlk->data = NULL; } else if (beforeBlk != NULL) { beforeBlk->size += sz; } else if (afterBlk != NULL) { afterBlk->data -= sz; afterBlk->size += sz; } pk_bucket_collapse_blocks(bkt); #ifdef PK_MEMORY_DEBUGGER if (PK_HAS_FLAG(bkt->flags, PK_MEMBUCKET_FLAG_TRANSIENT) == false) { int64_t debug_tracked_alloc_size = 0; int64_t debug_bucket_alloc_size = pk_bkt_data_sz(bkt); for (i = 0; i < bkt->debug_head_r; ++i) { debug_tracked_alloc_size += bkt->debug_blocks[i].size; } for (i = 0; i <= bkt->block_head_r; ++i) { k = pk_memblock_blocks_idx(bkt, i); debug_bucket_alloc_size -= bkt->blocks[k].size; } assert(debug_tracked_alloc_size == debug_bucket_alloc_size && "allocation size mismatch!"); } #endif mtx_unlock(&bkt->mtx); } void pk_delete_base(const void* ptr, size_t sz) { pk_delete_bkt(ptr, sz, client_bucket); } void pk_delete(const void* ptr, size_t sz, struct pk_membucket* bkt) { if (bkt != NULL) { pk_delete_bkt(ptr, sz, bkt); return; } pk_delete_base(ptr, sz); return; } #endif /* PK_IMPL_MEM */