1#ifndef HALIDE_RUNTIME_VULKAN_MEMORY_H
2#define HALIDE_RUNTIME_VULKAN_MEMORY_H
54 VkDevice dev, VkPhysicalDevice phys_dev,
66 bool collect(
void *user_context);
67 int release(
void *user_context);
68 int destroy(
void *user_context);
80 return this->physical_device;
83 return this->physical_device_limits;
86 return this->alloc_callbacks;
111 VkDevice dev, VkPhysicalDevice phys_dev,
117 uint32_t select_memory_type(
void *user_context,
118 VkPhysicalDevice physical_device,
124 size_t block_byte_count = 0;
125 size_t block_count = 0;
126 size_t region_byte_count = 0;
127 size_t region_count = 0;
128 void *owner_context =
nullptr;
130 VkDevice device =
nullptr;
131 VkPhysicalDevice physical_device =
nullptr;
142 if (system_allocator.
allocate ==
nullptr) {
143 error(user_context) <<
"VulkanBlockAllocator: Unable to create instance! Missing system allocator interface!\n";
150 if (result ==
nullptr) {
151 error(user_context) <<
"VulkanMemoryAllocator: Failed to create instance! Out of memory!\n";
155 result->initialize(user_context, cfg, dev, phys_dev, system_allocator, alloc_callbacks);
160 if (instance ==
nullptr) {
161 error(user_context) <<
"VulkanBlockAllocator: Unable to destroy instance! Invalide instance pointer!\n";
165 instance->
destroy(user_context);
168 error(user_context) <<
"VulkanBlockAllocator: Unable to destroy instance! Missing system allocator interface!\n";
175int VulkanMemoryAllocator::initialize(
void *user_context,
180 owner_context = user_context;
183 physical_device = phys_dev;
186 region_byte_count = 0;
188 block_byte_count = 0;
190 allocators.
system = system_allocator;
200 if (block_allocator ==
nullptr) {
201 error(user_context) <<
"VulkanMemoryAllocator: Failed to create BlockAllocator! Out of memory?!\n";
214#if defined(HL_VK_DEBUG_MEM)
215 debug(
nullptr) <<
"VulkanMemoryAllocator: Reserving memory ("
216 <<
"user_context=" << user_context <<
" "
217 <<
"block_allocator=" << (
void *)(block_allocator) <<
" "
219 <<
"device=" << (
void *)(device) <<
" "
220 <<
"physical_device=" << (
void *)(physical_device) <<
") ...\n";
223 if ((device ==
nullptr) || (physical_device ==
nullptr)) {
224 error(user_context) <<
"VulkanMemoryAllocator: Unable to reserve memory! Invalid device handle!\n";
228 if (block_allocator ==
nullptr) {
229 error(user_context) <<
"VulkanMemoryAllocator: Unable to reserve memory! Invalid block allocator!\n";
233 return block_allocator->
reserve(
this, request);
237#if defined(HL_VK_DEBUG_MEM)
238 debug(
nullptr) <<
"VulkanMemoryAllocator: Mapping region ("
239 <<
"user_context=" << user_context <<
" "
240 <<
"device=" << (
void *)(device) <<
" "
241 <<
"physical_device=" << (
void *)(physical_device) <<
" "
242 <<
"region=" << (
void *)(region) <<
" "
247 if ((device ==
nullptr) || (physical_device ==
nullptr)) {
248 error(user_context) <<
"VulkanMemoryAllocator: Unable to map memory! Invalid device handle!\n";
252 if (block_allocator ==
nullptr) {
253 error(user_context) <<
"VulkanMemoryAllocator: Unable to map memory! Invalid block allocator!\n";
259 if (region_allocator ==
nullptr) {
260 error(user_context) <<
"VulkanMemoryAllocator: Unable to map region! Invalid region allocator handle!\n";
265 if (block_resource ==
nullptr) {
266 error(user_context) <<
"VulkanMemoryAllocator: Unable to map region! Invalid block resource handle!\n";
270 VkDeviceMemory *device_memory =
reinterpret_cast<VkDeviceMemory *
>(block_resource->
memory.
handle);
271 if (device_memory ==
nullptr) {
272 error(user_context) <<
"VulkanMemoryAllocator: Unable to map region! Invalid device memory handle!\n";
276 void *mapped_ptr =
nullptr;
280 error(user_context) <<
"VulkanMemoryAllocator: Unable to map region! Invalid memory range !\n";
283#if defined(HL_VK_DEBUG_MEM)
284 debug(
nullptr) <<
"VulkanMemoryAllocator: MapMemory ("
285 <<
"user_context=" << user_context <<
"\n"
290 <<
" memory_offset=" << (
uint32_t)memory_offset <<
"\n"
291 <<
" memory_size=" << (
uint32_t)memory_size <<
"\n)\n";
293 VkResult result =
vkMapMemory(device, *device_memory, memory_offset, memory_size, 0, (
void **)(&mapped_ptr));
295 error(user_context) <<
"VulkanMemoryAllocator: Mapping region failed! vkMapMemory returned error code: " << vk_get_error_name(result) <<
"\n";
303#if defined(HL_VK_DEBUG_MEM)
304 debug(
nullptr) <<
"VulkanMemoryAllocator: Unmapping region ("
305 <<
"user_context=" << user_context <<
" "
306 <<
"device=" << (
void *)(device) <<
" "
307 <<
"physical_device=" << (
void *)(physical_device) <<
" "
308 <<
"region=" << (
void *)(region) <<
" "
313 if ((device ==
nullptr) || (physical_device ==
nullptr)) {
314 error(user_context) <<
"VulkanMemoryAllocator: Unable to unmap region! Invalid device handle!\n";
320 if (region_allocator ==
nullptr) {
321 error(user_context) <<
"VulkanMemoryAllocator: Unable to unmap region! Invalid region allocator handle!\n";
326 if (block_resource ==
nullptr) {
327 error(user_context) <<
"VulkanMemoryAllocator: Unable to unmap region! Invalid block resource handle!\n";
331 VkDeviceMemory *device_memory =
reinterpret_cast<VkDeviceMemory *
>(block_resource->
memory.
handle);
332 if (device_memory ==
nullptr) {
333 error(user_context) <<
"VulkanMemoryAllocator: Unable to unmap region! Invalid device memory handle!\n";
342#if defined(HL_VK_DEBUG_MEM)
343 debug(
nullptr) <<
"VulkanMemoryAllocator: Cropping region ("
344 <<
"user_context=" << user_context <<
" "
345 <<
"device=" << (
void *)(device) <<
" "
346 <<
"physical_device=" << (
void *)(physical_device) <<
" "
347 <<
"region=" << (
void *)(region) <<
" "
350 <<
"crop_offset=" << (
int64_t)offset <<
") ...\n";
352 if ((device ==
nullptr) || (physical_device ==
nullptr)) {
353 error(user_context) <<
"VulkanMemoryAllocator: Unable to crop region! Invalid device handle!\n";
359 if (region_allocator ==
nullptr) {
360 error(user_context) <<
"VulkanMemoryAllocator: Unable to unmap region! Invalid region allocator handle!\n";
365 int error_code = region_allocator->
retain(
this, owner);
367 error(user_context) <<
"VulkanMemoryAllocator: Unable to crop region! Failed to retain memory region!\n";
374 error(user_context) <<
"VulkanMemoryAllocator: Unable to create crop! Missing system allocator interface!\n";
381 if (memory_region ==
nullptr) {
382 error(user_context) <<
"VulkanMemoryAllocator: Failed to allocate memory region! Out of memory!\n";
389 memory_region->
handle = (
void *)owner;
391 return memory_region;
395 if (region ==
nullptr) {
396 error(user_context) <<
"VulkanMemoryAllocator: Failed to destroy crop! Invalid memory region!\n";
402 if (region_allocator ==
nullptr) {
403 error(user_context) <<
"VulkanMemoryAllocator: Unable to destroy crop region! Invalid region allocator handle!\n";
408 int error_code = region_allocator->
release(
this, owner);
410 error(user_context) <<
"VulkanBlockAllocator: Unable to destroy crop region! Region allocator failed to release memory region!\n";
417 error(user_context) <<
"VulkanBlockAllocator: Unable to destroy crop region! Missing system allocator interface!\n";
434#if defined(HL_VK_DEBUG_MEM)
435 debug(
nullptr) <<
"VulkanMemoryAllocator: Releasing region ("
436 <<
"user_context=" << user_context <<
" "
437 <<
"region=" << (
void *)(region) <<
" "
441 if ((device ==
nullptr) || (physical_device ==
nullptr)) {
442 error(user_context) <<
"VulkanMemoryAllocator: Unable to release region! Invalid device handle!\n";
445 if (block_allocator ==
nullptr) {
446 error(user_context) <<
"VulkanMemoryAllocator: Unable to release region! Invalid block allocator!\n";
449 return block_allocator->
release(
this, region);
453#if defined(HL_VK_DEBUG_MEM)
454 debug(
nullptr) <<
"VulkanMemoryAllocator: Reclaiming region ("
455 <<
"user_context=" << user_context <<
" "
456 <<
"region=" << (
void *)(region) <<
" "
460 if ((device ==
nullptr) || (physical_device ==
nullptr)) {
461 error(user_context) <<
"VulkanMemoryAllocator: Unable to reclaim region! Invalid device handle!\n";
464 if (block_allocator ==
nullptr) {
465 error(user_context) <<
"VulkanMemoryAllocator: Unable to reclaim region! Invalid block allocator!\n";
468 return block_allocator->
reclaim(
this, region);
472#if defined(HL_VK_DEBUG_MEM)
473 debug(
nullptr) <<
"VulkanMemoryAllocator: Retaining region ("
474 <<
"user_context=" << user_context <<
" "
475 <<
"region=" << (
void *)(region) <<
" "
479 if ((device ==
nullptr) || (physical_device ==
nullptr)) {
480 error(user_context) <<
"VulkanMemoryAllocator: Unable to retain region! Invalid device handle!\n";
483 if (block_allocator ==
nullptr) {
484 error(user_context) <<
"VulkanMemoryAllocator: Unable to retain region! Invalid block allocator!\n";
487 return block_allocator->
retain(
this, region);
491#if defined(HL_VK_DEBUG_MEM)
492 debug(
nullptr) <<
"VulkanMemoryAllocator: Collecting unused memory ("
493 <<
"user_context=" << user_context <<
") ... \n";
495 if ((device ==
nullptr) || (physical_device ==
nullptr) || (block_allocator ==
nullptr)) {
498 return block_allocator->
collect(
this);
502#if defined(HL_VK_DEBUG_MEM)
503 debug(
nullptr) <<
"VulkanMemoryAllocator: Releasing block allocator ("
504 <<
"user_context=" << user_context <<
") ... \n";
506 if ((device ==
nullptr) || (physical_device ==
nullptr)) {
507 error(user_context) <<
"VulkanMemoryAllocator: Unable to release allocator! Invalid device handle!\n";
510 if (block_allocator ==
nullptr) {
511 error(user_context) <<
"VulkanMemoryAllocator: Unable to release allocator! Invalid block allocator!\n";
515 return block_allocator->
release(
this);
519#if defined(HL_VK_DEBUG_MEM)
520 debug(
nullptr) <<
"VulkanMemoryAllocator: Destroying allocator ("
521 <<
"user_context=" << user_context <<
") ... \n";
523 if (block_allocator !=
nullptr) {
524 block_allocator->
destroy(
this);
527 region_byte_count = 0;
529 block_byte_count = 0;
540int VulkanMemoryAllocator::lookup_requirements(
void *user_context,
size_t size,
uint32_t usage_flags,
VkMemoryRequirements *memory_requirements) {
541#if defined(HL_VK_DEBUG_MEM)
542 debug(
nullptr) <<
"VulkanMemoryAllocator: Looking up requirements ("
543 <<
"user_context=" << user_context <<
" "
544 <<
"size=" << (
uint32_t)block->size <<
", "
545 <<
"usage_flags=" << usage_flags <<
") ... \n";
557 VkBuffer buffer = {0};
560#if defined(HL_VK_DEBUG_MEM)
561 debug(
nullptr) <<
"VulkanMemoryAllocator: Failed to create buffer to find requirements!\n\t"
562 <<
"vkCreateBuffer returned: " << vk_get_error_name(result) <<
"\n";
575 if (instance ==
nullptr) {
579 void *user_context = instance->owner_context;
580#if defined(HL_VK_DEBUG_MEM)
581 debug(
nullptr) <<
"VulkanMemoryAllocator: Conforming block request ("
582 <<
"user_context=" << user_context <<
" "
583 <<
"request=" << (
void *)(request) <<
") ... \n";
586 if ((instance->device ==
nullptr) || (instance->physical_device ==
nullptr)) {
587 error(user_context) <<
"VulkanRegionAllocator: Unable to conform block request! Invalid device handle!\n";
592 uint32_t usage_flags = instance->select_memory_usage(user_context, request->
properties);
593 int error_code = instance->lookup_requirements(user_context, request->
size, usage_flags, &memory_requirements);
595 error(user_context) <<
"VulkanRegionAllocator: Failed to conform block request! Unable to lookup requirements!\n";
599#if defined(HL_VK_DEBUG_MEM)
600 debug(
nullptr) <<
"VulkanMemoryAllocator: Block allocated ("
603 <<
"required_size=" << (
uint32_t)memory_requirements.
size <<
", "
606 <<
"dedicated=" << (request->
dedicated ?
"true" :
"false") <<
")\n";
609 request->
size = memory_requirements.
size;
616 if (instance ==
nullptr) {
620 void *user_context = instance->owner_context;
621 if ((instance->device ==
nullptr) || (instance->physical_device ==
nullptr)) {
622 error(user_context) <<
"VulkanBlockAllocator: Unable to deallocate block! Invalid device handle!\n";
626 if (block ==
nullptr) {
627 error(user_context) <<
"VulkanBlockAllocator: Unable to deallocate block! Invalid pointer!\n";
631#if defined(HL_VK_DEBUG_MEM)
632 debug(
nullptr) <<
"VulkanMemoryAllocator: Allocating block ("
633 <<
"user_context=" << user_context <<
" "
634 <<
"block=" << (
void *)(block) <<
" "
636 <<
"dedicated=" << (block->
dedicated ?
"true" :
"false") <<
" "
643 uint32_t memory_type = instance->select_memory_type(user_context, instance->physical_device, block->
properties, 0);
644 if (memory_type == invalid_memory_type) {
645 error(user_context) <<
"VulkanMemoryAllocator: Unable to find appropriate memory type for device!\n";
657 VkDeviceMemory *device_memory = (VkDeviceMemory *)vk_host_malloc(
nullptr,
sizeof(VkDeviceMemory), 0,
VK_SYSTEM_ALLOCATION_SCOPE_OBJECT, instance->alloc_callbacks);
658 if (device_memory ==
nullptr) {
659 debug(
nullptr) <<
"VulkanBlockAllocator: Unable to allocate block! Failed to allocate device memory handle!\n";
665 debug(
nullptr) <<
"VulkanMemoryAllocator: Allocation failed! vkAllocateMemory returned: " << vk_get_error_name(result) <<
"\n";
669 debug(
nullptr) <<
"vkAllocateMemory: Allocated memory for device region (" << (
uint64_t)block->
size <<
" bytes) ...\n";
672 block->
handle = (
void *)device_memory;
673 instance->block_byte_count += block->
size;
674 instance->block_count++;
680 if (instance ==
nullptr) {
684 void *user_context = instance->owner_context;
685#if defined(HL_VK_DEBUG_MEM)
686 debug(
nullptr) <<
"VulkanMemoryAllocator: Deallocating block ("
687 <<
"user_context=" << user_context <<
" "
688 <<
"block=" << (
void *)(block) <<
") ... \n";
691 if ((instance->device ==
nullptr) || (instance->physical_device ==
nullptr)) {
692 error(user_context) <<
"VulkanBlockAllocator: Unable to deallocate block! Invalid device handle!\n";
696 if (block ==
nullptr) {
697 error(user_context) <<
"VulkanBlockAllocator: Unable to deallocate block! Invalid pointer!\n";
701#if defined(HL_VK_DEBUG_MEM)
702 debug(
nullptr) <<
"VulkanBlockAllocator: deallocating block ("
704 <<
"dedicated=" << (block->
dedicated ?
"true" :
"false") <<
" "
710 if (block->
handle ==
nullptr) {
711 error(user_context) <<
"VulkanBlockAllocator: Unable to deallocate block! Invalid handle!\n";
715 VkDeviceMemory *device_memory =
reinterpret_cast<VkDeviceMemory *
>(block->
handle);
716 if (device_memory ==
nullptr) {
717 error(user_context) <<
"VulkanBlockAllocator: Unable to deallocate block! Invalid device memory handle!\n";
721 vkFreeMemory(instance->device, *device_memory, instance->alloc_callbacks);
723 debug(
nullptr) <<
"vkFreeMemory: Deallocated memory for device region (" << (
uint64_t)block->
size <<
" bytes) ...\n";
726 if (instance->block_count > 0) {
727 instance->block_count--;
729 error(
nullptr) <<
"VulkanRegionAllocator: Block counter invalid ... reseting to zero!\n";
730 instance->block_count = 0;
734 instance->block_byte_count -= block->
size;
736 error(
nullptr) <<
"VulkanRegionAllocator: Block byte counter invalid ... reseting to zero!\n";
737 instance->block_byte_count = 0;
741 vk_host_free(
nullptr, device_memory, instance->alloc_callbacks);
742 device_memory =
nullptr;
751 return block_byte_count;
754uint32_t VulkanMemoryAllocator::select_memory_type(
void *user_context,
755 VkPhysicalDevice physical_device,
778 error(
nullptr) <<
"VulkanMemoryAllocator: Unable to convert type! Invalid memory visibility request!\n\t"
780 return invalid_memory_type;
804 error(user_context) <<
"VulkanMemoryAllocator: Unable to convert type! Invalid memory caching request!\n\t"
806 return invalid_memory_type;
812 uint32_t result = invalid_memory_type;
816 if (required_flags) {
817 if (((required_flags >> i) & 1) == 0) {
824 if ((properties & need_flags) != need_flags) {
830 if ((properties & want_flags) != want_flags) {
839 if (result == invalid_memory_type) {
840 error(user_context) <<
"VulkanBlockAllocator: Failed to find appropriate memory type for given properties:\n\t"
844 return invalid_memory_type;
862 int error_code = lookup_requirements(user_context, request->
size, usage_flags, &memory_requirements);
864 error(user_context) <<
"VulkanRegionAllocator: Failed to conform block request! Unable to lookup requirements!\n";
868#if defined(HL_VK_DEBUG_MEM)
869 debug(
nullptr) <<
"VulkanMemoryAllocator: Buffer requirements ("
870 <<
"requested_size=" << (
uint32_t)region->size <<
", "
872 <<
"required_size=" << (
uint32_t)memory_requirements.
size <<
")\n";
877 if ((request->
alignment % this->physical_device_limits.minStorageBufferOffsetAlignment) != 0) {
881 if ((request->
alignment % this->physical_device_limits.minUniformBufferOffsetAlignment) != 0) {
895#if defined(HL_VK_DEBUG_MEM)
896 if ((request->
size != actual_size) || (request->
alignment != actual_alignment) || (request->
offset != actual_offset)) {
897 debug(
nullptr) <<
"VulkanMemoryAllocator: Adjusting request to match requirements (\n"
901 <<
" required.size = " << (
uint64_t)memory_requirements.
size <<
",\n"
902 <<
" required.alignment = " << (
uint64_t)memory_requirements.
alignment <<
"\n)\n";
905 request->
size = actual_size;
907 request->
offset = actual_offset;
915 if (instance ==
nullptr) {
919 void *user_context = instance->owner_context;
920#if defined(HL_VK_DEBUG_MEM)
921 debug(
nullptr) <<
"VulkanMemoryAllocator: Conforming region request ("
922 <<
"user_context=" << user_context <<
" "
923 <<
"request=" << (
void *)(region) <<
") ... \n";
926 if ((instance->device ==
nullptr) || (instance->physical_device ==
nullptr)) {
927 error(user_context) <<
"VulkanRegionAllocator: Unable to conform region request! Invalid device handle!\n";
931#if defined(HL_VK_DEBUG_MEM)
932 debug(
nullptr) <<
"VulkanRegionAllocator: Conforming region request ("
935 <<
"dedicated=" << (request->
dedicated ?
"true" :
"false") <<
" "
941 return instance->
conform(user_context, request);
947 if (instance ==
nullptr) {
951 void *user_context = instance->owner_context;
952#if defined(HL_VK_DEBUG_MEM)
953 debug(
nullptr) <<
"VulkanMemoryAllocator: Allocating region ("
954 <<
"user_context=" << user_context <<
" "
955 <<
"region=" << (
void *)(region) <<
") ... \n";
958 if ((instance->device ==
nullptr) || (instance->physical_device ==
nullptr)) {
959 error(user_context) <<
"VulkanRegionAllocator: Unable to allocate region! Invalid device handle!\n";
963 if (region ==
nullptr) {
964 error(user_context) <<
"VulkanRegionAllocator: Unable to allocate region! Invalid pointer!\n";
968#if defined(HL_VK_DEBUG_MEM)
969 debug(
nullptr) <<
"VulkanRegionAllocator: Allocating region ("
972 <<
"dedicated=" << (region->
dedicated ?
"true" :
"false") <<
" "
978 uint32_t usage_flags = instance->select_memory_usage(user_context, region->
properties);
990 if (buffer ==
nullptr) {
991 error(user_context) <<
"VulkanRegionAllocator: Unable to allocate region! Failed to allocate buffer handle!\n";
997 error(user_context) <<
"VulkanRegionAllocator: Failed to create buffer!\n\t"
998 <<
"vkCreateBuffer returned: " << vk_get_error_name(result) <<
"\n";
1010#if defined(HL_VK_DEBUG_MEM)
1011 debug(
nullptr) <<
"VulkanMemoryAllocator: Buffer requirements ("
1014 <<
"required_size=" << (
uint32_t)memory_requirements.
size <<
")\n";
1017 if (memory_requirements.
size > region->
size) {
1018 vkDestroyBuffer(instance->device, *buffer, instance->alloc_callbacks);
1020 debug(
nullptr) <<
"VulkanMemoryAllocator: Reallocating buffer to match required size ("
1023 create_info.
size = memory_requirements.
size;
1026 error(user_context) <<
"VulkanRegionAllocator: Failed to recreate buffer!\n\t"
1027 <<
"vkCreateBuffer returned: " << vk_get_error_name(result) <<
"\n";
1033 debug(
nullptr) <<
"vkCreateBuffer: Created buffer for device region (" << (
uint64_t)region->
size <<
" bytes) ...\n";
1037 if (region_allocator ==
nullptr) {
1038 error(user_context) <<
"VulkanBlockAllocator: Unable to allocate region! Invalid region allocator!\n";
1043 if (block_resource ==
nullptr) {
1044 error(user_context) <<
"VulkanBlockAllocator: Unable to allocate region! Invalid block resource handle!\n";
1048 VkDeviceMemory *device_memory =
reinterpret_cast<VkDeviceMemory *
>(block_resource->
memory.
handle);
1049 if (device_memory ==
nullptr) {
1050 error(user_context) <<
"VulkanBlockAllocator: Unable to allocate region! Invalid device memory handle!\n";
1057 error(user_context) <<
"VulkanRegionAllocator: Failed to bind buffer!\n\t"
1058 <<
"vkBindBufferMemory returned: " << vk_get_error_name(result) <<
"\n";
1062 region->
handle = (
void *)buffer;
1064 instance->region_byte_count += region->
size;
1065 instance->region_count++;
1071 if (instance ==
nullptr) {
1075 void *user_context = instance->owner_context;
1076#if defined(HL_VK_DEBUG_MEM)
1077 debug(
nullptr) <<
"VulkanMemoryAllocator: Deallocating region ("
1078 <<
"user_context=" << user_context <<
" "
1079 <<
"region=" << (
void *)(region) <<
") ... \n";
1082 if ((instance->device ==
nullptr) || (instance->physical_device ==
nullptr)) {
1083 error(user_context) <<
"VulkanRegionAllocator: Unable to deallocate region! Invalid device handle!\n";
1087 if (region ==
nullptr) {
1088 error(user_context) <<
"VulkanRegionAllocator: Unable to deallocate region! Invalid pointer!\n";
1092#if defined(HL_VK_DEBUG_MEM)
1093 debug(
nullptr) <<
"VulkanRegionAllocator: Deallocating region ("
1096 <<
"dedicated=" << (region->
dedicated ?
"true" :
"false") <<
" "
1102 if (region->
handle ==
nullptr) {
1103 error(user_context) <<
"VulkanRegionAllocator: Unable to deallocate region! Invalid handle!\n";
1107 VkBuffer *buffer =
reinterpret_cast<VkBuffer *
>(region->
handle);
1108 if (buffer ==
nullptr) {
1109 error(user_context) <<
"VulkanRegionAllocator: Unable to deallocate region! Invalid buffer handle!\n";
1113 vkDestroyBuffer(instance->device, *buffer, instance->alloc_callbacks);
1115 debug(
nullptr) <<
"vkDestroyBuffer: Destroyed buffer for device region (" << (
uint64_t)region->
size <<
" bytes) ...\n";
1117 region->
handle =
nullptr;
1118 if (instance->region_count > 0) {
1119 instance->region_count--;
1121 error(
nullptr) <<
"VulkanRegionAllocator: Region counter invalid ... reseting to zero!\n";
1122 instance->region_count = 0;
1127 instance->region_byte_count -= region->
size;
1129 error(
nullptr) <<
"VulkanRegionAllocator: Region byte counter invalid ... reseting to zero!\n";
1130 instance->region_byte_count = 0;
1133 vk_host_free(
nullptr, buffer, instance->alloc_callbacks);
1139 return region_count;
1143 return region_byte_count;
1148 switch (properties.
usage) {
1168 error(user_context) <<
"VulkanRegionAllocator: Unable to convert type! Invalid memory usage request!\n\t"
1170 return invalid_usage_flags;
1173 if (result == invalid_usage_flags) {
1174 error(user_context) <<
"VulkanRegionAllocator: Failed to find appropriate memory usage for given properties:\n\t"
1178 return invalid_usage_flags;
1190void *vk_system_malloc(
void *user_context,
size_t size) {
1194void vk_system_free(
void *user_context,
void *ptr) {
1201 return callbacks->
pfnAllocation(user_context, size, alignment, scope);
1203 return vk_system_malloc(user_context, size);
1209 return callbacks->
pfnFree(user_context, ptr);
1211 return vk_system_free(user_context, ptr);
1215VulkanMemoryAllocator *vk_create_memory_allocator(
void *user_context,
1217 VkPhysicalDevice physical_device,
1230 const char *
alloc_config = vk_get_alloc_config_internal(user_context);
1234 if (alloc_config_values.
size() > 0) {
1235 config.maximum_pool_size =
atoi(alloc_config_values[0]) * 1024 * 1024;
1236 print(user_context) <<
"Vulkan: Configuring allocator with " << (
uint32_t)config.maximum_pool_size <<
" for maximum pool size (in bytes)\n";
1238 if (alloc_config_values.
size() > 1) {
1239 config.minimum_block_size =
atoi(alloc_config_values[1]) * 1024 * 1024;
1240 print(user_context) <<
"Vulkan: Configuring allocator with " << (
uint32_t)config.minimum_block_size <<
" for minimum block size (in bytes)\n";
1242 if (alloc_config_values.
size() > 2) {
1243 config.maximum_block_size =
atoi(alloc_config_values[2]) * 1024 * 1024;
1244 print(user_context) <<
"Vulkan: Configuring allocator with " << (
uint32_t)config.maximum_block_size <<
" for maximum block size (in bytes)\n";
1246 if (alloc_config_values.
size() > 3) {
1247 config.maximum_block_count =
atoi(alloc_config_values[3]);
1248 print(user_context) <<
"Vulkan: Configuring allocator with " << (
uint32_t)config.maximum_block_count <<
" for maximum block count\n";
1250 if (alloc_config_values.
size() > 4) {
1251 config.nearest_multiple =
atoi(alloc_config_values[4]);
1252 print(user_context) <<
"Vulkan: Configuring allocator with " << (
uint32_t)config.nearest_multiple <<
" for nearest multiple\n";
1257 config, device, physical_device,
1258 system_allocator, alloc_callbacks);
1261int vk_destroy_memory_allocator(
void *user_context, VulkanMemoryAllocator *allocator) {
1262 if (allocator !=
nullptr) {
1264 allocator =
nullptr;
1271int vk_clear_device_buffer(
void *user_context,
1272 VulkanMemoryAllocator *allocator,
1273 VkCommandPool command_pool,
1274 VkQueue command_queue,
1275 VkBuffer device_buffer) {
1279 <<
" vk_clear_device_buffer (user_context: " << user_context <<
", "
1280 <<
"allocator: " << (
void *)allocator <<
", "
1281 <<
"command_pool: " << (
void *)command_pool <<
", "
1282 <<
"command_queue: " << (
void *)command_queue <<
", "
1283 <<
"device_buffer: " << (
void *)device_buffer <<
")\n";
1287 VkCommandBuffer command_buffer;
1288 int error_code = vk_create_command_buffer(user_context, allocator, command_pool, &command_buffer);
1290 error(user_context) <<
"Vulkan: Failed to create command buffer!\n";
1305 error(user_context) <<
"Vulkan: vkBeginCommandBuffer returned " << vk_get_error_name(result) <<
"\n";
1315 error(user_context) <<
"Vulkan: vkEndCommandBuffer returned " << vk_get_error_name(result) <<
"\n";
1335 error(user_context) <<
"Vulkan: vkQueueSubmit returned " << vk_get_error_name(result) <<
"\n";
1342 error(user_context) <<
"Vulkan: vkQueueWaitIdle returned " << vk_get_error_name(result) <<
"\n";
1346 error_code = vk_destroy_command_buffer(user_context, allocator, command_pool, command_buffer);
1348 error(user_context) <<
"Vulkan: Failed to destroy command buffer!\n";
1372 custom_allocation_callbacks = callbacks;
1378 return custom_allocation_callbacks;
@ halide_error_code_internal_error
There is a bug in the Halide compiler.
@ halide_error_code_generic_error
An uncategorized error occurred.
@ halide_error_code_success
There was no error.
@ halide_error_code_device_malloc_failed
The Halide runtime encountered an error while trying to allocate memory on device.
@ halide_error_code_out_of_memory
A call to halide_malloc returned NULL.
Allocator class interface for managing large contiguous blocks of memory, which are then sub-allocate...
bool collect(void *user_context)
MemoryRegion * reserve(void *user_context, const MemoryRequest &request)
int release(void *user_context, MemoryRegion *region)
int retain(void *user_context, MemoryRegion *region)
static void destroy(void *user_context, BlockAllocator *block_allocator)
static BlockAllocator * create(void *user_context, const Config &config, const MemoryAllocators &allocators)
int reclaim(void *user_context, MemoryRegion *region)
const MemoryAllocators & current_allocators() const
Allocator class interface for sub-allocating a contiguous memory block into smaller regions of memory...
static RegionAllocator * find_allocator(void *user_context, MemoryRegion *memory_region)
int retain(void *user_context, MemoryRegion *memory_region)
int release(void *user_context, MemoryRegion *memory_region)
BlockResource * block_resource() const
size_t parse(void *user_context, const char *str, const char *delim)
Vulkan Memory Allocator class interface for managing large memory requests stored as contiguous block...
int reclaim(void *user_context, MemoryRegion *region)
static const VulkanMemoryConfig & default_config()
MemoryRegion * reserve(void *user_context, const MemoryRequest &request)
VulkanMemoryAllocator(const VulkanMemoryAllocator &)=delete
int release(void *user_context, MemoryRegion *region)
static int allocate_block(void *instance_ptr, MemoryBlock *block)
int conform(void *user_context, MemoryRequest *request)
static int deallocate_region(void *instance_ptr, MemoryRegion *region)
VulkanMemoryAllocator & operator=(const VulkanMemoryAllocator &)=delete
static int destroy(void *user_context, VulkanMemoryAllocator *allocator)
int unmap(void *user_context, MemoryRegion *region)
static int conform_region_request(void *instance_ptr, MemoryRequest *request)
~VulkanMemoryAllocator()=delete
MemoryRegion * create_crop(void *user_context, MemoryRegion *region, uint64_t offset)
int destroy_crop(void *user_context, MemoryRegion *region)
size_t regions_allocated() const
size_t blocks_allocated() const
static int allocate_region(void *instance_ptr, MemoryRegion *region)
static int conform_block_request(void *instance_ptr, MemoryRequest *request)
MemoryRegion * owner_of(void *user_context, MemoryRegion *region)
VkDevice current_device() const
VkPhysicalDeviceLimits current_physical_device_limits() const
bool collect(void *user_context)
size_t bytes_allocated_for_blocks() const
const VkAllocationCallbacks * callbacks() const
static int deallocate_block(void *instance_ptr, MemoryBlock *block)
VulkanMemoryAllocator()=delete
void * map(void *user_context, MemoryRegion *region)
VkPhysicalDevice current_physical_device() const
size_t bytes_allocated_for_regions() const
int retain(void *user_context, MemoryRegion *region)
static VulkanMemoryAllocator * create(void *user_context, const VulkanMemoryConfig &config, VkDevice dev, VkPhysicalDevice phys_dev, const SystemMemoryAllocatorFns &system_allocator, const VkAllocationCallbacks *alloc_callbacks=nullptr)
WEAK const char * halide_memory_caching_name(MemoryCaching value)
WEAK const char * halide_memory_usage_name(MemoryUsage value)
WEAK const char * halide_memory_visibility_name(MemoryVisibility value)
VKAPI_ATTR VkResult VKAPI_CALL vkBindBufferMemory(VkDevice device, VkBuffer buffer, VkDeviceMemory memory, VkDeviceSize memoryOffset)
VkFlags VkMemoryPropertyFlags
VKAPI_ATTR void VKAPI_CALL vkCmdFillBuffer(VkCommandBuffer commandBuffer, VkBuffer dstBuffer, VkDeviceSize dstOffset, VkDeviceSize size, uint32_t data)
VKAPI_ATTR VkResult VKAPI_CALL vkAllocateMemory(VkDevice device, const VkMemoryAllocateInfo *pAllocateInfo, const VkAllocationCallbacks *pAllocator, VkDeviceMemory *pMemory)
#define VK_MAX_MEMORY_TYPES
@ VK_SHARING_MODE_EXCLUSIVE
@ VK_MEMORY_PROPERTY_HOST_COHERENT_BIT
@ VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT
@ VK_MEMORY_PROPERTY_HOST_CACHED_BIT
@ VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT
VKAPI_ATTR void VKAPI_CALL vkGetBufferMemoryRequirements(VkDevice device, VkBuffer buffer, VkMemoryRequirements *pMemoryRequirements)
@ VK_SYSTEM_ALLOCATION_SCOPE_OBJECT
VKAPI_ATTR VkResult VKAPI_CALL vkQueueWaitIdle(VkQueue queue)
VKAPI_ATTR void VKAPI_CALL vkFreeMemory(VkDevice device, VkDeviceMemory memory, const VkAllocationCallbacks *pAllocator)
@ VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT
VKAPI_ATTR void VKAPI_CALL vkGetPhysicalDeviceProperties(VkPhysicalDevice physicalDevice, VkPhysicalDeviceProperties *pProperties)
VKAPI_ATTR void VKAPI_CALL vkGetPhysicalDeviceMemoryProperties(VkPhysicalDevice physicalDevice, VkPhysicalDeviceMemoryProperties *pMemoryProperties)
VKAPI_ATTR VkResult VKAPI_CALL vkCreateBuffer(VkDevice device, const VkBufferCreateInfo *pCreateInfo, const VkAllocationCallbacks *pAllocator, VkBuffer *pBuffer)
VKAPI_ATTR VkResult VKAPI_CALL vkQueueSubmit(VkQueue queue, uint32_t submitCount, const VkSubmitInfo *pSubmits, VkFence fence)
VKAPI_ATTR VkResult VKAPI_CALL vkMapMemory(VkDevice device, VkDeviceMemory memory, VkDeviceSize offset, VkDeviceSize size, VkMemoryMapFlags flags, void **ppData)
@ VK_BUFFER_USAGE_TRANSFER_DST_BIT
@ VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT
@ VK_BUFFER_USAGE_STORAGE_BUFFER_BIT
@ VK_BUFFER_USAGE_TRANSFER_SRC_BIT
VKAPI_ATTR void VKAPI_CALL vkUnmapMemory(VkDevice device, VkDeviceMemory memory)
VKAPI_ATTR VkResult VKAPI_CALL vkEndCommandBuffer(VkCommandBuffer commandBuffer)
VKAPI_ATTR void VKAPI_CALL vkDestroyBuffer(VkDevice device, VkBuffer buffer, const VkAllocationCallbacks *pAllocator)
@ VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO
@ VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO
@ VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO
@ VK_STRUCTURE_TYPE_SUBMIT_INFO
VKAPI_ATTR VkResult VKAPI_CALL vkBeginCommandBuffer(VkCommandBuffer commandBuffer, const VkCommandBufferBeginInfo *pBeginInfo)
WEAK VulkanMemoryConfig memory_allocator_config
WEAK const VkAllocationCallbacks * custom_allocation_callbacks
WEAK ScopedSpinLock::AtomicFlag custom_allocation_callbacks_lock
WEAK char alloc_config[1024]
ALWAYS_INLINE size_t conform_alignment(size_t requested, size_t required)
ALWAYS_INLINE size_t conform_size(size_t offset, size_t size, size_t alignment, size_t nearest_multiple)
ALWAYS_INLINE size_t aligned_offset(size_t offset, size_t alignment)
This file defines the class FunctionDAG, which is our representation of a Halide pipeline,...
@ Internal
Not visible externally, similar to 'static' linkage in C.
Expr print(const std::vector< Expr > &values)
Create an Expr that prints out its value whenever it is evaluated.
unsigned __INT64_TYPE__ uint64_t
signed __INT64_TYPE__ int64_t
void * memcpy(void *s1, const void *s2, size_t n)
void * memset(void *s, int val, size_t n)
unsigned __INT32_TYPE__ uint32_t
size_t maximum_block_count
size_t minimum_block_size
size_t maximum_block_size
MemoryRegionAllocatorFns region
MemoryBlockAllocatorFns block
SystemMemoryAllocatorFns system
MemoryProperties properties
MemoryVisibility visibility
MemoryProperties properties
MemoryProperties properties
static bool is_empty(const char *str)
DeallocateSystemFn deallocate
AllocateSystemFn allocate
size_t maximum_block_size
size_t minimum_block_size
size_t maximum_block_count
PFN_vkAllocationFunction pfnAllocation
PFN_vkFreeFunction pfnFree
VkMemoryPropertyFlags propertyFlags
VkDeviceSize minStorageBufferOffsetAlignment
VkDeviceSize minUniformBufferOffsetAlignment
VkMemoryType memoryTypes[VK_MAX_MEMORY_TYPES]
VkPhysicalDeviceLimits limits
WEAK void halide_vulkan_set_allocation_callbacks(const VkAllocationCallbacks *callbacks)
WEAK const VkAllocationCallbacks * halide_vulkan_get_allocation_callbacks(void *user_context)