Halide 19.0.0
Halide compiler and libraries
Loading...
Searching...
No Matches
vulkan_memory.h
Go to the documentation of this file.
1#ifndef HALIDE_RUNTIME_VULKAN_MEMORY_H
2#define HALIDE_RUNTIME_VULKAN_MEMORY_H
3
6#include "vulkan_internal.h"
7
8// Uncomment to enable verbose memory allocation debugging
9// #define HL_VK_DEBUG_MEM 1
10
11namespace Halide {
12namespace Runtime {
13namespace Internal {
14namespace Vulkan {
15
16// --------------------------------------------------------------------------
17
18// Enable external client to override Vulkan allocation callbacks (if they so desire)
20WEAK const VkAllocationCallbacks *custom_allocation_callbacks = nullptr; // nullptr => use Vulkan runtime implementation
21
22// --------------------------------------------------------------------------
23
24// Runtime configuration parameters to adjust the behaviour of the block allocator
26 size_t maximum_pool_size = 0; //< Maximum number of bytes to allocate for the entire pool (including all blocks). Specified in bytes. Zero means no constraint
27 size_t minimum_block_size = 32 * 1024 * 1024; //< Default block size is 32MB
28 size_t maximum_block_size = 0; //< Specified in bytes. Zero means no constraint
29 size_t maximum_block_count = 0; //< Maximum number of blocks to allocate. Zero means no constraint
30 size_t nearest_multiple = 32; //< Always round up the requested region sizes to the given integer value. Zero means no constraint
31};
33
34// --------------------------------------------------------------------------
35
36/** Vulkan Memory Allocator class interface for managing large
37 * memory requests stored as contiguous blocks of memory, which
38 * are then sub-allocated into smaller regions of
39 * memory to avoid the excessive cost of vkAllocate and the limited
40 * number of available allocation calls through the API.
41 */
43public:
44 // disable copy constructors and assignment
47
48 // disable non-factory constrction
51
52 // Factory methods for creation / destruction
53 static VulkanMemoryAllocator *create(void *user_context, const VulkanMemoryConfig &config,
54 VkDevice dev, VkPhysicalDevice phys_dev,
55 const SystemMemoryAllocatorFns &system_allocator,
56 const VkAllocationCallbacks *alloc_callbacks = nullptr);
57
58 static int destroy(void *user_context, VulkanMemoryAllocator *allocator);
59
60 // Public interface methods
61 MemoryRegion *reserve(void *user_context, const MemoryRequest &request);
62 int conform(void *user_context, MemoryRequest *request); //< conforms the given memory request into one that can be allocated
63 int release(void *user_context, MemoryRegion *region); //< unmark and cache the region for reuse
64 int reclaim(void *user_context, MemoryRegion *region); //< free the region and consolidate
65 int retain(void *user_context, MemoryRegion *region); //< retain the region and increase its use count
66 bool collect(void *user_context); //< returns true if any blocks were removed
67 int release(void *user_context);
68 int destroy(void *user_context);
69
70 void *map(void *user_context, MemoryRegion *region);
71 int unmap(void *user_context, MemoryRegion *region);
72 MemoryRegion *create_crop(void *user_context, MemoryRegion *region, uint64_t offset);
73 int destroy_crop(void *user_context, MemoryRegion *region);
74 MemoryRegion *owner_of(void *user_context, MemoryRegion *region);
75
76 VkDevice current_device() const {
77 return this->device;
78 }
79 VkPhysicalDevice current_physical_device() const {
80 return this->physical_device;
81 }
83 return this->physical_device_limits;
84 }
86 return this->alloc_callbacks;
87 }
88
89 static const VulkanMemoryConfig &default_config();
90
91 static int allocate_block(void *instance_ptr, MemoryBlock *block);
92 static int deallocate_block(void *instance_ptr, MemoryBlock *block);
93 static int conform_block_request(void *instance_ptr, MemoryRequest *request);
94
95 static int allocate_region(void *instance_ptr, MemoryRegion *region);
96 static int deallocate_region(void *instance_ptr, MemoryRegion *region);
97 static int conform_region_request(void *instance_ptr, MemoryRequest *request);
98
99 size_t bytes_allocated_for_blocks() const;
100 size_t blocks_allocated() const;
101
102 size_t bytes_allocated_for_regions() const;
103 size_t regions_allocated() const;
104
105private:
106 static constexpr uint32_t invalid_usage_flags = uint32_t(-1);
107 static constexpr uint32_t invalid_memory_type = uint32_t(VK_MAX_MEMORY_TYPES);
108
109 // Initializes a new instance
110 int initialize(void *user_context, const VulkanMemoryConfig &config,
111 VkDevice dev, VkPhysicalDevice phys_dev,
112 const SystemMemoryAllocatorFns &system_allocator,
113 const VkAllocationCallbacks *alloc_callbacks = nullptr);
114
115 uint32_t select_memory_usage(void *user_context, MemoryProperties properties) const;
116
117 uint32_t select_memory_type(void *user_context,
118 VkPhysicalDevice physical_device,
119 MemoryProperties properties,
120 uint32_t required_flags) const;
121
122 int lookup_requirements(void *user_context, size_t size, uint32_t usage_flags, VkMemoryRequirements *memory_requirements);
123
124 size_t block_byte_count = 0;
125 size_t block_count = 0;
126 size_t region_byte_count = 0;
127 size_t region_count = 0;
128 void *owner_context = nullptr;
129 VulkanMemoryConfig config;
130 VkDevice device = nullptr;
131 VkPhysicalDevice physical_device = nullptr;
132 VkPhysicalDeviceLimits physical_device_limits = {};
133 const VkAllocationCallbacks *alloc_callbacks = nullptr;
134 BlockAllocator *block_allocator = nullptr;
135};
136
138 const VulkanMemoryConfig &cfg, VkDevice dev, VkPhysicalDevice phys_dev,
139 const SystemMemoryAllocatorFns &system_allocator,
140 const VkAllocationCallbacks *alloc_callbacks) {
141
142 if (system_allocator.allocate == nullptr) {
143 error(user_context) << "VulkanBlockAllocator: Unable to create instance! Missing system allocator interface!\n";
144 return nullptr;
145 }
146
147 VulkanMemoryAllocator *result = reinterpret_cast<VulkanMemoryAllocator *>(
148 system_allocator.allocate(user_context, sizeof(VulkanMemoryAllocator)));
149
150 if (result == nullptr) {
151 error(user_context) << "VulkanMemoryAllocator: Failed to create instance! Out of memory!\n";
152 return nullptr; // caller must handle error case for out-of-memory
153 }
154
155 result->initialize(user_context, cfg, dev, phys_dev, system_allocator, alloc_callbacks);
156 return result;
157}
158
159int VulkanMemoryAllocator::destroy(void *user_context, VulkanMemoryAllocator *instance) {
160 if (instance == nullptr) {
161 error(user_context) << "VulkanBlockAllocator: Unable to destroy instance! Invalide instance pointer!\n";
163 }
164 const BlockAllocator::MemoryAllocators &allocators = instance->block_allocator->current_allocators();
165 instance->destroy(user_context);
166 BlockAllocator::destroy(user_context, instance->block_allocator);
167 if (allocators.system.deallocate == nullptr) {
168 error(user_context) << "VulkanBlockAllocator: Unable to destroy instance! Missing system allocator interface!\n";
170 }
171 allocators.system.deallocate(user_context, instance);
173}
174
175int VulkanMemoryAllocator::initialize(void *user_context,
176 const VulkanMemoryConfig &cfg, VkDevice dev, VkPhysicalDevice phys_dev,
177 const SystemMemoryAllocatorFns &system_allocator,
178 const VkAllocationCallbacks *callbacks) {
179
180 owner_context = user_context;
181 config = cfg;
182 device = dev;
183 physical_device = phys_dev;
184 alloc_callbacks = callbacks;
185 region_count = 0;
186 region_byte_count = 0;
187 block_count = 0;
188 block_byte_count = 0;
190 allocators.system = system_allocator;
193 BlockAllocator::Config block_allocator_config = {0};
194 block_allocator_config.maximum_pool_size = cfg.maximum_pool_size;
195 block_allocator_config.maximum_block_count = cfg.maximum_block_count;
196 block_allocator_config.maximum_block_size = cfg.maximum_block_size;
197 block_allocator_config.minimum_block_size = cfg.minimum_block_size;
198 block_allocator_config.nearest_multiple = cfg.nearest_multiple;
199 block_allocator = BlockAllocator::create(user_context, block_allocator_config, allocators);
200 if (block_allocator == nullptr) {
201 error(user_context) << "VulkanMemoryAllocator: Failed to create BlockAllocator! Out of memory?!\n";
203 }
204
205 // get the physical device properties to determine limits and allocation requirements
206 VkPhysicalDeviceProperties physical_device_properties = {0};
207 memset(&physical_device_limits, 0, sizeof(VkPhysicalDeviceLimits));
208 vkGetPhysicalDeviceProperties(physical_device, &physical_device_properties);
209 memcpy(&physical_device_limits, &(physical_device_properties.limits), sizeof(VkPhysicalDeviceLimits));
211}
212
213MemoryRegion *VulkanMemoryAllocator::reserve(void *user_context, const MemoryRequest &request) {
214#if defined(HL_VK_DEBUG_MEM)
215 debug(nullptr) << "VulkanMemoryAllocator: Reserving memory ("
216 << "user_context=" << user_context << " "
217 << "block_allocator=" << (void *)(block_allocator) << " "
218 << "request_size=" << (uint32_t)(request.size) << " "
219 << "device=" << (void *)(device) << " "
220 << "physical_device=" << (void *)(physical_device) << ") ...\n";
221#endif
222
223 if ((device == nullptr) || (physical_device == nullptr)) {
224 error(user_context) << "VulkanMemoryAllocator: Unable to reserve memory! Invalid device handle!\n";
225 return nullptr;
226 }
227
228 if (block_allocator == nullptr) {
229 error(user_context) << "VulkanMemoryAllocator: Unable to reserve memory! Invalid block allocator!\n";
230 return nullptr;
231 }
232
233 return block_allocator->reserve(this, request);
234}
235
236void *VulkanMemoryAllocator::map(void *user_context, MemoryRegion *region) {
237#if defined(HL_VK_DEBUG_MEM)
238 debug(nullptr) << "VulkanMemoryAllocator: Mapping region ("
239 << "user_context=" << user_context << " "
240 << "device=" << (void *)(device) << " "
241 << "physical_device=" << (void *)(physical_device) << " "
242 << "region=" << (void *)(region) << " "
243 << "region_size=" << (uint32_t)region->size << " "
244 << "region_offset=" << (uint32_t)region->offset << " "
245 << "crop_offset=" << (uint32_t)region->range.head_offset << ") ...\n";
246#endif
247 if ((device == nullptr) || (physical_device == nullptr)) {
248 error(user_context) << "VulkanMemoryAllocator: Unable to map memory! Invalid device handle!\n";
249 return nullptr;
250 }
251
252 if (block_allocator == nullptr) {
253 error(user_context) << "VulkanMemoryAllocator: Unable to map memory! Invalid block allocator!\n";
254 return nullptr;
255 }
256
257 MemoryRegion *owner = owner_of(user_context, region);
258 RegionAllocator *region_allocator = RegionAllocator::find_allocator(user_context, owner);
259 if (region_allocator == nullptr) {
260 error(user_context) << "VulkanMemoryAllocator: Unable to map region! Invalid region allocator handle!\n";
261 return nullptr; // NOTE: caller must handle nullptr
262 }
263
264 BlockResource *block_resource = region_allocator->block_resource();
265 if (block_resource == nullptr) {
266 error(user_context) << "VulkanMemoryAllocator: Unable to map region! Invalid block resource handle!\n";
267 return nullptr; // NOTE: caller must handle nullptr
268 }
269
270 VkDeviceMemory *device_memory = reinterpret_cast<VkDeviceMemory *>(block_resource->memory.handle);
271 if (device_memory == nullptr) {
272 error(user_context) << "VulkanMemoryAllocator: Unable to map region! Invalid device memory handle!\n";
273 return nullptr; // NOTE: caller must handle nullptr
274 }
275
276 void *mapped_ptr = nullptr;
277 VkDeviceSize memory_offset = region->offset + region->range.head_offset;
278 VkDeviceSize memory_size = region->size - region->range.tail_offset - region->range.head_offset;
279 if (((double)region->size - (double)region->range.tail_offset - (double)region->range.head_offset) <= 0.0) {
280 error(user_context) << "VulkanMemoryAllocator: Unable to map region! Invalid memory range !\n";
281 return nullptr;
282 }
283#if defined(HL_VK_DEBUG_MEM)
284 debug(nullptr) << "VulkanMemoryAllocator: MapMemory ("
285 << "user_context=" << user_context << "\n"
286 << " region_size=" << (uint32_t)region->size << "\n"
287 << " region_offset=" << (uint32_t)region->offset << "\n"
288 << " region_range.head_offset=" << (uint32_t)region->range.head_offset << "\n"
289 << " region_range.tail_offset=" << (uint32_t)region->range.tail_offset << "\n"
290 << " memory_offset=" << (uint32_t)memory_offset << "\n"
291 << " memory_size=" << (uint32_t)memory_size << "\n)\n";
292#endif
293 VkResult result = vkMapMemory(device, *device_memory, memory_offset, memory_size, 0, (void **)(&mapped_ptr));
294 if (result != VK_SUCCESS) {
295 error(user_context) << "VulkanMemoryAllocator: Mapping region failed! vkMapMemory returned error code: " << vk_get_error_name(result) << "\n";
296 return nullptr;
297 }
298
299 return mapped_ptr;
300}
301
302int VulkanMemoryAllocator::unmap(void *user_context, MemoryRegion *region) {
303#if defined(HL_VK_DEBUG_MEM)
304 debug(nullptr) << "VulkanMemoryAllocator: Unmapping region ("
305 << "user_context=" << user_context << " "
306 << "device=" << (void *)(device) << " "
307 << "physical_device=" << (void *)(physical_device) << " "
308 << "region=" << (void *)(region) << " "
309 << "region_size=" << (uint32_t)region->size << " "
310 << "region_offset=" << (uint32_t)region->offset << " "
311 << "crop_offset=" << (uint32_t)region->range.head_offset << ") ...\n";
312#endif
313 if ((device == nullptr) || (physical_device == nullptr)) {
314 error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid device handle!\n";
316 }
317
318 MemoryRegion *owner = owner_of(user_context, region);
319 RegionAllocator *region_allocator = RegionAllocator::find_allocator(user_context, owner);
320 if (region_allocator == nullptr) {
321 error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid region allocator handle!\n";
323 }
324
325 BlockResource *block_resource = region_allocator->block_resource();
326 if (block_resource == nullptr) {
327 error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid block resource handle!\n";
329 }
330
331 VkDeviceMemory *device_memory = reinterpret_cast<VkDeviceMemory *>(block_resource->memory.handle);
332 if (device_memory == nullptr) {
333 error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid device memory handle!\n";
335 }
336
337 vkUnmapMemory(device, *device_memory);
339}
340
342#if defined(HL_VK_DEBUG_MEM)
343 debug(nullptr) << "VulkanMemoryAllocator: Cropping region ("
344 << "user_context=" << user_context << " "
345 << "device=" << (void *)(device) << " "
346 << "physical_device=" << (void *)(physical_device) << " "
347 << "region=" << (void *)(region) << " "
348 << "region_size=" << (uint32_t)region->size << " "
349 << "region_offset=" << (uint32_t)region->offset << " "
350 << "crop_offset=" << (int64_t)offset << ") ...\n";
351#endif
352 if ((device == nullptr) || (physical_device == nullptr)) {
353 error(user_context) << "VulkanMemoryAllocator: Unable to crop region! Invalid device handle!\n";
354 return nullptr;
355 }
356
357 MemoryRegion *owner = owner_of(user_context, region);
358 RegionAllocator *region_allocator = RegionAllocator::find_allocator(user_context, owner);
359 if (region_allocator == nullptr) {
360 error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid region allocator handle!\n";
361 return nullptr; // NOTE: caller must handle nullptr
362 }
363
364 // increment usage count
365 int error_code = region_allocator->retain(this, owner);
366 if (error_code != halide_error_code_success) {
367 error(user_context) << "VulkanMemoryAllocator: Unable to crop region! Failed to retain memory region!\n";
368 return nullptr; // NOTE: caller must handle nullptr
369 }
370
371 // create a new region to return, and copy all the other region's properties
372 const BlockAllocator::MemoryAllocators &allocators = block_allocator->current_allocators();
373 if (allocators.system.allocate == nullptr) {
374 error(user_context) << "VulkanMemoryAllocator: Unable to create crop! Missing system allocator interface!\n";
375 return nullptr;
376 }
377
378 MemoryRegion *memory_region = reinterpret_cast<MemoryRegion *>(
379 allocators.system.allocate(user_context, sizeof(MemoryRegion)));
380
381 if (memory_region == nullptr) {
382 error(user_context) << "VulkanMemoryAllocator: Failed to allocate memory region! Out of memory!\n";
383 return nullptr;
384 }
385 memcpy(memory_region, owner, sizeof(MemoryRegion));
386
387 // point the handle to the owner of the allocated region, and update the head offset
388 memory_region->is_owner = false;
389 memory_region->handle = (void *)owner;
390 memory_region->range.head_offset = owner->range.head_offset + offset;
391 return memory_region;
392}
393
394int VulkanMemoryAllocator::destroy_crop(void *user_context, MemoryRegion *region) {
395 if (region == nullptr) {
396 error(user_context) << "VulkanMemoryAllocator: Failed to destroy crop! Invalid memory region!\n";
398 }
399
400 MemoryRegion *owner = owner_of(user_context, region);
401 RegionAllocator *region_allocator = RegionAllocator::find_allocator(user_context, owner);
402 if (region_allocator == nullptr) {
403 error(user_context) << "VulkanMemoryAllocator: Unable to destroy crop region! Invalid region allocator handle!\n";
405 }
406
407 // decrement usage count
408 int error_code = region_allocator->release(this, owner);
409 if (error_code != halide_error_code_success) {
410 error(user_context) << "VulkanBlockAllocator: Unable to destroy crop region! Region allocator failed to release memory region!\n";
411 return error_code;
412 }
413
414 // discard the copied region struct
415 const BlockAllocator::MemoryAllocators &allocators = block_allocator->current_allocators();
416 if (allocators.system.deallocate == nullptr) {
417 error(user_context) << "VulkanBlockAllocator: Unable to destroy crop region! Missing system allocator interface!\n";
419 }
420 allocators.system.deallocate(user_context, region);
422}
423
425 if (region->is_owner) {
426 return region;
427 } else {
428 // If this is a cropped region, use the handle to retrieve the owner of the allocation
429 return reinterpret_cast<MemoryRegion *>(region->handle);
430 }
431}
432
433int VulkanMemoryAllocator::release(void *user_context, MemoryRegion *region) {
434#if defined(HL_VK_DEBUG_MEM)
435 debug(nullptr) << "VulkanMemoryAllocator: Releasing region ("
436 << "user_context=" << user_context << " "
437 << "region=" << (void *)(region) << " "
438 << "size=" << (uint32_t)region->size << " "
439 << "offset=" << (uint32_t)region->offset << ") ...\n";
440#endif
441 if ((device == nullptr) || (physical_device == nullptr)) {
442 error(user_context) << "VulkanMemoryAllocator: Unable to release region! Invalid device handle!\n";
444 }
445 if (block_allocator == nullptr) {
446 error(user_context) << "VulkanMemoryAllocator: Unable to release region! Invalid block allocator!\n";
448 }
449 return block_allocator->release(this, region);
450}
451
452int VulkanMemoryAllocator::reclaim(void *user_context, MemoryRegion *region) {
453#if defined(HL_VK_DEBUG_MEM)
454 debug(nullptr) << "VulkanMemoryAllocator: Reclaiming region ("
455 << "user_context=" << user_context << " "
456 << "region=" << (void *)(region) << " "
457 << "size=" << (uint32_t)region->size << " "
458 << "offset=" << (uint32_t)region->offset << ") ...\n";
459#endif
460 if ((device == nullptr) || (physical_device == nullptr)) {
461 error(user_context) << "VulkanMemoryAllocator: Unable to reclaim region! Invalid device handle!\n";
463 }
464 if (block_allocator == nullptr) {
465 error(user_context) << "VulkanMemoryAllocator: Unable to reclaim region! Invalid block allocator!\n";
467 }
468 return block_allocator->reclaim(this, region);
469}
470
471int VulkanMemoryAllocator::retain(void *user_context, MemoryRegion *region) {
472#if defined(HL_VK_DEBUG_MEM)
473 debug(nullptr) << "VulkanMemoryAllocator: Retaining region ("
474 << "user_context=" << user_context << " "
475 << "region=" << (void *)(region) << " "
476 << "size=" << (uint32_t)region->size << " "
477 << "offset=" << (uint32_t)region->offset << ") ...\n";
478#endif
479 if ((device == nullptr) || (physical_device == nullptr)) {
480 error(user_context) << "VulkanMemoryAllocator: Unable to retain region! Invalid device handle!\n";
482 }
483 if (block_allocator == nullptr) {
484 error(user_context) << "VulkanMemoryAllocator: Unable to retain region! Invalid block allocator!\n";
486 }
487 return block_allocator->retain(this, region);
488}
489
490bool VulkanMemoryAllocator::collect(void *user_context) {
491#if defined(HL_VK_DEBUG_MEM)
492 debug(nullptr) << "VulkanMemoryAllocator: Collecting unused memory ("
493 << "user_context=" << user_context << ") ... \n";
494#endif
495 if ((device == nullptr) || (physical_device == nullptr) || (block_allocator == nullptr)) {
496 return false;
497 }
498 return block_allocator->collect(this);
499}
500
501int VulkanMemoryAllocator::release(void *user_context) {
502#if defined(HL_VK_DEBUG_MEM)
503 debug(nullptr) << "VulkanMemoryAllocator: Releasing block allocator ("
504 << "user_context=" << user_context << ") ... \n";
505#endif
506 if ((device == nullptr) || (physical_device == nullptr)) {
507 error(user_context) << "VulkanMemoryAllocator: Unable to release allocator! Invalid device handle!\n";
509 }
510 if (block_allocator == nullptr) {
511 error(user_context) << "VulkanMemoryAllocator: Unable to release allocator! Invalid block allocator!\n";
513 }
514
515 return block_allocator->release(this);
516}
517
518int VulkanMemoryAllocator::destroy(void *user_context) {
519#if defined(HL_VK_DEBUG_MEM)
520 debug(nullptr) << "VulkanMemoryAllocator: Destroying allocator ("
521 << "user_context=" << user_context << ") ... \n";
522#endif
523 if (block_allocator != nullptr) {
524 block_allocator->destroy(this);
525 }
526 region_count = 0;
527 region_byte_count = 0;
528 block_count = 0;
529 block_byte_count = 0;
531}
532
533const VulkanMemoryConfig &
535 static VulkanMemoryConfig result;
536 return result;
537}
538
539// --
540int VulkanMemoryAllocator::lookup_requirements(void *user_context, size_t size, uint32_t usage_flags, VkMemoryRequirements *memory_requirements) {
541#if defined(HL_VK_DEBUG_MEM)
542 debug(nullptr) << "VulkanMemoryAllocator: Looking up requirements ("
543 << "user_context=" << user_context << " "
544 << "size=" << (uint32_t)block->size << ", "
545 << "usage_flags=" << usage_flags << ") ... \n";
546#endif
547 VkBufferCreateInfo create_info = {
549 nullptr, // struct extending this
550 0, // create flags
551 size, // buffer size (in bytes)
552 usage_flags, // buffer usage flags
553 VK_SHARING_MODE_EXCLUSIVE, // sharing mode
554 0, nullptr};
555
556 // Create a buffer to determine alignment requirements
557 VkBuffer buffer = {0};
558 VkResult result = vkCreateBuffer(this->device, &create_info, this->alloc_callbacks, &buffer);
559 if (result != VK_SUCCESS) {
560#if defined(HL_VK_DEBUG_MEM)
561 debug(nullptr) << "VulkanMemoryAllocator: Failed to create buffer to find requirements!\n\t"
562 << "vkCreateBuffer returned: " << vk_get_error_name(result) << "\n";
563#endif
565 }
566
567 vkGetBufferMemoryRequirements(this->device, buffer, memory_requirements);
568 vkDestroyBuffer(this->device, buffer, this->alloc_callbacks);
570}
571
573
574 VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
575 if (instance == nullptr) {
577 }
578
579 void *user_context = instance->owner_context;
580#if defined(HL_VK_DEBUG_MEM)
581 debug(nullptr) << "VulkanMemoryAllocator: Conforming block request ("
582 << "user_context=" << user_context << " "
583 << "request=" << (void *)(request) << ") ... \n";
584#endif
585
586 if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
587 error(user_context) << "VulkanRegionAllocator: Unable to conform block request! Invalid device handle!\n";
589 }
590
591 VkMemoryRequirements memory_requirements = {0};
592 uint32_t usage_flags = instance->select_memory_usage(user_context, request->properties);
593 int error_code = instance->lookup_requirements(user_context, request->size, usage_flags, &memory_requirements);
594 if (error_code != halide_error_code_success) {
595 error(user_context) << "VulkanRegionAllocator: Failed to conform block request! Unable to lookup requirements!\n";
596 return error_code;
597 }
598
599#if defined(HL_VK_DEBUG_MEM)
600 debug(nullptr) << "VulkanMemoryAllocator: Block allocated ("
601 << "size=" << (uint32_t)request->size << ", "
602 << "required_alignment=" << (uint32_t)memory_requirements.alignment << ", "
603 << "required_size=" << (uint32_t)memory_requirements.size << ", "
604 << "uniform_buffer_offset_alignment=" << (uint32_t)instance->physical_device_limits.minUniformBufferOffsetAlignment << ", "
605 << "storage_buffer_offset_alignment=" << (uint32_t)instance->physical_device_limits.minStorageBufferOffsetAlignment << ", "
606 << "dedicated=" << (request->dedicated ? "true" : "false") << ")\n";
607#endif
608
609 request->size = memory_requirements.size;
610 request->properties.alignment = memory_requirements.alignment;
612}
613
614int VulkanMemoryAllocator::allocate_block(void *instance_ptr, MemoryBlock *block) {
615 VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
616 if (instance == nullptr) {
618 }
619
620 void *user_context = instance->owner_context;
621 if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
622 error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid device handle!\n";
624 }
625
626 if (block == nullptr) {
627 error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid pointer!\n";
629 }
630
631#if defined(HL_VK_DEBUG_MEM)
632 debug(nullptr) << "VulkanMemoryAllocator: Allocating block ("
633 << "user_context=" << user_context << " "
634 << "block=" << (void *)(block) << " "
635 << "size=" << (uint64_t)block->size << ", "
636 << "dedicated=" << (block->dedicated ? "true" : "false") << " "
637 << "usage=" << halide_memory_usage_name(block->properties.usage) << " "
638 << "caching=" << halide_memory_caching_name(block->properties.caching) << " "
639 << "visibility=" << halide_memory_visibility_name(block->properties.visibility) << ")\n";
640#endif
641
642 // Find an appropriate memory type given the flags
643 uint32_t memory_type = instance->select_memory_type(user_context, instance->physical_device, block->properties, 0);
644 if (memory_type == invalid_memory_type) {
645 error(user_context) << "VulkanMemoryAllocator: Unable to find appropriate memory type for device!\n";
647 }
648
649 // Allocate memory
650 VkMemoryAllocateInfo alloc_info = {
652 nullptr, // struct extending this
653 block->size, // size of allocation in bytes
654 memory_type // memory type index from physical device
655 };
656
657 VkDeviceMemory *device_memory = (VkDeviceMemory *)vk_host_malloc(nullptr, sizeof(VkDeviceMemory), 0, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT, instance->alloc_callbacks);
658 if (device_memory == nullptr) {
659 debug(nullptr) << "VulkanBlockAllocator: Unable to allocate block! Failed to allocate device memory handle!\n";
661 }
662
663 VkResult result = vkAllocateMemory(instance->device, &alloc_info, instance->alloc_callbacks, device_memory);
664 if (result != VK_SUCCESS) {
665 debug(nullptr) << "VulkanMemoryAllocator: Allocation failed! vkAllocateMemory returned: " << vk_get_error_name(result) << "\n";
667 }
668#ifdef DEBUG_RUNTIME
669 debug(nullptr) << "vkAllocateMemory: Allocated memory for device region (" << (uint64_t)block->size << " bytes) ...\n";
670#endif
671
672 block->handle = (void *)device_memory;
673 instance->block_byte_count += block->size;
674 instance->block_count++;
676}
677
679 VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
680 if (instance == nullptr) {
682 }
683
684 void *user_context = instance->owner_context;
685#if defined(HL_VK_DEBUG_MEM)
686 debug(nullptr) << "VulkanMemoryAllocator: Deallocating block ("
687 << "user_context=" << user_context << " "
688 << "block=" << (void *)(block) << ") ... \n";
689#endif
690
691 if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
692 error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid device handle!\n";
694 }
695
696 if (block == nullptr) {
697 error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid pointer!\n";
699 }
700
701#if defined(HL_VK_DEBUG_MEM)
702 debug(nullptr) << "VulkanBlockAllocator: deallocating block ("
703 << "size=" << (uint32_t)block->size << ", "
704 << "dedicated=" << (block->dedicated ? "true" : "false") << " "
705 << "usage=" << halide_memory_usage_name(block->properties.usage) << " "
706 << "caching=" << halide_memory_caching_name(block->properties.caching) << " "
707 << "visibility=" << halide_memory_visibility_name(block->properties.visibility) << ")\n";
708#endif
709
710 if (block->handle == nullptr) {
711 error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid handle!\n";
713 }
714
715 VkDeviceMemory *device_memory = reinterpret_cast<VkDeviceMemory *>(block->handle);
716 if (device_memory == nullptr) {
717 error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid device memory handle!\n";
719 }
720
721 vkFreeMemory(instance->device, *device_memory, instance->alloc_callbacks);
722#ifdef DEBUG_RUNTIME
723 debug(nullptr) << "vkFreeMemory: Deallocated memory for device region (" << (uint64_t)block->size << " bytes) ...\n";
724#endif
725
726 if (instance->block_count > 0) {
727 instance->block_count--;
728 } else {
729 error(nullptr) << "VulkanRegionAllocator: Block counter invalid ... reseting to zero!\n";
730 instance->block_count = 0;
731 }
732
733 if (int64_t(instance->block_byte_count) - int64_t(block->size) >= 0) {
734 instance->block_byte_count -= block->size;
735 } else {
736 error(nullptr) << "VulkanRegionAllocator: Block byte counter invalid ... reseting to zero!\n";
737 instance->block_byte_count = 0;
738 }
739
740 block->handle = nullptr;
741 vk_host_free(nullptr, device_memory, instance->alloc_callbacks);
742 device_memory = nullptr;
744}
745
747 return block_count;
748}
749
751 return block_byte_count;
752}
753
754uint32_t VulkanMemoryAllocator::select_memory_type(void *user_context,
755 VkPhysicalDevice physical_device,
756 MemoryProperties properties,
757 uint32_t required_flags) const {
758
759 uint32_t want_flags = 0; //< preferred memory flags for requested access type
760 uint32_t need_flags = 0; //< must have in order to enable requested access
761 switch (properties.visibility) {
764 break;
767 break;
771 break;
774 break;
777 default:
778 error(nullptr) << "VulkanMemoryAllocator: Unable to convert type! Invalid memory visibility request!\n\t"
779 << "visibility=" << halide_memory_visibility_name(properties.visibility) << "\n";
780 return invalid_memory_type;
781 };
782
783 switch (properties.caching) {
785 if (need_flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) {
787 }
788 break;
790 if (need_flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) {
792 }
793 break;
795 if (need_flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) {
797 }
798 break;
801 break;
803 default:
804 error(user_context) << "VulkanMemoryAllocator: Unable to convert type! Invalid memory caching request!\n\t"
805 << "caching=" << halide_memory_caching_name(properties.caching) << "\n";
806 return invalid_memory_type;
807 };
808
809 VkPhysicalDeviceMemoryProperties device_memory_properties;
810 vkGetPhysicalDeviceMemoryProperties(physical_device, &device_memory_properties);
811
812 uint32_t result = invalid_memory_type;
813 for (uint32_t i = 0; i < device_memory_properties.memoryTypeCount; ++i) {
814
815 // if required flags are given, see if the memory type matches the requirement
816 if (required_flags) {
817 if (((required_flags >> i) & 1) == 0) {
818 continue;
819 }
820 }
821
822 const VkMemoryPropertyFlags properties = device_memory_properties.memoryTypes[i].propertyFlags;
823 if (need_flags) {
824 if ((properties & need_flags) != need_flags) {
825 continue;
826 }
827 }
828
829 if (want_flags) {
830 if ((properties & want_flags) != want_flags) {
831 continue;
832 }
833 }
834
835 result = i;
836 break;
837 }
838
839 if (result == invalid_memory_type) {
840 error(user_context) << "VulkanBlockAllocator: Failed to find appropriate memory type for given properties:\n\t"
841 << "usage=" << halide_memory_usage_name(properties.usage) << " "
842 << "caching=" << halide_memory_caching_name(properties.caching) << " "
843 << "visibility=" << halide_memory_visibility_name(properties.visibility) << "\n";
844 return invalid_memory_type;
845 }
846
847 return result;
848}
849
850// --
851
852int VulkanMemoryAllocator::conform(void *user_context, MemoryRequest *request) {
853
854 // NOTE: Vulkan will only allow us to bind device memory to a buffer if the memory requirements are met.
855 // So now we have to check those (on every allocation) and potentially recreate the buffer if the requirements
856 // don't match the requested VkBuffer's properties. Note that this is the internal storage for the driver,
857 // whose size may be required to larger than our requested size (even though we will only ever touch the
858 // size of the region we're managing as within our block)
859
860 VkMemoryRequirements memory_requirements = {0};
861 uint32_t usage_flags = select_memory_usage(user_context, request->properties);
862 int error_code = lookup_requirements(user_context, request->size, usage_flags, &memory_requirements);
863 if (error_code != halide_error_code_success) {
864 error(user_context) << "VulkanRegionAllocator: Failed to conform block request! Unable to lookup requirements!\n";
865 return error_code;
866 }
867
868#if defined(HL_VK_DEBUG_MEM)
869 debug(nullptr) << "VulkanMemoryAllocator: Buffer requirements ("
870 << "requested_size=" << (uint32_t)region->size << ", "
871 << "required_alignment=" << (uint32_t)memory_requirements.alignment << ", "
872 << "required_size=" << (uint32_t)memory_requirements.size << ")\n";
873#endif
874
875 // Enforce any alignment constraints reported by the device limits for each usage type
876 if (usage_flags & VK_BUFFER_USAGE_STORAGE_BUFFER_BIT) {
877 if ((request->alignment % this->physical_device_limits.minStorageBufferOffsetAlignment) != 0) {
878 request->alignment = this->physical_device_limits.minStorageBufferOffsetAlignment;
879 }
880 } else if (usage_flags & VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT) {
881 if ((request->alignment % this->physical_device_limits.minUniformBufferOffsetAlignment) != 0) {
882 request->alignment = this->physical_device_limits.minUniformBufferOffsetAlignment;
883 }
884 }
885
886 // Ensure the request ends on an aligned address
887 if (request->alignment > config.nearest_multiple) {
888 request->properties.nearest_multiple = request->alignment;
889 }
890
891 size_t actual_alignment = conform_alignment(request->alignment, memory_requirements.alignment);
892 size_t actual_offset = aligned_offset(request->offset, actual_alignment);
893 size_t actual_size = conform_size(actual_offset, memory_requirements.size, actual_alignment, request->properties.nearest_multiple);
894
895#if defined(HL_VK_DEBUG_MEM)
896 if ((request->size != actual_size) || (request->alignment != actual_alignment) || (request->offset != actual_offset)) {
897 debug(nullptr) << "VulkanMemoryAllocator: Adjusting request to match requirements (\n"
898 << " size = " << (uint64_t)request->size << " => " << (uint64_t)actual_size << ",\n"
899 << " alignment = " << (uint64_t)request->alignment << " => " << (uint64_t)actual_alignment << ",\n"
900 << " offset = " << (uint64_t)request->offset << " => " << (uint64_t)actual_offset << ",\n"
901 << " required.size = " << (uint64_t)memory_requirements.size << ",\n"
902 << " required.alignment = " << (uint64_t)memory_requirements.alignment << "\n)\n";
903 }
904#endif
905 request->size = actual_size;
906 request->alignment = actual_alignment;
907 request->offset = actual_offset;
908
910}
911
913
914 VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
915 if (instance == nullptr) {
917 }
918
919 void *user_context = instance->owner_context;
920#if defined(HL_VK_DEBUG_MEM)
921 debug(nullptr) << "VulkanMemoryAllocator: Conforming region request ("
922 << "user_context=" << user_context << " "
923 << "request=" << (void *)(region) << ") ... \n";
924#endif
925
926 if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
927 error(user_context) << "VulkanRegionAllocator: Unable to conform region request! Invalid device handle!\n";
929 }
930
931#if defined(HL_VK_DEBUG_MEM)
932 debug(nullptr) << "VulkanRegionAllocator: Conforming region request ("
933 << "size=" << (uint32_t)request->size << ", "
934 << "offset=" << (uint32_t)request->offset << ", "
935 << "dedicated=" << (request->dedicated ? "true" : "false") << " "
936 << "usage=" << halide_memory_usage_name(request->properties.usage) << " "
937 << "caching=" << halide_memory_caching_name(request->properties.caching) << " "
938 << "visibility=" << halide_memory_visibility_name(request->properties.visibility) << ")\n";
939#endif
940
941 return instance->conform(user_context, request);
942}
943
944int VulkanMemoryAllocator::allocate_region(void *instance_ptr, MemoryRegion *region) {
945
946 VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
947 if (instance == nullptr) {
949 }
950
951 void *user_context = instance->owner_context;
952#if defined(HL_VK_DEBUG_MEM)
953 debug(nullptr) << "VulkanMemoryAllocator: Allocating region ("
954 << "user_context=" << user_context << " "
955 << "region=" << (void *)(region) << ") ... \n";
956#endif
957
958 if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
959 error(user_context) << "VulkanRegionAllocator: Unable to allocate region! Invalid device handle!\n";
961 }
962
963 if (region == nullptr) {
964 error(user_context) << "VulkanRegionAllocator: Unable to allocate region! Invalid pointer!\n";
966 }
967
968#if defined(HL_VK_DEBUG_MEM)
969 debug(nullptr) << "VulkanRegionAllocator: Allocating region ("
970 << "size=" << (uint32_t)region->size << ", "
971 << "offset=" << (uint32_t)region->offset << ", "
972 << "dedicated=" << (region->dedicated ? "true" : "false") << " "
973 << "usage=" << halide_memory_usage_name(region->properties.usage) << " "
974 << "caching=" << halide_memory_caching_name(region->properties.caching) << " "
975 << "visibility=" << halide_memory_visibility_name(region->properties.visibility) << ")\n";
976#endif
977
978 uint32_t usage_flags = instance->select_memory_usage(user_context, region->properties);
979
980 VkBufferCreateInfo create_info = {
982 nullptr, // struct extending this
983 0, // create flags
984 region->size, // buffer size (in bytes)
985 usage_flags, // buffer usage flags
986 VK_SHARING_MODE_EXCLUSIVE, // sharing mode
987 0, nullptr};
988
989 VkBuffer *buffer = (VkBuffer *)vk_host_malloc(nullptr, sizeof(VkBuffer), 0, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT, instance->alloc_callbacks);
990 if (buffer == nullptr) {
991 error(user_context) << "VulkanRegionAllocator: Unable to allocate region! Failed to allocate buffer handle!\n";
993 }
994
995 VkResult result = vkCreateBuffer(instance->device, &create_info, instance->alloc_callbacks, buffer);
996 if (result != VK_SUCCESS) {
997 error(user_context) << "VulkanRegionAllocator: Failed to create buffer!\n\t"
998 << "vkCreateBuffer returned: " << vk_get_error_name(result) << "\n";
1000 }
1001
1002 // NOTE: Vulkan will only allow us to bind device memory to a buffer if the memory requirements are met.
1003 // So now we have to check those (on every allocation) and potentially recreate the buffer if the requirements
1004 // don't match the requested VkBuffer's properties. Note that this is the internal storage for the driver,
1005 // whose size may be required to larger than our requested size (even though we will only ever touch the
1006 // size of the region we're managing as within our block)
1007 VkMemoryRequirements memory_requirements = {0};
1008 vkGetBufferMemoryRequirements(instance->device, *buffer, &memory_requirements);
1009
1010#if defined(HL_VK_DEBUG_MEM)
1011 debug(nullptr) << "VulkanMemoryAllocator: Buffer requirements ("
1012 << "requested_size=" << (uint32_t)region->size << ", "
1013 << "required_alignment=" << (uint32_t)memory_requirements.alignment << ", "
1014 << "required_size=" << (uint32_t)memory_requirements.size << ")\n";
1015#endif
1016
1017 if (memory_requirements.size > region->size) {
1018 vkDestroyBuffer(instance->device, *buffer, instance->alloc_callbacks);
1019#ifdef DEBUG_RUNTIME
1020 debug(nullptr) << "VulkanMemoryAllocator: Reallocating buffer to match required size ("
1021 << (uint64_t)region->size << " => " << (uint64_t)memory_requirements.size << " bytes) ...\n";
1022#endif
1023 create_info.size = memory_requirements.size;
1024 VkResult result = vkCreateBuffer(instance->device, &create_info, instance->alloc_callbacks, buffer);
1025 if (result != VK_SUCCESS) {
1026 error(user_context) << "VulkanRegionAllocator: Failed to recreate buffer!\n\t"
1027 << "vkCreateBuffer returned: " << vk_get_error_name(result) << "\n";
1029 }
1030 }
1031
1032#ifdef DEBUG_RUNTIME
1033 debug(nullptr) << "vkCreateBuffer: Created buffer for device region (" << (uint64_t)region->size << " bytes) ...\n";
1034#endif
1035
1036 RegionAllocator *region_allocator = RegionAllocator::find_allocator(user_context, region);
1037 if (region_allocator == nullptr) {
1038 error(user_context) << "VulkanBlockAllocator: Unable to allocate region! Invalid region allocator!\n";
1040 }
1041
1042 BlockResource *block_resource = region_allocator->block_resource();
1043 if (block_resource == nullptr) {
1044 error(user_context) << "VulkanBlockAllocator: Unable to allocate region! Invalid block resource handle!\n";
1046 }
1047
1048 VkDeviceMemory *device_memory = reinterpret_cast<VkDeviceMemory *>(block_resource->memory.handle);
1049 if (device_memory == nullptr) {
1050 error(user_context) << "VulkanBlockAllocator: Unable to allocate region! Invalid device memory handle!\n";
1052 }
1053
1054 // Finally, bind buffer to the device memory
1055 result = vkBindBufferMemory(instance->device, *buffer, *device_memory, region->offset);
1056 if (result != VK_SUCCESS) {
1057 error(user_context) << "VulkanRegionAllocator: Failed to bind buffer!\n\t"
1058 << "vkBindBufferMemory returned: " << vk_get_error_name(result) << "\n";
1060 }
1061
1062 region->handle = (void *)buffer;
1063 region->is_owner = true;
1064 instance->region_byte_count += region->size;
1065 instance->region_count++;
1067}
1068
1070 VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
1071 if (instance == nullptr) {
1073 }
1074
1075 void *user_context = instance->owner_context;
1076#if defined(HL_VK_DEBUG_MEM)
1077 debug(nullptr) << "VulkanMemoryAllocator: Deallocating region ("
1078 << "user_context=" << user_context << " "
1079 << "region=" << (void *)(region) << ") ... \n";
1080#endif
1081
1082 if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
1083 error(user_context) << "VulkanRegionAllocator: Unable to deallocate region! Invalid device handle!\n";
1085 }
1086
1087 if (region == nullptr) {
1088 error(user_context) << "VulkanRegionAllocator: Unable to deallocate region! Invalid pointer!\n";
1090 }
1091
1092#if defined(HL_VK_DEBUG_MEM)
1093 debug(nullptr) << "VulkanRegionAllocator: Deallocating region ("
1094 << "size=" << (uint32_t)region->size << ", "
1095 << "offset=" << (uint32_t)region->offset << ", "
1096 << "dedicated=" << (region->dedicated ? "true" : "false") << " "
1097 << "usage=" << halide_memory_usage_name(region->properties.usage) << " "
1098 << "caching=" << halide_memory_caching_name(region->properties.caching) << " "
1099 << "visibility=" << halide_memory_visibility_name(region->properties.visibility) << ")\n";
1100#endif
1101
1102 if (region->handle == nullptr) {
1103 error(user_context) << "VulkanRegionAllocator: Unable to deallocate region! Invalid handle!\n";
1105 }
1106
1107 VkBuffer *buffer = reinterpret_cast<VkBuffer *>(region->handle);
1108 if (buffer == nullptr) {
1109 error(user_context) << "VulkanRegionAllocator: Unable to deallocate region! Invalid buffer handle!\n";
1111 }
1112
1113 vkDestroyBuffer(instance->device, *buffer, instance->alloc_callbacks);
1114#ifdef DEBUG_RUNTIME
1115 debug(nullptr) << "vkDestroyBuffer: Destroyed buffer for device region (" << (uint64_t)region->size << " bytes) ...\n";
1116#endif
1117 region->handle = nullptr;
1118 if (instance->region_count > 0) {
1119 instance->region_count--;
1120 } else {
1121 error(nullptr) << "VulkanRegionAllocator: Region counter invalid ... reseting to zero!\n";
1122 instance->region_count = 0;
1124 }
1125
1126 if (int64_t(instance->region_byte_count) - int64_t(region->size) >= 0) {
1127 instance->region_byte_count -= region->size;
1128 } else {
1129 error(nullptr) << "VulkanRegionAllocator: Region byte counter invalid ... reseting to zero!\n";
1130 instance->region_byte_count = 0;
1132 }
1133 vk_host_free(nullptr, buffer, instance->alloc_callbacks);
1134 buffer = nullptr;
1136}
1137
1139 return region_count;
1140}
1141
1143 return region_byte_count;
1144}
1145
1146uint32_t VulkanMemoryAllocator::select_memory_usage(void *user_context, MemoryProperties properties) const {
1147 uint32_t result = 0;
1148 switch (properties.usage) {
1151 break;
1155 break;
1158 break;
1161 break;
1164 break;
1167 default:
1168 error(user_context) << "VulkanRegionAllocator: Unable to convert type! Invalid memory usage request!\n\t"
1169 << "usage=" << halide_memory_usage_name(properties.usage) << "\n";
1170 return invalid_usage_flags;
1171 };
1172
1173 if (result == invalid_usage_flags) {
1174 error(user_context) << "VulkanRegionAllocator: Failed to find appropriate memory usage for given properties:\n\t"
1175 << "usage=" << halide_memory_usage_name(properties.usage) << " "
1176 << "caching=" << halide_memory_caching_name(properties.caching) << " "
1177 << "visibility=" << halide_memory_visibility_name(properties.visibility) << "\n";
1178 return invalid_usage_flags;
1179 }
1180
1181 return result;
1182}
1183
1184// --------------------------------------------------------------------------
1185
1186namespace {
1187
1188// --------------------------------------------------------------------------
1189// Halide System allocator for host allocations
1190void *vk_system_malloc(void *user_context, size_t size) {
1191 return malloc(size);
1192}
1193
1194void vk_system_free(void *user_context, void *ptr) {
1195 free(ptr);
1196}
1197
1198// Vulkan host-side allocation
1199void *vk_host_malloc(void *user_context, size_t size, size_t alignment, VkSystemAllocationScope scope, const VkAllocationCallbacks *callbacks) {
1200 if (callbacks) {
1201 return callbacks->pfnAllocation(user_context, size, alignment, scope);
1202 } else {
1203 return vk_system_malloc(user_context, size);
1204 }
1205}
1206
1207void vk_host_free(void *user_context, void *ptr, const VkAllocationCallbacks *callbacks) {
1208 if (callbacks) {
1209 return callbacks->pfnFree(user_context, ptr);
1210 } else {
1211 return vk_system_free(user_context, ptr);
1212 }
1213}
1214
1215VulkanMemoryAllocator *vk_create_memory_allocator(void *user_context,
1216 VkDevice device,
1217 VkPhysicalDevice physical_device,
1218 const VkAllocationCallbacks *alloc_callbacks) {
1219
1220 SystemMemoryAllocatorFns system_allocator = {vk_system_malloc, vk_system_free};
1221 VulkanMemoryConfig config = memory_allocator_config;
1222
1223 // Parse the allocation config string (if specified).
1224 //
1225 // `HL_VK_ALLOC_CONFIG=N:N:N` will tell Halide to configure the Vulkan memory
1226 // allocator use the given constraints specified as three integer values
1227 // separated by a `:` or `;`. These values correspond to `minimum_block_size`,
1228 // `maximum_block_size` and `maximum_block_count`.
1229 //
1230 const char *alloc_config = vk_get_alloc_config_internal(user_context);
1232 StringTable alloc_config_values;
1233 alloc_config_values.parse(user_context, alloc_config, HL_VK_ENV_DELIM);
1234 if (alloc_config_values.size() > 0) {
1235 config.maximum_pool_size = atoi(alloc_config_values[0]) * 1024 * 1024;
1236 print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.maximum_pool_size << " for maximum pool size (in bytes)\n";
1237 }
1238 if (alloc_config_values.size() > 1) {
1239 config.minimum_block_size = atoi(alloc_config_values[1]) * 1024 * 1024;
1240 print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.minimum_block_size << " for minimum block size (in bytes)\n";
1241 }
1242 if (alloc_config_values.size() > 2) {
1243 config.maximum_block_size = atoi(alloc_config_values[2]) * 1024 * 1024;
1244 print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.maximum_block_size << " for maximum block size (in bytes)\n";
1245 }
1246 if (alloc_config_values.size() > 3) {
1247 config.maximum_block_count = atoi(alloc_config_values[3]);
1248 print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.maximum_block_count << " for maximum block count\n";
1249 }
1250 if (alloc_config_values.size() > 4) {
1251 config.nearest_multiple = atoi(alloc_config_values[4]);
1252 print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.nearest_multiple << " for nearest multiple\n";
1253 }
1254 }
1255
1256 return VulkanMemoryAllocator::create(user_context,
1257 config, device, physical_device,
1258 system_allocator, alloc_callbacks);
1259}
1260
1261int vk_destroy_memory_allocator(void *user_context, VulkanMemoryAllocator *allocator) {
1262 if (allocator != nullptr) {
1263 VulkanMemoryAllocator::destroy(user_context, allocator);
1264 allocator = nullptr;
1265 }
1267}
1268
1269// --------------------------------------------------------------------------
1270
1271int vk_clear_device_buffer(void *user_context,
1272 VulkanMemoryAllocator *allocator,
1273 VkCommandPool command_pool,
1274 VkQueue command_queue,
1275 VkBuffer device_buffer) {
1276
1277#ifdef DEBUG_RUNTIME
1278 debug(user_context)
1279 << " vk_clear_device_buffer (user_context: " << user_context << ", "
1280 << "allocator: " << (void *)allocator << ", "
1281 << "command_pool: " << (void *)command_pool << ", "
1282 << "command_queue: " << (void *)command_queue << ", "
1283 << "device_buffer: " << (void *)device_buffer << ")\n";
1284#endif
1285
1286 // create a command buffer
1287 VkCommandBuffer command_buffer;
1288 int error_code = vk_create_command_buffer(user_context, allocator, command_pool, &command_buffer);
1289 if (error_code != halide_error_code_success) {
1290 error(user_context) << "Vulkan: Failed to create command buffer!\n";
1291 return error_code;
1292 }
1293
1294 // begin the command buffer
1295 VkCommandBufferBeginInfo command_buffer_begin_info =
1296 {
1298 nullptr, // pointer to struct extending this
1300 nullptr // pointer to parent command buffer
1301 };
1302
1303 VkResult result = vkBeginCommandBuffer(command_buffer, &command_buffer_begin_info);
1304 if (result != VK_SUCCESS) {
1305 error(user_context) << "Vulkan: vkBeginCommandBuffer returned " << vk_get_error_name(result) << "\n";
1307 }
1308
1309 // fill buffer with zero values up to the size of the buffer
1310 vkCmdFillBuffer(command_buffer, device_buffer, 0, VK_WHOLE_SIZE, 0);
1311
1312 // end the command buffer
1313 result = vkEndCommandBuffer(command_buffer);
1314 if (result != VK_SUCCESS) {
1315 error(user_context) << "Vulkan: vkEndCommandBuffer returned " << vk_get_error_name(result) << "\n";
1317 }
1318
1319 // submit the command buffer
1320 VkSubmitInfo submit_info =
1321 {
1322 VK_STRUCTURE_TYPE_SUBMIT_INFO, // struct type
1323 nullptr, // pointer to struct extending this
1324 0, // wait semaphore count
1325 nullptr, // semaphores
1326 nullptr, // pipeline stages where semaphore waits occur
1327 1, // how many command buffers to execute
1328 &command_buffer, // the command buffers
1329 0, // number of semaphores to signal
1330 nullptr // the semaphores to signal
1331 };
1332
1333 result = vkQueueSubmit(command_queue, 1, &submit_info, 0);
1334 if (result != VK_SUCCESS) {
1335 error(user_context) << "Vulkan: vkQueueSubmit returned " << vk_get_error_name(result) << "\n";
1337 }
1338
1339 // wait for memset to finish
1340 result = vkQueueWaitIdle(command_queue);
1341 if (result != VK_SUCCESS) {
1342 error(user_context) << "Vulkan: vkQueueWaitIdle returned " << vk_get_error_name(result) << "\n";
1344 }
1345
1346 error_code = vk_destroy_command_buffer(user_context, allocator, command_pool, command_buffer);
1347 if (error_code != halide_error_code_success) {
1348 error(user_context) << "Vulkan: Failed to destroy command buffer!\n";
1349 return error_code;
1350 }
1351
1353}
1354
1355// --------------------------------------------------------------------------
1356
1357} // namespace
1358} // namespace Vulkan
1359} // namespace Internal
1360} // namespace Runtime
1361} // namespace Halide
1362
1363// --------------------------------------------------------------------------
1364
1365extern "C" {
1366
1367// --------------------------------------------------------------------------
1368
1370 using namespace Halide::Runtime::Internal::Vulkan;
1371 ScopedSpinLock lock(&custom_allocation_callbacks_lock);
1372 custom_allocation_callbacks = callbacks;
1373}
1374
1376 using namespace Halide::Runtime::Internal::Vulkan;
1377 ScopedSpinLock lock(&custom_allocation_callbacks_lock);
1378 return custom_allocation_callbacks;
1379}
1380
1381// --------------------------------------------------------------------------
1382
1383} // extern "C"
1384
1385#endif // HALIDE_RUNTIME_VULKAN_MEMORY_H
@ halide_error_code_internal_error
There is a bug in the Halide compiler.
@ halide_error_code_generic_error
An uncategorized error occurred.
@ halide_error_code_success
There was no error.
@ halide_error_code_device_malloc_failed
The Halide runtime encountered an error while trying to allocate memory on device.
@ halide_error_code_out_of_memory
A call to halide_malloc returned NULL.
Allocator class interface for managing large contiguous blocks of memory, which are then sub-allocate...
MemoryRegion * reserve(void *user_context, const MemoryRequest &request)
int release(void *user_context, MemoryRegion *region)
int retain(void *user_context, MemoryRegion *region)
static void destroy(void *user_context, BlockAllocator *block_allocator)
static BlockAllocator * create(void *user_context, const Config &config, const MemoryAllocators &allocators)
int reclaim(void *user_context, MemoryRegion *region)
const MemoryAllocators & current_allocators() const
Allocator class interface for sub-allocating a contiguous memory block into smaller regions of memory...
static RegionAllocator * find_allocator(void *user_context, MemoryRegion *memory_region)
int retain(void *user_context, MemoryRegion *memory_region)
int release(void *user_context, MemoryRegion *memory_region)
size_t parse(void *user_context, const char *str, const char *delim)
Vulkan Memory Allocator class interface for managing large memory requests stored as contiguous block...
int reclaim(void *user_context, MemoryRegion *region)
static const VulkanMemoryConfig & default_config()
MemoryRegion * reserve(void *user_context, const MemoryRequest &request)
VulkanMemoryAllocator(const VulkanMemoryAllocator &)=delete
int release(void *user_context, MemoryRegion *region)
static int allocate_block(void *instance_ptr, MemoryBlock *block)
int conform(void *user_context, MemoryRequest *request)
static int deallocate_region(void *instance_ptr, MemoryRegion *region)
VulkanMemoryAllocator & operator=(const VulkanMemoryAllocator &)=delete
static int destroy(void *user_context, VulkanMemoryAllocator *allocator)
int unmap(void *user_context, MemoryRegion *region)
static int conform_region_request(void *instance_ptr, MemoryRequest *request)
MemoryRegion * create_crop(void *user_context, MemoryRegion *region, uint64_t offset)
int destroy_crop(void *user_context, MemoryRegion *region)
static int allocate_region(void *instance_ptr, MemoryRegion *region)
static int conform_block_request(void *instance_ptr, MemoryRequest *request)
MemoryRegion * owner_of(void *user_context, MemoryRegion *region)
VkPhysicalDeviceLimits current_physical_device_limits() const
const VkAllocationCallbacks * callbacks() const
static int deallocate_block(void *instance_ptr, MemoryBlock *block)
void * map(void *user_context, MemoryRegion *region)
int retain(void *user_context, MemoryRegion *region)
static VulkanMemoryAllocator * create(void *user_context, const VulkanMemoryConfig &config, VkDevice dev, VkPhysicalDevice phys_dev, const SystemMemoryAllocatorFns &system_allocator, const VkAllocationCallbacks *alloc_callbacks=nullptr)
WEAK const char * halide_memory_caching_name(MemoryCaching value)
WEAK const char * halide_memory_usage_name(MemoryUsage value)
WEAK const char * halide_memory_visibility_name(MemoryVisibility value)
VKAPI_ATTR VkResult VKAPI_CALL vkBindBufferMemory(VkDevice device, VkBuffer buffer, VkDeviceMemory memory, VkDeviceSize memoryOffset)
VkFlags VkMemoryPropertyFlags
VKAPI_ATTR void VKAPI_CALL vkCmdFillBuffer(VkCommandBuffer commandBuffer, VkBuffer dstBuffer, VkDeviceSize dstOffset, VkDeviceSize size, uint32_t data)
VKAPI_ATTR VkResult VKAPI_CALL vkAllocateMemory(VkDevice device, const VkMemoryAllocateInfo *pAllocateInfo, const VkAllocationCallbacks *pAllocator, VkDeviceMemory *pMemory)
#define VK_MAX_MEMORY_TYPES
@ VK_SHARING_MODE_EXCLUSIVE
@ VK_MEMORY_PROPERTY_HOST_COHERENT_BIT
@ VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT
@ VK_MEMORY_PROPERTY_HOST_CACHED_BIT
@ VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT
VKAPI_ATTR void VKAPI_CALL vkGetBufferMemoryRequirements(VkDevice device, VkBuffer buffer, VkMemoryRequirements *pMemoryRequirements)
VkSystemAllocationScope
@ VK_SYSTEM_ALLOCATION_SCOPE_OBJECT
VKAPI_ATTR VkResult VKAPI_CALL vkQueueWaitIdle(VkQueue queue)
uint64_t VkDeviceSize
Definition mini_vulkan.h:71
VKAPI_ATTR void VKAPI_CALL vkFreeMemory(VkDevice device, VkDeviceMemory memory, const VkAllocationCallbacks *pAllocator)
@ VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT
VKAPI_ATTR void VKAPI_CALL vkGetPhysicalDeviceProperties(VkPhysicalDevice physicalDevice, VkPhysicalDeviceProperties *pProperties)
VKAPI_ATTR void VKAPI_CALL vkGetPhysicalDeviceMemoryProperties(VkPhysicalDevice physicalDevice, VkPhysicalDeviceMemoryProperties *pMemoryProperties)
VKAPI_ATTR VkResult VKAPI_CALL vkCreateBuffer(VkDevice device, const VkBufferCreateInfo *pCreateInfo, const VkAllocationCallbacks *pAllocator, VkBuffer *pBuffer)
VKAPI_ATTR VkResult VKAPI_CALL vkQueueSubmit(VkQueue queue, uint32_t submitCount, const VkSubmitInfo *pSubmits, VkFence fence)
VKAPI_ATTR VkResult VKAPI_CALL vkMapMemory(VkDevice device, VkDeviceMemory memory, VkDeviceSize offset, VkDeviceSize size, VkMemoryMapFlags flags, void **ppData)
#define VK_WHOLE_SIZE
@ VK_BUFFER_USAGE_TRANSFER_DST_BIT
@ VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT
@ VK_BUFFER_USAGE_STORAGE_BUFFER_BIT
@ VK_BUFFER_USAGE_TRANSFER_SRC_BIT
VkResult
@ VK_SUCCESS
VKAPI_ATTR void VKAPI_CALL vkUnmapMemory(VkDevice device, VkDeviceMemory memory)
VKAPI_ATTR VkResult VKAPI_CALL vkEndCommandBuffer(VkCommandBuffer commandBuffer)
VKAPI_ATTR void VKAPI_CALL vkDestroyBuffer(VkDevice device, VkBuffer buffer, const VkAllocationCallbacks *pAllocator)
@ VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO
@ VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO
@ VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO
@ VK_STRUCTURE_TYPE_SUBMIT_INFO
VKAPI_ATTR VkResult VKAPI_CALL vkBeginCommandBuffer(VkCommandBuffer commandBuffer, const VkCommandBufferBeginInfo *pBeginInfo)
WEAK VulkanMemoryConfig memory_allocator_config
WEAK const VkAllocationCallbacks * custom_allocation_callbacks
WEAK ScopedSpinLock::AtomicFlag custom_allocation_callbacks_lock
ALWAYS_INLINE size_t conform_alignment(size_t requested, size_t required)
ALWAYS_INLINE size_t conform_size(size_t offset, size_t size, size_t alignment, size_t nearest_multiple)
ALWAYS_INLINE size_t aligned_offset(size_t offset, size_t alignment)
This file defines the class FunctionDAG, which is our representation of a Halide pipeline,...
@ Internal
Not visible externally, similar to 'static' linkage in C.
Expr print(const std::vector< Expr > &values)
Create an Expr that prints out its value whenever it is evaluated.
unsigned __INT64_TYPE__ uint64_t
signed __INT64_TYPE__ int64_t
int atoi(const char *)
void * malloc(size_t)
void * memcpy(void *s1, const void *s2, size_t n)
void * memset(void *s, int val, size_t n)
unsigned __INT32_TYPE__ uint32_t
#define WEAK
void free(void *)
static bool is_empty(const char *str)
PFN_vkAllocationFunction pfnAllocation
PFN_vkFreeFunction pfnFree
VkDeviceSize size
VkDeviceSize alignment
VkMemoryPropertyFlags propertyFlags
VkDeviceSize minStorageBufferOffsetAlignment
VkDeviceSize minUniformBufferOffsetAlignment
VkMemoryType memoryTypes[VK_MAX_MEMORY_TYPES]
VkPhysicalDeviceLimits limits
#define HL_VK_ENV_DELIM
WEAK void halide_vulkan_set_allocation_callbacks(const VkAllocationCallbacks *callbacks)
WEAK const VkAllocationCallbacks * halide_vulkan_get_allocation_callbacks(void *user_context)