Halide 19.0.0
Halide compiler and libraries
Loading...
Searching...
No Matches
region_allocator.h
Go to the documentation of this file.
1#ifndef HALIDE_RUNTIME_REGION_ALLOCATOR_H
2#define HALIDE_RUNTIME_REGION_ALLOCATOR_H
3
4#include "../HalideRuntime.h"
5#include "../printer.h"
6#include "memory_arena.h"
7#include "memory_resources.h"
8
9namespace Halide {
10namespace Runtime {
11namespace Internal {
12
13// --
14
15/** Allocator class interface for sub-allocating a contiguous
16 * memory block into smaller regions of memory. This class only
17 * manages the address creation for the regions -- allocation
18 * callback functions are used to request the memory from the
19 * necessary system or API calls. This class is intended to be
20 * used inside of a higher level memory management class that
21 * provides thread safety, policy management and API
22 * integration for a specific runtime API (eg Vulkan, OpenCL, etc)
23 */
24class RegionAllocator {
25public:
26 // disable copy constructors and assignment
27 RegionAllocator(const RegionAllocator &) = delete;
28 RegionAllocator &operator=(const RegionAllocator &) = delete;
29
30 // disable non-factory based construction
31 RegionAllocator() = delete;
32 ~RegionAllocator() = delete;
33
34 // Allocators for the different types of memory we need to allocate
39
40 // Factory methods for creation / destruction
41 static RegionAllocator *create(void *user_context, BlockResource *block, const MemoryAllocators &ma);
42 static int destroy(void *user_context, RegionAllocator *region_allocator);
43
44 // Returns the allocator class instance for the given allocation (or nullptr)
45 static RegionAllocator *find_allocator(void *user_context, MemoryRegion *memory_region);
46
47 // Public interface methods
48 MemoryRegion *reserve(void *user_context, const MemoryRequest &request);
49 int conform(void *user_context, MemoryRequest *request) const; //< conform the given request into a suitable allocation
50 int release(void *user_context, MemoryRegion *memory_region); //< unmark and cache the region for reuse
51 int reclaim(void *user_context, MemoryRegion *memory_region); //< free the region and consolidate
52 int retain(void *user_context, MemoryRegion *memory_region); //< retain the region and increase usage count
53 bool collect(void *user_context); //< returns true if any blocks were removed
54 int release(void *user_context);
55 int destroy(void *user_context);
56
57 // Returns the currently managed block resource
58 BlockResource *block_resource() const;
59
60private:
61 // Initializes a new instance
62 int initialize(void *user_context, BlockResource *block, const MemoryAllocators &ma);
63
64 // Search through allocated block regions (Best-Fit)
65 BlockRegion *find_block_region(void *user_context, const MemoryRequest &request);
66
67 // Returns true if block region is unused and available
68 bool is_available(const BlockRegion *region) const;
69
70 // Returns true if neighbouring block regions to the given region can be coalesced into one
71 bool can_coalesce(const BlockRegion *region) const;
72
73 // Merges available neighbouring block regions into the given region
74 BlockRegion *coalesce_block_regions(void *user_context, BlockRegion *region);
75
76 // Returns true if the given region can be split to accomodate the given size
77 bool can_split(const BlockRegion *region, const MemoryRequest &request) const;
78
79 // Splits the given block region into a smaller region to accomodate the given size, followed by empty space for the remaining
80 BlockRegion *split_block_region(void *user_context, BlockRegion *region, const MemoryRequest &request);
81
82 // Creates a new block region and adds it to the region list
83 BlockRegion *create_block_region(void *user_context, const MemoryRequest &request);
84
85 // Creates a new block region and adds it to the region list
86 int destroy_block_region(void *user_context, BlockRegion *region);
87
88 // Invokes the allocation callback to allocate memory for the block region
89 int alloc_block_region(void *user_context, BlockRegion *region);
90
91 // Releases a block region and leaves it in the list for further allocations
92 int release_block_region(void *user_context, BlockRegion *region);
93
94 // Invokes the deallocation callback to free memory for the block region
95 int free_block_region(void *user_context, BlockRegion *region);
96
97 // Returns true if the given block region is the last region in the list
98 bool is_last_block_region(void *user_context, const BlockRegion *region) const;
99
100 // Returns true if the given block region is compatible with the given properties
101 bool is_compatible_block_region(const BlockRegion *region, const MemoryProperties &properties) const;
102
103 // Returns true if the given block region is suitable for the requested allocation
104 bool is_block_region_suitable_for_request(void *user_context, const BlockRegion *region, const MemoryRequest &request) const;
105
106 // Returns the number of active regions for the block;
107 size_t region_count(void *user_context) const;
108
109 BlockResource *block = nullptr;
110 MemoryArena *arena = nullptr;
111 MemoryAllocators allocators;
112};
113
114RegionAllocator *RegionAllocator::create(void *user_context, BlockResource *block_resource, const MemoryAllocators &allocators) {
115 halide_abort_if_false(user_context, allocators.system.allocate != nullptr);
116 RegionAllocator *result = reinterpret_cast<RegionAllocator *>(
117 allocators.system.allocate(user_context, sizeof(RegionAllocator)));
118
119 if (result == nullptr) {
120 return nullptr;
121 }
122
123 result->initialize(user_context, block_resource, allocators);
124 return result;
125}
126
127int RegionAllocator::destroy(void *user_context, RegionAllocator *instance) {
128 halide_abort_if_false(user_context, instance != nullptr);
129 const MemoryAllocators &allocators = instance->allocators;
130 instance->destroy(user_context);
131 halide_abort_if_false(user_context, allocators.system.deallocate != nullptr);
132 allocators.system.deallocate(user_context, instance);
133 return 0;
134}
135
136int RegionAllocator::initialize(void *user_context, BlockResource *mb, const MemoryAllocators &ma) {
137 block = mb;
138 allocators = ma;
139 arena = MemoryArena::create(user_context, {sizeof(BlockRegion), MemoryArena::default_capacity, 0}, allocators.system);
140 halide_abort_if_false(user_context, arena != nullptr);
141 MemoryRequest block_request = {};
142 block_request.size = block->memory.size;
143 block_request.offset = 0;
144 block_request.alignment = block->memory.properties.alignment;
145 block_request.properties = block->memory.properties;
146 block_request.dedicated = block->memory.dedicated;
147 block->allocator = this;
148 block->regions = create_block_region(user_context, block_request);
149 return 0;
150}
151
152int RegionAllocator::conform(void *user_context, MemoryRequest *request) const {
153 if (allocators.region.conform) {
154 return allocators.region.conform(user_context, request);
155 } else {
156 size_t actual_alignment = conform_alignment(request->alignment, block->memory.properties.alignment);
157 size_t actual_offset = aligned_offset(request->offset, actual_alignment);
158 size_t actual_size = conform_size(actual_offset, request->size, actual_alignment, block->memory.properties.nearest_multiple);
159 request->alignment = actual_alignment;
160 request->offset = actual_offset;
161 request->size = actual_size;
162 }
163 return 0;
164}
165
166MemoryRegion *RegionAllocator::reserve(void *user_context, const MemoryRequest &request) {
167 halide_abort_if_false(user_context, request.size > 0);
168
169 MemoryRequest region_request = request;
170
171 int error_code = conform(user_context, &region_request);
172 if (error_code) {
173#ifdef DEBUG_RUNTIME_INTERNAL
174 debug(user_context) << "RegionAllocator: Failed to conform region request! Unable to reserve memory ...\n";
175#endif
176 return nullptr;
177 }
178
179 size_t remaining = block->memory.size - block->reserved;
180 if (remaining < region_request.size) {
181#ifdef DEBUG_RUNTIME_INTERNAL
182 debug(user_context) << "RegionAllocator: Unable to reserve more memory from block "
183 << "-- requested size (" << (int32_t)(region_request.size) << " bytes) "
184 << "greater than available (" << (int32_t)(remaining) << " bytes)";
185#endif
186 return nullptr;
187 }
188
189 BlockRegion *block_region = find_block_region(user_context, region_request);
190 if (block_region == nullptr) {
191#ifdef DEBUG_RUNTIME_INTERNAL
192 debug(user_context) << "RegionAllocator: Failed to locate region for requested size ("
193 << (int32_t)(request.size) << " bytes)";
194#endif
195 return nullptr;
196 }
197
198 if (can_split(block_region, region_request)) {
199#ifdef DEBUG_RUNTIME_INTERNAL
200 debug(user_context) << "RegionAllocator: Splitting region of size ( " << (int32_t)(block_region->memory.size) << ") "
201 << "to accomodate requested size (" << (int32_t)(region_request.size) << " bytes)";
202#endif
203 split_block_region(user_context, block_region, region_request);
204 }
205
206 alloc_block_region(user_context, block_region);
207 return reinterpret_cast<MemoryRegion *>(block_region);
208}
209
210int RegionAllocator::release(void *user_context, MemoryRegion *memory_region) {
211 BlockRegion *block_region = reinterpret_cast<BlockRegion *>(memory_region);
212 halide_abort_if_false(user_context, block_region != nullptr);
213 halide_abort_if_false(user_context, block_region->block_ptr == block);
214 if (block_region->usage_count > 0) {
215 block_region->usage_count--;
216 }
217 return release_block_region(user_context, block_region);
218}
219
220int RegionAllocator::reclaim(void *user_context, MemoryRegion *memory_region) {
221 BlockRegion *block_region = reinterpret_cast<BlockRegion *>(memory_region);
222 halide_abort_if_false(user_context, block_region != nullptr);
223 halide_abort_if_false(user_context, block_region->block_ptr == block);
224 if (block_region->usage_count > 0) {
225 block_region->usage_count--;
226 }
227 release_block_region(user_context, block_region);
228 free_block_region(user_context, block_region);
229 return 0;
230}
231
232int RegionAllocator::retain(void *user_context, MemoryRegion *memory_region) {
233 BlockRegion *block_region = reinterpret_cast<BlockRegion *>(memory_region);
234 halide_abort_if_false(user_context, block_region != nullptr);
235 halide_abort_if_false(user_context, block_region->block_ptr == block);
236 block_region->usage_count++;
237 return 0;
238}
239
240RegionAllocator *RegionAllocator::find_allocator(void *user_context, MemoryRegion *memory_region) {
241 BlockRegion *block_region = reinterpret_cast<BlockRegion *>(memory_region);
242 if (block_region == nullptr) {
243 return nullptr;
244 }
245 if (block_region->block_ptr == nullptr) {
246 return nullptr;
247 }
248 return block_region->block_ptr->allocator;
249}
250
251bool RegionAllocator::is_last_block_region(void *user_context, const BlockRegion *region) const {
252 return ((region == nullptr) || (region == region->next_ptr) || (region->next_ptr == nullptr));
253}
254
255bool RegionAllocator::is_block_region_suitable_for_request(void *user_context, const BlockRegion *region, const MemoryRequest &request) const {
256 if (!is_available(region)) {
257#ifdef DEBUG_RUNTIME_INTERNAL
258 debug(user_context) << " skipping block region ... not available! ("
259 << " block_region=" << (void *)region
260 << " region_size=" << (uint32_t)(region->memory.size)
261 << ")";
262#endif
263 return false;
264 }
265
266 MemoryRequest region_request = request;
267 int error_code = conform(user_context, &region_request);
268 if (error_code) {
269#ifdef DEBUG_RUNTIME_INTERNAL
270 debug(user_context) << "RegionAllocator: Failed to conform region request! Unable to reserve memory ...\n";
271#endif
272 return false;
273 }
274
275 // skip incompatible block regions for this request
276 if (!is_compatible_block_region(region, region_request.properties)) {
277#ifdef DEBUG_RUNTIME_INTERNAL
278 debug(user_context) << " skipping block region ... incompatible properties! ("
279 << " block_region=" << (void *)region
280 << " region_size=" << (uint32_t)(region->memory.size)
281 << ")";
282#endif
283 return false;
284 }
285
286 // is the adjusted size larger than the current region?
287 if (region_request.size > region->memory.size) {
288#ifdef DEBUG_RUNTIME_INTERNAL
289 debug(user_context) << " skipping block region ... not enough space for adjusted size! ("
290 << " block_region=" << (void *)region
291 << " request_size=" << (uint32_t)(request.size)
292 << " actual_size=" << (uint32_t)(region_request.size)
293 << " region_size=" << (uint32_t)(region->memory.size)
294 << ")";
295#endif
296 return false;
297 }
298
299 // will the adjusted size fit within the remaining unallocated space?
300 if ((region_request.size + block->reserved) <= block->memory.size) {
301#ifdef DEBUG_RUNTIME_INTERNAL
302 debug(user_context) << " found suitable block region! ("
303 << " block_region=" << (void *)region
304 << " request_size=" << (uint32_t)(request.size)
305 << " actual_size=" << (uint32_t)(region_request.size)
306 << " region_size=" << (uint32_t)(region->memory.size)
307 << ")";
308#endif
309 return true; // you betcha
310 }
311
312 return false;
313}
314
315BlockRegion *RegionAllocator::find_block_region(void *user_context, const MemoryRequest &request) {
316#ifdef DEBUG_RUNTIME_INTERNAL
317 debug(user_context) << "RegionAllocator: find block region ( "
318 << "user_context=" << (void *)(user_context) << " "
319 << "requested_size=" << (uint32_t)request.size << " "
320 << "requested_is_dedicated=" << (request.dedicated ? "true" : "false") << " "
321 << "requested_usage=" << halide_memory_usage_name(request.properties.usage) << " "
322 << "requested_caching=" << halide_memory_caching_name(request.properties.caching) << " "
323 << "requested_visibility=" << halide_memory_visibility_name(request.properties.visibility) << ")";
324#endif
325 BlockRegion *block_region = block->regions;
326 while (block_region != nullptr) {
327 if (is_block_region_suitable_for_request(user_context, block_region, request)) {
328#ifdef DEBUG_RUNTIME_INTERNAL
329 debug(user_context) << "RegionAllocator: found suitable region ( "
330 << "user_context=" << (void *)(user_context) << " "
331 << "block_resource=" << (void *)block << " "
332 << "block_size=" << (uint32_t)block->memory.size << " "
333 << "block_reserved=" << (uint32_t)block->reserved << " "
334 << "requested_size=" << (uint32_t)request.size << " "
335 << "requested_is_dedicated=" << (request.dedicated ? "true" : "false") << " "
336 << "requested_usage=" << halide_memory_usage_name(request.properties.usage) << " "
337 << "requested_caching=" << halide_memory_caching_name(request.properties.caching) << " "
338 << "requested_visibility=" << halide_memory_visibility_name(request.properties.visibility) << ")";
339#endif
340 return block_region;
341 }
342
343 if (is_last_block_region(user_context, block_region)) {
344 block_region = nullptr; // end of list ... nothing found
345 break;
346 }
347 block_region = block_region->next_ptr;
348 }
349
350 if (block_region == nullptr) {
351#ifdef DEBUG_RUNTIME_INTERNAL
352 debug(user_context) << "RegionAllocator: couldn't find suitable region! ("
353 << "user_context=" << (void *)(user_context) << " "
354 << "requested_size=" << (uint32_t)request.size << " "
355 << "requested_is_dedicated=" << (request.dedicated ? "true" : "false") << " "
356 << "requested_usage=" << halide_memory_usage_name(request.properties.usage) << " "
357 << "requested_caching=" << halide_memory_caching_name(request.properties.caching) << " "
358 << "requested_visibility=" << halide_memory_visibility_name(request.properties.visibility) << ")";
359#endif
360 }
361
362 return block_region;
363}
364
365bool RegionAllocator::is_available(const BlockRegion *block_region) const {
366 if (block_region == nullptr) {
367 return false;
368 }
369 if (block_region->usage_count > 0) {
370 return false;
371 }
372 if (block_region->status != AllocationStatus::Available) {
373 return false;
374 }
375 return true;
376}
377
378bool RegionAllocator::can_coalesce(const BlockRegion *block_region) const {
379 if (!is_available(block_region)) {
380 return false;
381 }
382 if (is_available(block_region->prev_ptr)) {
383 return true;
384 }
385 if (is_available(block_region->next_ptr)) {
386 return true;
387 }
388 return false;
389}
390
391BlockRegion *RegionAllocator::coalesce_block_regions(void *user_context, BlockRegion *block_region) {
392
393 if ((block_region->usage_count == 0) && (block_region->memory.handle != nullptr)) {
394#ifdef DEBUG_RUNTIME_INTERNAL
395 debug(user_context) << "RegionAllocator: Freeing unused region to coalesce ("
396 << "block_ptr=" << (void *)block_region->block_ptr << " "
397 << "block_region=" << (void *)block_region << " "
398 << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
399 << "block_reserved=" << (uint32_t)block->reserved << " "
400 << ")";
401#endif
402 halide_abort_if_false(user_context, allocators.region.deallocate != nullptr);
403 MemoryRegion *memory_region = &(block_region->memory);
404 allocators.region.deallocate(user_context, memory_region);
405 block_region->memory.handle = nullptr;
406 }
407
408 BlockRegion *prev_region = block_region->prev_ptr;
409 if (is_available(prev_region) && (prev_region != block_region)) {
410
411#ifdef DEBUG_RUNTIME_INTERNAL
412 debug(user_context) << "RegionAllocator: Coalescing "
413 << "previous region (offset=" << (int32_t)prev_region->memory.offset << " size=" << (int32_t)(prev_region->memory.size) << " bytes) "
414 << "into current region (offset=" << (int32_t)block_region->memory.offset << " size=" << (int32_t)(block_region->memory.size) << " bytes)!";
415#endif
416
417 prev_region->next_ptr = block_region->next_ptr;
418 if (block_region->next_ptr) {
419 block_region->next_ptr->prev_ptr = prev_region;
420 }
421 prev_region->memory.size += block_region->memory.size;
422 destroy_block_region(user_context, block_region);
423 block_region = prev_region;
424 }
425
426 BlockRegion *next_region = block_region->next_ptr;
427 if (is_available(next_region) && (next_region != block_region)) {
428
429#ifdef DEBUG_RUNTIME_INTERNAL
430 debug(user_context) << "RegionAllocator: Coalescing "
431 << "next region (offset=" << (int32_t)next_region->memory.offset << " size=" << (int32_t)(next_region->memory.size) << " bytes) "
432 << "into current region (offset=" << (int32_t)block_region->memory.offset << " size=" << (int32_t)(block_region->memory.size) << " bytes)";
433#endif
434
435 if (next_region->next_ptr) {
436 next_region->next_ptr->prev_ptr = block_region;
437 }
438 block_region->next_ptr = next_region->next_ptr;
439 block_region->memory.size += next_region->memory.size;
440 destroy_block_region(user_context, next_region);
441 }
442
443 return block_region;
444}
445
446bool RegionAllocator::can_split(const BlockRegion *block_region, const MemoryRequest &split_request) const {
447 return (block_region && (block_region->memory.size > split_request.size) && (block_region->usage_count == 0));
448}
449
450BlockRegion *RegionAllocator::split_block_region(void *user_context, BlockRegion *block_region, const MemoryRequest &request) {
451
452 if ((block_region->usage_count == 0) && (block_region->memory.handle != nullptr)) {
453#ifdef DEBUG_RUNTIME_INTERNAL
454 debug(user_context) << "RegionAllocator: Split deallocate region ("
455 << "block_ptr=" << (void *)block_region->block_ptr << " "
456 << "block_region=" << (void *)block_region << " "
457 << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
458 << "block_reserved=" << (uint32_t)block_region->block_ptr->reserved << " "
459 << ")";
460#endif
461 halide_abort_if_false(user_context, allocators.region.deallocate != nullptr);
462 MemoryRegion *memory_region = &(block_region->memory);
463 allocators.region.deallocate(user_context, memory_region);
464 block_region->memory.handle = nullptr;
465 }
466
467 MemoryRequest split_request = request;
468 split_request.size = block_region->memory.size - request.size;
469 split_request.offset = block_region->memory.offset + request.size;
470
471#ifdef DEBUG_RUNTIME_INTERNAL
472 debug(user_context) << "RegionAllocator: Splitting "
473 << "current region (offset=" << (int32_t)block_region->memory.offset << " size=" << (int32_t)(block_region->memory.size) << " bytes) "
474 << "to create empty region (offset=" << (int32_t)split_request.offset << " size=" << (int32_t)(split_request.size) << " bytes)";
475#endif
476 BlockRegion *next_region = block_region->next_ptr;
477 BlockRegion *empty_region = create_block_region(user_context, split_request);
478 halide_abort_if_false(user_context, empty_region != nullptr);
479
480 empty_region->next_ptr = next_region;
481 if (next_region) {
482 next_region->prev_ptr = empty_region;
483 }
484 empty_region->prev_ptr = block_region;
485 block_region->next_ptr = empty_region;
486 block_region->memory.size -= empty_region->memory.size;
487 return empty_region;
488}
489
490BlockRegion *RegionAllocator::create_block_region(void *user_context, const MemoryRequest &request) {
491#ifdef DEBUG_RUNTIME_INTERNAL
492 debug(user_context) << "RegionAllocator: Creating block region request ("
493 << "user_context=" << (void *)(user_context) << " "
494 << "offset=" << (uint32_t)request.offset << " "
495 << "size=" << (uint32_t)request.size << " "
496 << "alignment=" << (uint32_t)request.properties.alignment << " "
497 << "dedicated=" << (request.dedicated ? "true" : "false") << " "
498 << "usage=" << halide_memory_usage_name(request.properties.usage) << " "
499 << "caching=" << halide_memory_caching_name(request.properties.caching) << " "
500 << "visibility=" << halide_memory_visibility_name(request.properties.visibility) << ") ...";
501#endif
502
503 MemoryRequest region_request = request;
504 int error_code = conform(user_context, &region_request);
505 if (error_code) {
506#ifdef DEBUG_RUNTIME_INTERNAL
507 debug(user_context) << "RegionAllocator: Failed to conform request for new block region!\n";
508#endif
509 return nullptr;
510 }
511
512 if (region_request.size == 0) {
513#ifdef DEBUG_RUNTIME_INTERNAL
514 debug(user_context) << "RegionAllocator: Failed to allocate new block region ... region size was zero!\n";
515#endif
516 return nullptr;
517 }
518
519 BlockRegion *block_region = static_cast<BlockRegion *>(arena->reserve(user_context, true));
520 if (block_region == nullptr) {
521#ifdef DEBUG_RUNTIME_INTERNAL
522 debug(user_context) << "RegionAllocator: Failed to allocate new block region!\n";
523#endif
524 return nullptr;
525 }
526
527 block_region->memory.handle = nullptr;
528 block_region->memory.offset = region_request.offset;
529 block_region->memory.size = region_request.size;
530 block_region->memory.properties = region_request.properties;
531 block_region->memory.dedicated = region_request.dedicated;
532 block_region->status = AllocationStatus::Available;
533 block_region->block_ptr = block;
534 block_region->usage_count = 0;
535
536#ifdef DEBUG_RUNTIME_INTERNAL
537 debug(user_context) << "RegionAllocator: Created block region allocation ("
538 << "user_context=" << (void *)(user_context) << " "
539 << "block_ptr=" << (void *)block_region->block_ptr << " "
540 << "block_region=" << (void *)block_region << " "
541 << "memory_offset=" << (uint32_t)(block_region->memory.offset) << " "
542 << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
543 << ")";
544#endif
545
546 return block_region;
547}
548
549int RegionAllocator::release_block_region(void *user_context, BlockRegion *block_region) {
550#ifdef DEBUG_RUNTIME_INTERNAL
551 debug(user_context) << "RegionAllocator: Releasing block region ("
552 << "user_context=" << (void *)(user_context) << " "
553 << "block_ptr=" << ((block_region) ? ((void *)block_region->block_ptr) : nullptr) << " "
554 << "block_region=" << (void *)block_region << " "
555 << "usage_count=" << ((block_region) ? (uint32_t)(block_region->usage_count) : 0) << " "
556 << "memory_offset=" << ((block_region) ? (uint32_t)(block_region->memory.offset) : 0) << " "
557 << "memory_size=" << ((block_region) ? (uint32_t)(block_region->memory.size) : 0) << " "
558 << "block_reserved=" << (uint32_t)(block->reserved) << ") ... ";
559#endif
560 if (block_region == nullptr) {
561 return 0;
562 }
563
564 if (block_region->usage_count > 0) {
565 return 0;
566 }
567
568 if (block_region->status != AllocationStatus::Available) {
569
570#ifdef DEBUG_RUNTIME_INTERNAL
571 debug(user_context) << " releasing region ("
572 << "block_ptr=" << (void *)block_region->block_ptr << " "
573 << "block_region=" << (void *)block_region << " "
574 << "memory_offset=" << (uint32_t)(block_region->memory.offset) << " "
575 << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
576 << "block_reserved=" << (uint32_t)(block->reserved - block_region->memory.size) << " "
577 << ")";
578#endif
579
580 block->reserved -= block_region->memory.size;
581 }
582 block_region->status = AllocationStatus::Available;
583 return 0;
584}
585
586int RegionAllocator::destroy_block_region(void *user_context, BlockRegion *block_region) {
587#ifdef DEBUG_RUNTIME_INTERNAL
588 debug(user_context) << "RegionAllocator: Destroying block region ("
589 << "user_context=" << (void *)(user_context) << " "
590 << "block_region=" << (void *)(block_region) << ") ...";
591#endif
592
593 block_region->usage_count = 0;
594 release_block_region(user_context, block_region);
595 free_block_region(user_context, block_region);
596 arena->reclaim(user_context, block_region);
597 return 0;
598}
599
600int RegionAllocator::alloc_block_region(void *user_context, BlockRegion *block_region) {
601#ifdef DEBUG_RUNTIME_INTERNAL
602 debug(user_context) << "RegionAllocator: Allocating region (user_context=" << (void *)(user_context)
603 << " size=" << (int32_t)(block_region->memory.size)
604 << " offset=" << (int32_t)block_region->memory.offset << ")";
605#endif
606 halide_abort_if_false(user_context, allocators.region.allocate != nullptr);
607 halide_abort_if_false(user_context, block_region->status == AllocationStatus::Available);
608 int error_code = 0;
609 MemoryRegion *memory_region = &(block_region->memory);
610 if (memory_region->handle == nullptr) {
611 error_code = allocators.region.allocate(user_context, memory_region);
612 memory_region->is_owner = true;
613
614#ifdef DEBUG_RUNTIME_INTERNAL
615 debug(user_context) << " allocating region ("
616 << "block_ptr=" << (void *)block_region->block_ptr << " "
617 << "block_region=" << (void *)block_region << " "
618 << "memory_offset=" << (uint32_t)(block_region->memory.offset) << " "
619 << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
620 << "block_reserved=" << (uint32_t)block->reserved << " "
621 << ")";
622#endif
623
624 } else {
625
626#ifdef DEBUG_RUNTIME_INTERNAL
627 debug(user_context) << " re-using region ("
628 << "block_ptr=" << (void *)block_region->block_ptr << " "
629 << "block_region=" << (void *)block_region << " "
630 << "memory_offset=" << (uint32_t)(block_region->memory.offset) << " "
631 << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
632 << "block_reserved=" << (uint32_t)block->reserved << " "
633 << ")";
634#endif
635 }
636 block_region->status = block_region->memory.dedicated ? AllocationStatus::Dedicated : AllocationStatus::InUse;
637 block->reserved += block_region->memory.size;
638 return error_code;
639}
640
641int RegionAllocator::free_block_region(void *user_context, BlockRegion *block_region) {
642#ifdef DEBUG_RUNTIME_INTERNAL
643 debug(user_context) << "RegionAllocator: Freeing block region ("
644 << "user_context=" << (void *)(user_context) << " "
645 << "block_ptr=" << (void *)block_region->block_ptr << " "
646 << "block_region=" << (void *)(block_region) << " "
647 << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
648 << "status=" << (uint32_t)block_region->status << " "
649 << "usage_count=" << (uint32_t)block_region->usage_count << " "
650 << "block_reserved=" << (uint32_t)block->reserved << ")";
651#endif
652 if ((block_region->usage_count == 0) && (block_region->memory.handle != nullptr)) {
653#ifdef DEBUG_RUNTIME_INTERNAL
654 debug(user_context) << " deallocating region ("
655 << "block_ptr=" << (void *)block_region->block_ptr << " "
656 << "block_region=" << (void *)block_region << " "
657 << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
658 << "block_reserved=" << (uint32_t)block->reserved << " "
659 << ")";
660#endif
661 // NOTE: Deallocate but leave memory size as is, so that coalesce can compute region merging sizes
662 halide_abort_if_false(user_context, allocators.region.deallocate != nullptr);
663 MemoryRegion *memory_region = &(block_region->memory);
664 allocators.region.deallocate(user_context, memory_region);
665 block_region->memory.handle = nullptr;
666 }
667 block_region->usage_count = 0;
668 block_region->status = AllocationStatus::Available;
669 return 0;
670}
671
672int RegionAllocator::release(void *user_context) {
673#ifdef DEBUG_RUNTIME_INTERNAL
674 debug(user_context) << "RegionAllocator: Releasing all regions ("
675 << "user_context=" << (void *)(user_context) << ") ...";
676#endif
677
678 BlockRegion *block_region = block->regions;
679 while (block_region != nullptr) {
680 release_block_region(user_context, block_region);
681 if (is_last_block_region(user_context, block_region)) {
682 break;
683 }
684 block_region = block_region->next_ptr;
685 }
686 return 0;
687}
688
689bool RegionAllocator::collect(void *user_context) {
690#ifdef DEBUG_RUNTIME_INTERNAL
691 debug(user_context) << "RegionAllocator: Collecting free block regions ("
692 << "user_context=" << (void *)(user_context) << ") ...";
693
694 uint32_t collected_count = 0;
695 uint32_t remaining_count = 0;
696 uint64_t available_bytes = 0;
697 uint64_t scanned_bytes = 0;
698 uint64_t reserved = block->reserved;
699 debug(user_context) << " collecting unused regions ("
700 << "block_ptr=" << (void *)block << " "
701 << "block_reserved=" << (uint32_t)block->reserved << " "
702 << ")";
703#endif
704
705 bool has_collected = false;
706 BlockRegion *block_region = block->regions;
707 while (block_region != nullptr) {
708#ifdef DEBUG_RUNTIME_INTERNAL
709 scanned_bytes += block_region->memory.size;
710 debug(user_context) << " checking region ("
711 << "block_ptr=" << (void *)block_region->block_ptr << " "
712 << "block_region=" << (void *)block_region << " "
713 << "usage_count=" << (uint32_t)(block_region->usage_count) << " "
714 << "status=" << (uint32_t)(block_region->status) << " "
715 << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
716 << "block_reserved=" << (uint32_t)block->reserved << " "
717 << ")";
718#endif
719
720 if (can_coalesce(block_region)) {
721#ifdef DEBUG_RUNTIME_INTERNAL
722 collected_count++;
723 debug(user_context) << " collecting region ("
724 << "block_ptr=" << (void *)block_region->block_ptr << " "
725 << "block_region=" << (void *)block_region << " "
726 << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
727 << "block_reserved=" << (uint32_t)block->reserved << " "
728 << ")";
729#endif
730 block_region = coalesce_block_regions(user_context, block_region);
731 has_collected = true;
732 } else {
733#ifdef DEBUG_RUNTIME_INTERNAL
734 remaining_count++;
735#endif
736 }
737#ifdef DEBUG_RUNTIME_INTERNAL
738 available_bytes += is_available(block_region) ? block_region->memory.size : 0;
739#endif
740 if (is_last_block_region(user_context, block_region)) {
741 break;
742 }
743 block_region = block_region->next_ptr;
744 }
745#ifdef DEBUG_RUNTIME_INTERNAL
746 debug(user_context) << " scanned active regions ("
747 << "block_ptr=" << (void *)block << " "
748 << "total_count=" << (uint32_t)(collected_count + remaining_count) << " "
749 << "block_reserved=" << (uint32_t)(block->reserved) << " "
750 << "scanned_bytes=" << (uint32_t)(scanned_bytes) << " "
751 << "available_bytes=" << (uint32_t)(available_bytes) << " "
752 << ")";
753#endif
754
755 if (has_collected) {
756#ifdef DEBUG_RUNTIME_INTERNAL
757 debug(user_context) << " collected unused regions ("
758 << "block_ptr=" << (void *)block << " "
759 << "collected_count=" << (uint32_t)collected_count << " "
760 << "remaining_count=" << (uint32_t)remaining_count << " "
761 << "reclaimed=" << (uint32_t)(reserved - block->reserved) << " "
762 << ")";
763#endif
764 }
765 return has_collected;
766}
767
768int RegionAllocator::destroy(void *user_context) {
769#ifdef DEBUG_RUNTIME_INTERNAL
770 debug(user_context) << "RegionAllocator: Destroying all block regions ("
771 << "user_context=" << (void *)(user_context) << ") ...";
772#endif
773 if (block->regions != nullptr) {
774 for (BlockRegion *block_region = block->regions; block_region != nullptr;) {
775
776 if (is_last_block_region(user_context, block_region)) {
777 destroy_block_region(user_context, block_region);
778 block_region = nullptr;
779 } else {
780 BlockRegion *prev_region = block_region;
781 block_region = block_region->next_ptr;
782 destroy_block_region(user_context, prev_region);
783 }
784 }
785 }
786 block->reserved = 0;
787 block->regions = nullptr;
788 block->allocator = nullptr;
789 if (arena != nullptr) {
790 MemoryArena::destroy(user_context, arena);
791 }
792 arena = nullptr;
793 return 0;
794}
795
796bool RegionAllocator::is_compatible_block_region(const BlockRegion *block_region, const MemoryProperties &properties) const {
797 if (properties.caching != MemoryCaching::DefaultCaching) {
798 if (properties.caching != block_region->memory.properties.caching) {
799 return false;
800 }
801 }
802
803 if (properties.visibility != MemoryVisibility::DefaultVisibility) {
804 if (properties.visibility != block_region->memory.properties.visibility) {
805 return false;
806 }
807 }
808
809 if (properties.usage != MemoryUsage::DefaultUsage) {
810 if (properties.usage != block_region->memory.properties.usage) {
811 return false;
812 }
813 }
814
815 return true;
816}
817
818size_t RegionAllocator::region_count(void *user_context) const {
819 if (block == nullptr) {
820 return 0;
821 }
822 size_t count = 0;
823 for (BlockRegion const *region = block->regions; !is_last_block_region(user_context, region); region = region->next_ptr) {
824 ++count;
825 }
826 return count;
827}
828
829BlockResource *RegionAllocator::block_resource() const {
830 return block;
831}
832
833// --
834
835} // namespace Internal
836} // namespace Runtime
837} // namespace Halide
838
839#endif // HALIDE_RUNTIME_REGION_ALLOCATOR_H
HalideRuntime.h
This file declares the routines used by Halide internally in its runtime.
Halide::Runtime::Internal::MemoryArena::reserve
void * reserve(void *user_context, bool initialize=false)
Definition memory_arena.h:155
Halide::Runtime::Internal::MemoryArena::create
static MemoryArena * create(void *user_context, const Config &config, const SystemMemoryAllocatorFns &allocator=default_allocator())
Definition memory_arena.h:101
Halide::Runtime::Internal::MemoryArena::default_capacity
static constexpr uint32_t default_capacity
Definition memory_arena.h:24
Halide::Runtime::Internal::MemoryArena::destroy
static void destroy(void *user_context, MemoryArena *arena)
Definition memory_arena.h:115
Halide::Runtime::Internal::MemoryArena::reclaim
void reclaim(void *user_context, void *ptr)
Definition memory_arena.h:182
Halide::Runtime::Internal::RegionAllocator
Allocator class interface for sub-allocating a contiguous memory block into smaller regions of memory...
Definition region_allocator.h:24
Halide::Runtime::Internal::RegionAllocator::reserve
MemoryRegion * reserve(void *user_context, const MemoryRequest &request)
Definition region_allocator.h:166
Halide::Runtime::Internal::RegionAllocator::operator=
RegionAllocator & operator=(const RegionAllocator &)=delete
Halide::Runtime::Internal::RegionAllocator::RegionAllocator
RegionAllocator(const RegionAllocator &)=delete
Halide::Runtime::Internal::RegionAllocator::destroy
static int destroy(void *user_context, RegionAllocator *region_allocator)
Definition region_allocator.h:127
Halide::Runtime::Internal::RegionAllocator::conform
int conform(void *user_context, MemoryRequest *request) const
Definition region_allocator.h:152
Halide::Runtime::Internal::RegionAllocator::find_allocator
static RegionAllocator * find_allocator(void *user_context, MemoryRegion *memory_region)
Definition region_allocator.h:240
Halide::Runtime::Internal::RegionAllocator::retain
int retain(void *user_context, MemoryRegion *memory_region)
Definition region_allocator.h:232
Halide::Runtime::Internal::RegionAllocator::reclaim
int reclaim(void *user_context, MemoryRegion *memory_region)
Definition region_allocator.h:220
Halide::Runtime::Internal::RegionAllocator::release
int release(void *user_context, MemoryRegion *memory_region)
Definition region_allocator.h:210
Halide::Runtime::Internal::RegionAllocator::create
static RegionAllocator * create(void *user_context, BlockResource *block, const MemoryAllocators &ma)
Definition region_allocator.h:114
memory_arena.h
memory_resources.h
halide_memory_caching_name
WEAK const char * halide_memory_caching_name(MemoryCaching value)
Definition memory_resources.h:293
halide_memory_usage_name
WEAK const char * halide_memory_usage_name(MemoryUsage value)
Definition memory_resources.h:260
halide_memory_visibility_name
WEAK const char * halide_memory_visibility_name(MemoryVisibility value)
Definition memory_resources.h:233
Halide::Runtime::Internal::conform_alignment
ALWAYS_INLINE size_t conform_alignment(size_t requested, size_t required)
Definition memory_resources.h:135
Halide::Runtime::Internal::conform_size
ALWAYS_INLINE size_t conform_size(size_t offset, size_t size, size_t alignment, size_t nearest_multiple)
Definition memory_resources.h:151
Halide::Runtime::Internal::aligned_offset
ALWAYS_INLINE size_t aligned_offset(size_t offset, size_t alignment)
Definition memory_resources.h:128
Halide
This file defines the class FunctionDAG, which is our representation of a Halide pipeline,...
Definition AbstractGenerator.h:19
Halide::LinkageType::Internal
@ Internal
Not visible externally, similar to 'static' linkage in C.
uint64_t
unsigned __INT64_TYPE__ uint64_t
Definition runtime_internal.h:23
int32_t
signed __INT32_TYPE__ int32_t
Definition runtime_internal.h:24
uint32_t
unsigned __INT32_TYPE__ uint32_t
Definition runtime_internal.h:25
halide_abort_if_false
#define halide_abort_if_false(user_context, cond)
Definition runtime_internal.h:271