6#ifndef HALIDE_RUNTIME_BUFFER_H
7#define HALIDE_RUNTIME_BUFFER_H
21#include <AvailabilityVersions.h>
22#include <TargetConditionals.h>
25#if defined(__has_feature)
26#if __has_feature(memory_sanitizer)
27#include <sanitizer/msan_interface.h>
35#define HALIDE_ALLOCA _alloca
37#define HALIDE_ALLOCA __builtin_alloca
41#if __GNUC__ == 5 && __GNUC_MINOR__ == 1
42#pragma GCC diagnostic ignored "-Warray-bounds"
45#ifndef HALIDE_RUNTIME_BUFFER_CHECK_INDICES
46#define HALIDE_RUNTIME_BUFFER_CHECK_INDICES 0
49#ifndef HALIDE_RUNTIME_BUFFER_ALLOCATION_ALIGNMENT
53#define HALIDE_RUNTIME_BUFFER_ALLOCATION_ALIGNMENT 128
57 "HALIDE_RUNTIME_BUFFER_ALLOCATION_ALIGNMENT must be a power of 2.");
65#ifndef HALIDE_RUNTIME_BUFFER_USE_ALIGNED_ALLOC
74 #define HALIDE_RUNTIME_BUFFER_USE_ALIGNED_ALLOC 0
76#elif defined(__ANDROID_API__) && __ANDROID_API__ < 28
79 #define HALIDE_RUNTIME_BUFFER_USE_ALIGNED_ALLOC 0
81#elif defined(__APPLE__)
83 #if TARGET_OS_OSX && (__MAC_OS_X_VERSION_MIN_REQUIRED < __MAC_10_15)
86 #define HALIDE_RUNTIME_BUFFER_USE_ALIGNED_ALLOC 0
88 #elif TARGET_OS_IPHONE && (__IPHONE_OS_VERSION_MIN_REQUIRED < __IPHONE_14_0)
91 #define HALIDE_RUNTIME_BUFFER_USE_ALIGNED_ALLOC 0
96 #define HALIDE_RUNTIME_BUFFER_USE_ALIGNED_ALLOC 1
102 #if defined(__GLIBCXX__) && !defined(_GLIBCXX_HAVE_ALIGNED_ALLOC)
105 #define HALIDE_RUNTIME_BUFFER_USE_ALIGNED_ALLOC 0
110 #define HALIDE_RUNTIME_BUFFER_USE_ALIGNED_ALLOC 1
123template<
typename T,
int Dims,
int InClassDimStorage>
128template<
typename... Args>
134template<
typename T,
typename... Args>
142template<
typename... Args>
143struct AllInts<float, Args...> : std::false_type {};
145template<
typename... Args>
146struct AllInts<double, Args...> : std::false_type {};
150template<
typename Container>
161 static inline void *(*default_allocate_fn)(
size_t) =
nullptr;
221template<
typename T = void,
223 int InClassDimStorage = (Dims ==
AnyDims ? 4 : std::max(Dims, 1))>
233 AllocationHeader *alloc =
nullptr;
237 mutable DeviceRefCount *dev_ref_count =
nullptr;
240 static const bool T_is_void = std::is_same<typename std::remove_const<T>::type,
void>::value;
243 template<
typename T2>
244 using add_const_if_T_is_const =
typename std::conditional<std::is_const<T>::value,
const T2, T2>
::type;
248 using not_void_T =
typename std::conditional<T_is_void,
249 add_const_if_T_is_const<uint8_t>,
253 using not_const_T =
typename std::remove_const<T>::type;
259 using storage_T =
typename std::conditional<std::is_pointer<T>::value,
uint64_t, not_void_T>
::type;
268 return halide_type_of<typename std::remove_cv<not_void_T>::type>();
273 return alloc !=
nullptr;
288 void incref()
const {
293 if (!dev_ref_count) {
299 dev_ref_count =
new DeviceRefCount;
301 dev_ref_count->count++;
307 struct DevRefCountCropped : DeviceRefCount {
314 Buffer<T, AnyDims> cropped_from;
315 explicit DevRefCountCropped(
const Buffer<T, AnyDims> &cropped_from)
316 : cropped_from(cropped_from) {
323 assert(dev_ref_count ==
nullptr);
324 dev_ref_count =
new DevRefCountCropped(cropped_from);
329 void decref(
bool device_only =
false) {
331 int new_count = --(alloc->ref_count);
332 if (new_count == 0) {
333 void (*fn)(
void *) = alloc->deallocate_fn;
334 alloc->~AllocationHeader();
343 new_count = --(dev_ref_count->count);
345 if (new_count == 0) {
348 "Implicitly freeing a dirty device allocation while a host allocation still lives. "
349 "Call device_free explicitly if you want to drop dirty device-side data. "
350 "Call copy_to_host explicitly if you want the data copied to the host allocation "
351 "before the device allocation is freed.");
354 result = buf.device_interface->detach_native(
nullptr, &buf);
356 result = buf.device_interface->device_and_host_free(
nullptr, &buf);
358 result = buf.device_interface->device_release_crop(
nullptr, &buf);
360 result = buf.device_interface->device_free(
nullptr, &buf);
368 delete (DevRefCountCropped *)dev_ref_count;
370 delete dev_ref_count;
374 dev_ref_count =
nullptr;
376 buf.device_interface =
nullptr;
379 void free_shape_storage() {
380 if (buf.dim != shape) {
386 template<
int DimsSpecified>
387 void make_static_shape_storage() {
388 static_assert(Dims ==
AnyDims || Dims == DimsSpecified,
389 "Number of arguments to Buffer() does not match static dimensionality");
390 buf.dimensions = DimsSpecified;
391 if constexpr (Dims ==
AnyDims) {
392 if constexpr (DimsSpecified <= InClassDimStorage) {
395 static_assert(DimsSpecified >= 1);
399 static_assert(InClassDimStorage >= Dims);
404 void make_shape_storage(
const int dimensions) {
406 assert(
false &&
"Number of arguments to Buffer() does not match static dimensionality");
420 template<
typename T2,
int D2,
int S2>
422 if (other.shape == other.buf.dim) {
423 copy_shape_from(other.buf);
425 buf.dim = other.buf.dim;
426 other.buf.dim =
nullptr;
437 dev_ref_count =
new DeviceRefCount;
438 dev_ref_count->ownership = ownership;
443 void initialize_shape(
const int *sizes) {
444 for (
int i = 0; i < buf.dimensions; i++) {
446 buf.dim[i].extent = sizes[i];
448 buf.dim[i].stride = 1;
450 buf.dim[i].stride = buf.dim[i - 1].stride * buf.dim[i - 1].extent;
456 void initialize_shape(
const std::vector<int> &sizes) {
457 assert(buf.dimensions == (
int)sizes.size());
458 initialize_shape(sizes.data());
462 template<
typename Array,
size_t N>
463 void initialize_shape_from_array_shape(
int next, Array (&vals)[N]) {
464 buf.dim[next].min = 0;
465 buf.dim[next].extent = (int)N;
467 buf.dim[next].stride = 1;
469 initialize_shape_from_array_shape(next - 1, vals[0]);
470 buf.dim[next].stride = buf.dim[next - 1].stride * buf.dim[next - 1].extent;
475 template<
typename T2>
476 void initialize_shape_from_array_shape(
int,
const T2 &) {
480 template<
typename Array,
size_t N>
481 static int dimensionality_of_array(Array (&vals)[N]) {
482 return dimensionality_of_array(vals[0]) + 1;
485 template<
typename T2>
486 static int dimensionality_of_array(
const T2 &) {
491 template<
typename Array,
size_t N>
492 static halide_type_t scalar_type_of_array(Array (&vals)[N]) {
493 return scalar_type_of_array(vals[0]);
496 template<
typename T2>
497 static halide_type_t scalar_type_of_array(
const T2 &) {
498 return halide_type_of<typename std::remove_cv<T2>::type>();
502 void crop_host(
int d,
int min,
int extent) {
506 if (buf.host !=
nullptr) {
509 buf.dim[d].min =
min;
510 buf.dim[d].extent =
extent;
514 void crop_host(
const std::vector<std::pair<int, int>> &rect) {
515 assert(rect.size() <=
static_cast<decltype(rect.size())
>(std::numeric_limits<int>::max()));
516 int limit = (int)rect.size();
518 for (
int i = 0; i < limit; i++) {
519 crop_host(i, rect[i].first, rect[i].second);
524 assert(buf.device_interface !=
nullptr);
531 result_host_cropped.crop_from(((DevRefCountCropped *)dev_ref_count)->cropped_from);
533 result_host_cropped.crop_from(*
this);
539 void slice_host(
int d,
int pos) {
540 static_assert(Dims ==
AnyDims);
546 if (buf.host !=
nullptr) {
547 buf.host += (shift * buf.dim[d].stride) *
type().bytes();
549 for (
int i = d; i < buf.dimensions; i++) {
550 buf.dim[i] = buf.dim[i + 1];
552 buf.dim[buf.dimensions] = {0, 0, 0};
556 assert(buf.device_interface !=
nullptr);
557 if (buf.device_interface->device_slice(
nullptr, &this->buf, d, pos, &result_host_sliced.buf) ==
halide_error_code_success) {
564 result_host_sliced.crop_from(((DevRefCountCropped *)dev_ref_count)->cropped_from);
567 result_host_sliced.crop_from(*
this);
654 return buf.number_of_elements();
662 return buf.dimensions;
674 assert(buf.host !=
nullptr);
675 return (T *)buf.begin();
680 assert(buf.host !=
nullptr);
681 return (T *)buf.end();
686 return buf.size_in_bytes();
701 constexpr int buf_dimensions = (Dims ==
AnyDims) ? 0 : Dims;
702 make_static_shape_storage<buf_dimensions>();
709 initialize_from_buffer(buf, ownership);
713 template<
typename T2,
int D2,
int S2>
717 template<
typename T2,
int D2,
int S2>
718 static void static_assert_can_convert_from() {
719 static_assert((!std::is_const<T2>::value || std::is_const<T>::value),
720 "Can't convert from a Buffer<const T> to a Buffer<T>");
721 static_assert(std::is_same<typename std::remove_const<T>::type,
722 typename std::remove_const<T2>::type>::value ||
723 T_is_void || Buffer<T2, D2, S2>::T_is_void,
724 "type mismatch constructing Buffer");
726 "Can't convert from a Buffer with static dimensionality to a Buffer with different static dimensionality");
740 template<
typename T2,
int D2,
int S2>
742 static_assert_can_convert_from<T2, D2, S2>();
743 if (Buffer<T2, D2, S2>::T_is_void && !T_is_void) {
758 template<
typename T2,
int D2,
int S2>
763 static_assert_can_convert_from<T2, D2, S2>();
772 dev_ref_count = other.dev_ref_count;
773 copy_shape_from(other.buf);
782 template<
typename T2,
int D2,
int S2>
788 dev_ref_count = other.dev_ref_count;
789 copy_shape_from(other.buf);
796 dev_ref_count(other.dev_ref_count) {
797 other.dev_ref_count =
nullptr;
798 other.alloc =
nullptr;
805 template<
typename T2,
int D2,
int S2>
809 dev_ref_count(other.dev_ref_count) {
811 other.dev_ref_count =
nullptr;
812 other.alloc =
nullptr;
819 template<
typename T2,
int D2,
int S2>
821 if ((
const void *)
this == (
const void *)&other) {
827 dev_ref_count = other.dev_ref_count;
829 free_shape_storage();
831 copy_shape_from(other.buf);
838 if ((
const void *)
this == (
const void *)&other) {
843 dev_ref_count = other.dev_ref_count;
845 free_shape_storage();
847 copy_shape_from(other.buf);
854 template<
typename T2,
int D2,
int S2>
859 other.alloc =
nullptr;
860 dev_ref_count = other.dev_ref_count;
861 other.dev_ref_count =
nullptr;
862 free_shape_storage();
872 other.alloc =
nullptr;
873 dev_ref_count = other.dev_ref_count;
874 other.dev_ref_count =
nullptr;
875 free_shape_storage();
883 size_t size =
type().bytes();
888 size = (size << 1) >> 1;
892 assert(size == (
size_t)
type().bytes() &&
"Error: Overflow computing total size of buffer.");
897 void allocate(
void *(*allocate_fn)(
size_t) =
nullptr,
898 void (*deallocate_fn)(
void *) =
nullptr) {
907 const auto align_up = [=](
size_t value) ->
size_t {
908 return (value + alignment - 1) & ~(alignment - 1);
913#if HALIDE_RUNTIME_BUFFER_USE_ALIGNED_ALLOC
921 void *alloc_storage = ::aligned_alloc(alignment,
align_up(size) + alignment);
935 if (!deallocate_fn) {
937 if (!deallocate_fn) {
938 deallocate_fn =
free;
948 const size_t requested_size =
align_up(size + alignment +
950 (
int)
sizeof(std::max_align_t)));
951 void *alloc_storage = allocate_fn(requested_size);
976 template<
typename... Args,
977 typename =
typename std::enable_if<
AllInts<Args...>::value>
::type>
982 int extents[] = {first, (int)rest...};
984 constexpr int buf_dimensions = 1 + (int)(
sizeof...(rest));
985 make_static_shape_storage<buf_dimensions>();
986 initialize_shape(extents);
1000 static_assert(!T_is_void,
1001 "To construct an Buffer<void>, pass a halide_type_t as the first argument to the constructor");
1002 int extents[] = {first};
1004 constexpr int buf_dimensions = 1;
1005 make_static_shape_storage<buf_dimensions>();
1006 initialize_shape(extents);
1013 template<
typename... Args,
1014 typename =
typename std::enable_if<
AllInts<Args...>::value>
::type>
1015 Buffer(
int first,
int second, Args... rest) {
1016 static_assert(!T_is_void,
1017 "To construct an Buffer<void>, pass a halide_type_t as the first argument to the constructor");
1018 int extents[] = {first, second, (int)rest...};
1020 constexpr int buf_dimensions = 2 + (int)(
sizeof...(rest));
1021 make_static_shape_storage<buf_dimensions>();
1022 initialize_shape(extents);
1037 make_shape_storage((
int)sizes.size());
1038 initialize_shape(sizes);
1046 explicit Buffer(
const std::vector<int> &sizes)
1052 static std::vector<int> make_ordered_sizes(
const std::vector<int> &sizes,
const std::vector<int> &order) {
1053 assert(order.size() == sizes.size());
1054 std::vector<int> ordered_sizes(sizes.size());
1055 for (
size_t i = 0; i < sizes.size(); ++i) {
1056 ordered_sizes[i] = sizes.at(order[i]);
1058 return ordered_sizes;
1067 :
Buffer(t, make_ordered_sizes(sizes, storage_order)) {
1071 Buffer(
const std::vector<int> &sizes,
const std::vector<int> &storage_order)
1077 template<
typename Array,
size_t N>
1079 const int buf_dimensions = dimensionality_of_array(vals);
1080 buf.type = scalar_type_of_array(vals);
1082 make_shape_storage(buf_dimensions);
1083 initialize_shape_from_array_shape(buf.dimensions - 1, vals);
1090 template<
typename... Args,
1091 typename =
typename std::enable_if<
AllInts<Args...>::value>
::type>
1096 int extents[] = {first, (int)rest...};
1099 constexpr int buf_dimensions = 1 + (int)(
sizeof...(rest));
1100 make_static_shape_storage<buf_dimensions>();
1101 initialize_shape(extents);
1107 template<
typename... Args,
1108 typename =
typename std::enable_if<
AllInts<Args...>::value>
::type>
1110 int extents[] = {first, (int)rest...};
1112 buf.host = (
uint8_t *)
const_cast<typename std::remove_const<T>::type *
>(
data);
1113 constexpr int buf_dimensions = 1 + (int)(
sizeof...(rest));
1114 make_static_shape_storage<buf_dimensions>();
1115 initialize_shape(extents);
1124 buf.host = (
uint8_t *)
const_cast<typename std::remove_const<T>::type *
>(
data);
1125 make_shape_storage((
int)sizes.size());
1126 initialize_shape(sizes);
1139 make_shape_storage((
int)sizes.size());
1140 initialize_shape(sizes);
1152 make_shape_storage(d);
1153 for (
int i = 0; i < d; i++) {
1154 buf.dim[i] = shape[i];
1162 const std::vector<halide_dimension_t> &shape)
1171 buf.host = (
uint8_t *)
const_cast<typename std::remove_const<T>::type *
>(
data);
1172 make_shape_storage(d);
1173 for (
int i = 0; i < d; i++) {
1174 buf.dim[i] = shape[i];
1181 explicit Buffer(T *
data,
const std::vector<halide_dimension_t> &shape)
1190 free_shape_storage();
1217 template<
typename T2,
int D2 = Dims>
1230 template<
typename T2,
int D2 = Dims>
1243 template<
typename T2,
int D2 = Dims>
1272 template<typename T2 = T, typename = typename std::enable_if<!std::is_const<T2>::value>
::type>
1279 template<
typename TVoid,
1281 typename =
typename std::enable_if<std::is_same<TVoid, void>::value &&
1282 !std::is_void<T2>::value &&
1283 !std::is_const<T2>::value>
::type>
1290 template<
typename TVoid,
1292 typename =
typename std::enable_if<std::is_same<TVoid, void>::value &&
1293 !std::is_void<T2>::value &&
1294 std::is_const<T2>::value>
::type>
1347 void (*deallocate_fn)(
void *) =
nullptr)
const {
1358 void (*deallocate_fn)(
void *) =
nullptr)
const {
1359 static_assert(Dims ==
AnyDims || Dims == 3);
1363 dst.
allocate(allocate_fn, deallocate_fn);
1372 void (*deallocate_fn)(
void *) =
nullptr)
const {
1373 std::vector<int> mins, extents;
1376 extents.reserve(dims);
1377 for (
int d = 0; d < dims; ++d) {
1378 mins.push_back(
dim(d).
min());
1383 dst.
allocate(allocate_fn, deallocate_fn);
1410 template<
typename T2,
int D2,
int S2>
1412 static_assert(!std::is_const<T>::value,
"Cannot call copy_from() on a Buffer<const T>");
1413 assert(!
device_dirty() &&
"Cannot call Halide::Runtime::Buffer::copy_from on a device dirty destination.");
1414 assert(!src.
device_dirty() &&
"Cannot call Halide::Runtime::Buffer::copy_from on a device dirty source.");
1423 for (
int i = 0; i < d; i++) {
1424 int min_coord = std::max(dst.
dim(i).
min(), src.
dim(i).
min());
1425 int max_coord = std::min(dst.
dim(i).
max(), src.
dim(i).
max());
1426 if (max_coord < min_coord) {
1430 dst.
crop(i, min_coord, max_coord - min_coord + 1);
1431 src.
crop(i, min_coord, max_coord - min_coord + 1);
1438 if (T_is_void ? (
type().bytes() == 1) : (
sizeof(not_void_T) == 1)) {
1442 typed_dst.for_each_value([&](MemType &dst, MemType src) { dst = src; }, typed_src);
1443 }
else if (T_is_void ? (
type().bytes() == 2) : (
sizeof(not_void_T) == 2)) {
1447 typed_dst.for_each_value([&](MemType &dst, MemType src) { dst = src; }, typed_src);
1448 }
else if (T_is_void ? (
type().bytes() == 4) : (
sizeof(not_void_T) == 4)) {
1452 typed_dst.for_each_value([&](MemType &dst, MemType src) { dst = src; }, typed_src);
1453 }
else if (T_is_void ? (
type().bytes() == 8) : (
sizeof(not_void_T) == 8)) {
1457 typed_dst.for_each_value([&](MemType &dst, MemType src) { dst = src; }, typed_src);
1459 assert(
false &&
"type().bytes() must be 1, 2, 4, or 8");
1479 if (buf.device_interface !=
nullptr) {
1480 complete_device_crop(im);
1493 if (buf.device_interface !=
nullptr) {
1515 if (buf.device_interface !=
nullptr) {
1516 complete_device_crop(im);
1525 void crop(
const std::vector<std::pair<int, int>> &rect) {
1530 if (buf.device_interface !=
nullptr) {
1552 buf.dim[d].min += delta;
1567 assert(delta.size() <=
static_cast<decltype(delta.size())
>(std::numeric_limits<int>::max()));
1568 int limit = (int)delta.size();
1570 for (
int i = 0; i < limit; i++) {
1578 assert(mins.size() <=
static_cast<decltype(mins.size())
>(
dimensions()));
1580 for (
size_t i = 0; i < mins.size(); i++) {
1581 buf.dim[i].min = mins[i];
1585 template<
typename... Args>
1587 set_min(std::vector<int>{args...});
1594 assert(coords.size() <=
static_cast<decltype(coords.size())
>(
dimensions()));
1595 for (
size_t i = 0; i < coords.size(); i++) {
1596 if (coords[i] <
dim((
int)i).
min() || coords[i] >
dim((
int)i).
max()) {
1603 template<
typename... Args>
1605 return contains(std::vector<int>{args...});
1629 std::swap(buf.dim[d1], buf.dim[d2]);
1643 std::vector<int> order_sorted = order;
1644 for (
size_t i = 1; i < order_sorted.size(); i++) {
1645 for (
size_t j = i; j > 0 && order_sorted[j - 1] > order_sorted[j]; j--) {
1646 std::swap(order_sorted[j], order_sorted[j - 1]);
1664 static_assert(Dims ==
AnyDims || Dims > 0,
"Cannot slice a 0-dimensional buffer");
1674 im.slice_host(d, pos);
1675 if (buf.device_interface !=
nullptr) {
1676 complete_device_slice(im, d, pos);
1685 static_assert(Dims ==
AnyDims || Dims > 0,
"Cannot slice a 0-dimensional buffer");
1697 static_assert(Dims ==
AnyDims,
"Cannot call slice() on a Buffer with static dimensionality.");
1704 if (buf.device_interface !=
nullptr) {
1736 static_assert(Dims ==
AnyDims,
"Cannot call embed() on a Buffer with static dimensionality.");
1750 static_assert(Dims ==
AnyDims,
"Cannot call add_dimension() on a Buffer with static dimensionality.");
1751 const int dims = buf.dimensions;
1753 if (buf.dim != shape) {
1756 for (
int i = 0; i < dims; i++) {
1757 new_shape[i] = buf.dim[i];
1760 buf.dim = new_shape;
1761 }
else if (dims == InClassDimStorage) {
1763 make_shape_storage(buf.dimensions);
1764 for (
int i = 0; i < dims; i++) {
1765 buf.dim[i] = shape[i];
1770 buf.dim[dims] = {0, 1, 0};
1772 buf.dim[dims].stride = 1;
1774 buf.dim[dims].stride = buf.dim[dims - 1].extent * buf.dim[dims - 1].stride;
1783 buf.dim[buf.dimensions - 1].stride = s;
1792 assert((!v || !
device_dirty()) &&
"Cannot set host dirty when device is already dirty. Call copy_to_host() before accessing the buffer from host.");
1793 buf.set_host_dirty(v);
1801 return buf.device_dirty();
1805 return buf.host_dirty();
1809 assert((!v || !
host_dirty()) &&
"Cannot set device dirty when host is already dirty.");
1810 buf.set_device_dirty(v);
1815 return buf.device_interface->copy_to_host(ctx, &buf);
1822 return device_interface->
copy_to_device(ctx, &buf, device_interface);
1828 return device_interface->
device_malloc(ctx, &buf, device_interface);
1832 if (dev_ref_count) {
1834 "Can't call device_free on an unmanaged or wrapped native device handle. "
1835 "Free the source allocation or call device_detach_native instead.");
1837 assert(dev_ref_count->count == 1 &&
1838 "Multiple Halide::Runtime::Buffer objects share this device "
1839 "allocation. Freeing it would create dangling references. "
1840 "Don't call device_free on Halide buffers that you have copied or "
1841 "passed by value.");
1844 if (buf.device_interface) {
1845 ret = buf.device_interface->device_free(ctx, &buf);
1847 if (dev_ref_count) {
1848 delete dev_ref_count;
1849 dev_ref_count =
nullptr;
1855 uint64_t handle,
void *ctx =
nullptr) {
1856 assert(device_interface);
1859 return device_interface->
wrap_native(ctx, &buf, handle, device_interface);
1863 assert(dev_ref_count &&
1865 "Only call device_detach_native on buffers wrapping a native "
1866 "device handle via device_wrap_native. This buffer was allocated "
1867 "using device_malloc, or is unmanaged. "
1868 "Call device_free or free the original allocation instead.");
1870 assert(dev_ref_count->count == 1 &&
1871 "Multiple Halide::Runtime::Buffer objects share this device "
1872 "allocation. Freeing it could create dangling references. "
1873 "Don't call device_detach_native on Halide buffers that you "
1874 "have copied or passed by value.");
1876 if (buf.device_interface) {
1877 ret = buf.device_interface->detach_native(ctx, &buf);
1879 delete dev_ref_count;
1880 dev_ref_count =
nullptr;
1889 if (dev_ref_count) {
1891 "Can't call device_and_host_free on a device handle not allocated with device_and_host_malloc. "
1892 "Free the source allocation or call device_detach_native instead.");
1894 assert(dev_ref_count->count == 1 &&
1895 "Multiple Halide::Runtime::Buffer objects share this device "
1896 "allocation. Freeing it would create dangling references. "
1897 "Don't call device_and_host_free on Halide buffers that you have copied or "
1898 "passed by value.");
1901 if (buf.device_interface) {
1902 ret = buf.device_interface->device_and_host_free(ctx, &buf);
1904 if (dev_ref_count) {
1905 delete dev_ref_count;
1906 dev_ref_count =
nullptr;
1912 return buf.device_sync(ctx);
1916 return buf.device != 0;
1921 if (dev_ref_count ==
nullptr) {
1924 return dev_ref_count->ownership;
1935 static_assert(Dims ==
AnyDims || Dims == 3,
"make_interleaved() must be called on a Buffer that can represent 3 dimensions.");
1957 static_assert(Dims ==
AnyDims || Dims == 3,
"make_interleaved() must be called on a Buffer that can represent 3 dimensions.");
1971 static_assert(Dims ==
AnyDims || Dims == 0,
"make_scalar() must be called on a Buffer that can represent 0 dimensions.");
1979 static_assert(Dims ==
AnyDims || Dims == 0,
"make_scalar() must be called on a Buffer that can represent 0 dimensions.");
1987 static_assert(Dims ==
AnyDims || Dims == 0,
"make_scalar() must be called on a Buffer that can represent 0 dimensions.");
1995 template<
typename T2,
int D2,
int S2>
1998 void *(*allocate_fn)(
size_t) =
nullptr,
1999 void (*deallocate_fn)(
void *) =
nullptr) {
2002 static_assert(Dims == D2 || Dims ==
AnyDims);
2003 const halide_type_t dst_type = T_is_void ? src.
type() : halide_type_of<typename std::remove_cv<not_void_T>::type>();
2004 return Buffer<>::make_with_shape_of_helper(dst_type, src.
dimensions(), src.buf.
dim,
2005 allocate_fn, deallocate_fn);
2012 void *(*allocate_fn)(
size_t),
2013 void (*deallocate_fn)(
void *)) {
2015 std::vector<int> swaps;
2017 for (
int j = i; j > 0; j--) {
2019 std::swap(shape[j - 1], shape[j]);
2029 shape[i].stride = 1;
2031 shape[i].stride = shape[i - 1].extent * shape[i - 1].stride;
2036 while (!swaps.empty()) {
2037 int j = swaps.back();
2038 std::swap(shape[j - 1], shape[j]);
2045 dst.allocate(allocate_fn, deallocate_fn);
2050 template<
typename... Args>
2053 offset_of(
int d,
int first, Args... rest)
const {
2054#if HALIDE_RUNTIME_BUFFER_CHECK_INDICES
2055 assert(first >= this->buf.dim[d].min);
2056 assert(first < this->buf.dim[d].min + this->buf.dim[d].extent);
2058 return offset_of(d + 1, rest...) + (
ptrdiff_t)this->buf.dim[d].stride * (first - this->buf.dim[d].min);
2066 template<
typename... Args>
2069 return (storage_T *)(this->buf.host) + offset_of(0, args...) *
type().bytes();
2071 return (storage_T *)(this->buf.host) + offset_of(0, args...);
2076 ptrdiff_t offset_of(
const int *pos)
const {
2078 for (
int i = this->
dimensions() - 1; i >= 0; i--) {
2079#if HALIDE_RUNTIME_BUFFER_CHECK_INDICES
2080 assert(pos[i] >= this->buf.dim[i].min);
2081 assert(pos[i] < this->buf.dim[i].min + this->buf.dim[i].extent);
2083 offset += (
ptrdiff_t)this->buf.dim[i].stride * (pos[i] - this->buf.dim[i].min);
2089 storage_T *address_of(
const int *pos)
const {
2091 return (storage_T *)this->buf.host + offset_of(pos) *
type().bytes();
2093 return (storage_T *)this->buf.host + offset_of(pos);
2100 return (T *)(this->buf.
host);
2110 template<
typename... Args,
2111 typename =
typename std::enable_if<
AllInts<Args...>::value>
::type>
2113 static_assert(!T_is_void,
2114 "Cannot use operator() on Buffer<void> types");
2115 constexpr int expected_dims = 1 + (int)(
sizeof...(rest));
2116 static_assert(Dims ==
AnyDims || Dims == expected_dims,
"Buffer with static dimensions was accessed with the wrong number of coordinates in operator()");
2118 return *((
const not_void_T *)(address_of(first, rest...)));
2123 static_assert(!T_is_void,
2124 "Cannot use operator() on Buffer<void> types");
2125 constexpr int expected_dims = 0;
2126 static_assert(Dims ==
AnyDims || Dims == expected_dims,
"Buffer with static dimensions was accessed with the wrong number of coordinates in operator()");
2128 return *((
const not_void_T *)(
data()));
2134 static_assert(!T_is_void,
2135 "Cannot use operator() on Buffer<void> types");
2137 return *((
const not_void_T *)(address_of(pos)));
2140 template<
typename... Args,
2141 typename =
typename std::enable_if<
AllInts<Args...>::value>
::type>
2143 static_assert(!T_is_void,
2144 "Cannot use operator() on Buffer<void> types");
2145 constexpr int expected_dims = 1 + (int)(
sizeof...(rest));
2146 static_assert(Dims ==
AnyDims || Dims == expected_dims,
"Buffer with static dimensions was accessed with the wrong number of coordinates in operator()");
2148 return *((not_void_T *)(address_of(first, rest...)));
2154 static_assert(!T_is_void,
2155 "Cannot use operator() on Buffer<void> types");
2156 constexpr int expected_dims = 0;
2157 static_assert(Dims ==
AnyDims || Dims == expected_dims,
"Buffer with static dimensions was accessed with the wrong number of coordinates in operator()");
2159 return *((not_void_T *)(
data()));
2165 static_assert(!T_is_void,
2166 "Cannot use operator() on Buffer<void> types");
2168 return *((not_void_T *)(address_of(pos)));
2175 for_each_element([&](
const int *pos) {
all_equal &= (*this)(pos) == val; });
2189 struct for_each_value_task_dim {
2191 std::ptrdiff_t
stride[N];
2197 template<
typename Ptr,
typename... Ptrs>
2200 advance_ptrs(
stride + 1, ptrs...);
2204 static void advance_ptrs(
const std::ptrdiff_t *) {
2207 template<
typename Fn,
typename Ptr,
typename... Ptrs>
2208 HALIDE_NEVER_INLINE static void for_each_value_helper(Fn &&f,
int d,
bool innermost_strides_are_one,
2209 const for_each_value_task_dim<
sizeof...(Ptrs) + 1> *t, Ptr ptr, Ptrs... ptrs) {
2211 if (innermost_strides_are_one) {
2212 Ptr
end = ptr + t[0].extent;
2213 while (ptr !=
end) {
2214 f(*ptr++, (*ptrs++)...);
2217 for (std::ptrdiff_t i = t[0].
extent; i != 0; i--) {
2218 f(*ptr, (*ptrs)...);
2219 advance_ptrs(t[0].
stride, ptr, ptrs...);
2223 for (std::ptrdiff_t i = t[d].
extent; i != 0; i--) {
2224 for_each_value_helper(f, d - 1, innermost_strides_are_one, t, ptr, ptrs...);
2225 advance_ptrs(t[d].
stride, ptr, ptrs...);
2232 HALIDE_NEVER_INLINE static std::pair<int, bool> for_each_value_prep(for_each_value_task_dim<N> *t,
2238 for (
int i = 0; i < N; i++) {
2239 if (buffers[i]->device) {
2240 assert(buffers[i]->host &&
2241 "Buffer passed to for_each_value has device allocation but no host allocation. Call allocate() and copy_to_host() first");
2243 "Buffer passed to for_each_value is dirty on device. Call copy_to_host() first");
2245 assert(buffers[i]->host &&
2246 "Buffer passed to for_each_value has no host or device allocation");
2252 for (
int j = 0; j < N; j++) {
2256 const int s = buffers[j]->
dim[i].
stride;
2259 t[i].extent = buffers[0]->
dim[i].
extent;
2264 for (
int j = i; j > 0 && t[j].stride[N - 1] < t[j - 1].stride[N - 1]; j--) {
2265 std::swap(t[j], t[j - 1]);
2272 for (
int i = 1; i < d; i++) {
2274 for (
int j = 0; j < N; j++) {
2275 flat = flat && t[i - 1].stride[j] * t[i - 1].extent == t[i].stride[j];
2278 t[i - 1].extent *= t[i].extent;
2279 for (
int j = i; j < d - 1; j++) {
2290 bool innermost_strides_are_one =
true;
2291 for (
int i = 0; i < N; i++) {
2292 innermost_strides_are_one &= (t[0].stride[i] == 1);
2295 return {d, innermost_strides_are_one};
2298 template<
typename Fn,
typename... Args,
int N =
sizeof...(Args) + 1>
2299 void for_each_value_impl(Fn &&f, Args &&...other_buffers)
const {
2301 const size_t alloc_size =
dimensions() *
sizeof(for_each_value_task_dim<N>);
2310 innermost_strides_are_one,
2312 data(), (other_buffers.data())...);
2319 f(*
data(), (*other_buffers.data())...);
2339 template<
typename Fn,
typename... Args,
int N =
sizeof...(Args) + 1>
2341 for_each_value_impl(f, std::forward<Args>(other_buffers)...);
2345 template<
typename Fn,
typename... Args,
int N =
sizeof...(Args) + 1>
2349 for_each_value_impl(f, std::forward<Args>(other_buffers)...);
2356 struct for_each_element_task_dim {
2363 template<
typename Fn,
2365 typename =
decltype(std::declval<Fn>()(std::declval<Args>()...))>
2366 HALIDE_ALWAYS_INLINE static void for_each_element_variadic(
int,
int,
const for_each_element_task_dim *, Fn &&f, Args... args) {
2372 template<
typename Fn,
2374 HALIDE_ALWAYS_INLINE static void for_each_element_variadic(
double,
int d,
const for_each_element_task_dim *t, Fn &&f, Args... args) {
2375 for (
int i = t[d].
min; i <= t[d].max; i++) {
2376 for_each_element_variadic(0, d - 1, t, std::forward<Fn>(f), i, args...);
2382 template<
typename Fn,
2384 typename =
decltype(std::declval<Fn>()(std::declval<Args>()...))>
2386 return (
int)(
sizeof...(Args));
2392 template<
typename Fn,
2395 static_assert(
sizeof...(args) <= 256,
2396 "Callable passed to for_each_element must accept either a const int *,"
2397 " or up to 256 ints. No such operator found. Expect infinite template recursion.");
2398 return num_args(0, std::forward<Fn>(f), 0, args...);
2408 typename =
typename std::enable_if<(d >= 0)>
::type>
2409 HALIDE_ALWAYS_INLINE static void for_each_element_array_helper(
int,
const for_each_element_task_dim *t, Fn &&f,
int *pos) {
2410 for (pos[d] = t[d].
min; pos[d] <= t[d].max; pos[d]++) {
2411 for_each_element_array_helper<d - 1>(0, t, std::forward<Fn>(f), pos);
2418 typename =
typename std::enable_if<(d < 0)>::type>
2419 HALIDE_ALWAYS_INLINE static void for_each_element_array_helper(
double,
const for_each_element_task_dim *t, Fn &&f,
int *pos) {
2428 template<
typename Fn>
2429 static void for_each_element_array(
int d,
const for_each_element_task_dim *t, Fn &&f,
int *pos) {
2432 }
else if (d == 0) {
2436 for_each_element_array_helper<0, Fn>(0, t, std::forward<Fn>(f), pos);
2437 }
else if (d == 1) {
2438 for_each_element_array_helper<1, Fn>(0, t, std::forward<Fn>(f), pos);
2439 }
else if (d == 2) {
2440 for_each_element_array_helper<2, Fn>(0, t, std::forward<Fn>(f), pos);
2441 }
else if (d == 3) {
2442 for_each_element_array_helper<3, Fn>(0, t, std::forward<Fn>(f), pos);
2444 for (pos[d] = t[d].
min; pos[d] <= t[d].max; pos[d]++) {
2445 for_each_element_array(d - 1, t, std::forward<Fn>(f), pos);
2453 template<
typename Fn,
2454 typename =
decltype(std::declval<Fn>()((
const int *)
nullptr))>
2455 static void for_each_element(
int,
int dims,
const for_each_element_task_dim *t, Fn &&f,
int check = 0) {
2456 const int size = dims *
sizeof(int);
2461 for_each_element_array(dims - 1, t, std::forward<Fn>(f), pos);
2466 template<
typename Fn>
2467 HALIDE_ALWAYS_INLINE static void for_each_element(
double,
int dims,
const for_each_element_task_dim *t, Fn &&f) {
2468 int args = num_args(0, std::forward<Fn>(f));
2469 assert(dims >= args);
2470 for_each_element_variadic(0, args - 1, t, std::forward<Fn>(f));
2473 template<
typename Fn>
2474 void for_each_element_impl(Fn &&f)
const {
2475 for_each_element_task_dim *t =
2481 for_each_element(0,
dimensions(), t, std::forward<Fn>(f));
2542 template<
typename Fn>
2544 for_each_element_impl(f);
2548 template<
typename Fn>
2552 for_each_element_impl(f);
2558 template<
typename Fn>
2563 template<
typename... Args,
2564 typename =
decltype(std::declval<Fn>()(std::declval<Args>()...))>
2565 void operator()(Args... args) {
2566 (*buf)(args...) = f(args...);
2570 : f(std::forward<Fn>(f)), buf(buf) {
2579 template<
typename Fn,
2580 typename =
typename std::enable_if<!std::is_arithmetic<typename std::decay<Fn>::type>::value>
::type>
2583 FillHelper<Fn> wrapper(std::forward<Fn>(f),
this);
2584 return for_each_element(wrapper);
2592 return buf.is_bounds_query();
2601#if defined(__has_feature)
2602#if __has_feature(memory_sanitizer)
2606 for_each_value([](T &v) { __msan_check_mem_is_initialized(&v,
sizeof(T)); ; });
#define HALIDE_RUNTIME_BUFFER_ALLOCATION_ALIGNMENT
This file declares the routines used by Halide internally in its runtime.
struct halide_dimension_t halide_dimension_t
#define HALIDE_NEVER_INLINE
@ halide_error_code_success
There was no error.
#define HALIDE_ALWAYS_INLINE
struct halide_buffer_t halide_buffer_t
The raw representation of an image passed around by generated Halide code.
Dimension(const halide_dimension_t &dim)
halide_type_t type() const
Get the type of the elements.
Read-only access to the shape.
HALIDE_ALWAYS_INLINE int min() const
The lowest coordinate in this dimension.
Dimension(const halide_dimension_t &dim)
HALIDE_ALWAYS_INLINE int max() const
The highest coordinate in this dimension.
HALIDE_ALWAYS_INLINE iterator end() const
An iterator that points to one past the max coordinate.
HALIDE_ALWAYS_INLINE int stride() const
The number of elements in memory you have to step over to increment this coordinate by one.
HALIDE_ALWAYS_INLINE iterator begin() const
An iterator that points to the min coordinate.
HALIDE_ALWAYS_INLINE int extent() const
The extent of the image along this dimension.
A templated Buffer class that wraps halide_buffer_t and adds functionality.
Buffer< T, Dims, InClassDimStorage > & operator=(const Buffer< T2, D2, S2 > &other)
Assign from another Buffer of possibly-different dimensionality and type.
Buffer< not_const_T, Dims, InClassDimStorage > copy_to_planar(void *(*allocate_fn)(size_t)=nullptr, void(*deallocate_fn)(void *)=nullptr) const
Like copy(), but the copy is created in planar memory layout (vs.
Buffer< T, Dims, InClassDimStorage > transposed(const std::vector< int > &order) const
Make a buffer which refers to the same data in the same layout using a different ordering of the dime...
void translate(int d, int delta)
Translate an image in-place along one dimension by changing how it is indexed.
Buffer(const halide_buffer_t &buf, BufferDeviceOwnership ownership=BufferDeviceOwnership::Unmanaged)
Make a Buffer from a halide_buffer_t.
void allocate(void *(*allocate_fn)(size_t)=nullptr, void(*deallocate_fn)(void *)=nullptr)
Allocate memory for this Buffer.
Buffer< not_const_T, Dims, InClassDimStorage > copy(void *(*allocate_fn)(size_t)=nullptr, void(*deallocate_fn)(void *)=nullptr) const
Make a new image which is a deep copy of this image.
Buffer< T,(Dims==AnyDims ? AnyDims :Dims+1)> embedded(int d, int pos=0) const
Make a new buffer that views this buffer as a single slice in a higher-dimensional space.
friend class Buffer
Give Buffers access to the members of Buffers of different dimensionalities and types.
void add_dimension()
Add a new dimension with a min of zero and an extent of one.
void slice(int d)
Slice a buffer in-place at the dimension's minimum.
static void set_default_allocate_fn(void *(*allocate_fn)(size_t))
bool owns_host_memory() const
Does this Buffer own the host memory it refers to?
int width() const
Conventional names for the first three dimensions.
void transpose(const std::vector< int > &order)
A generalized transpose: instead of swapping two dimensions, pass a vector that lists each dimension ...
void set_min(const std::vector< int > &mins)
Set the min coordinate of an image in the first N dimensions.
HALIDE_ALWAYS_INLINE Buffer< T, Dims, InClassDimStorage > & for_each_element(Fn &&f)
Buffer(halide_type_t t, add_const_if_T_is_const< void > *data, const std::vector< int > &sizes)
Initialize an Buffer of runtime type from a pointer and a vector of sizes.
HALIDE_ALWAYS_INLINE Buffer< T2, D2, InClassDimStorage > as() &&
Return an rval reference to this Buffer.
int copy_to_host(void *ctx=nullptr)
Buffer(halide_type_t t, const std::vector< int > &sizes)
Allocate a new image of unknown type using a vector of ints as the size.
int device_malloc(const struct halide_device_interface_t *device_interface, void *ctx=nullptr)
int device_free(void *ctx=nullptr)
bool contains(Args... args) const
void crop(const std::vector< std::pair< int, int > > &rect)
Crop an image in-place along the first N dimensions.
HALIDE_ALWAYS_INLINE const Buffer< typename std::add_const< T >::type, Dims, InClassDimStorage > & as_const() const &
void set_device_dirty(bool v=true)
HALIDE_ALWAYS_INLINE const not_void_T & operator()(const int *pos) const
Buffer(T *data, int d, const halide_dimension_t *shape)
Initialize an Buffer from a pointer to the min coordinate and an array describing the shape.
Buffer(Buffer< T2, D2, S2 > &&other)
Move-construct a Buffer from a Buffer of different dimensionality and type.
void slice(int d, int pos)
Rewrite the buffer to refer to a single lower-dimensional slice of itself along the given dimension a...
HALIDE_ALWAYS_INLINE const not_void_T & operator()(int first, Args... rest) const
Access elements.
HALIDE_ALWAYS_INLINE void set_host_dirty(bool v=true)
Methods for managing any GPU allocation.
void msan_check_mem_is_initialized(bool entire=false) const
Convenient check to verify that all of the interesting bytes in the Buffer are initialized under MSAN...
HALIDE_ALWAYS_INLINE Buffer< typename std::add_const< T >::type, Dims, InClassDimStorage > as_const() &&
Buffer< T, Dims, InClassDimStorage > & operator=(Buffer< T, Dims, InClassDimStorage > &&other) noexcept
Standard move-assignment operator.
int device_detach_native(void *ctx=nullptr)
int device_wrap_native(const struct halide_device_interface_t *device_interface, uint64_t handle, void *ctx=nullptr)
static constexpr bool has_static_halide_type
True if the Halide type is not void (or const void).
Buffer< T, Dims, InClassDimStorage > translated(const std::vector< int > &delta) const
Make an image which refers to the same data translated along the first N dimensions.
HALIDE_ALWAYS_INLINE Dimension dim(int i) const
Access the shape of the buffer.
Buffer(int first, int second, Args... rest)
HALIDE_ALWAYS_INLINE Buffer< typename std::add_const< T >::type, Dims, InClassDimStorage > & as_const() &
as_const() is syntactic sugar for .as<const T>(), to avoid the need to recapitulate the type argument...
Buffer< T, Dims, InClassDimStorage > transposed(int d1, int d2) const
Make a buffer which refers to the same data in the same layout using a swapped indexing order for the...
HALIDE_ALWAYS_INLINE Buffer< T, Dims, InClassDimStorage > & for_each_value(Fn &&f, Args &&...other_buffers)
HALIDE_ALWAYS_INLINE not_void_T & operator()()
BufferDeviceOwnership device_ownership() const
Return the method by which the device field is managed.
void check_overflow()
Check the product of the extents fits in memory.
static bool can_convert_from(const Buffer< T2, D2, S2 > &other)
Determine if a Buffer<T, Dims, InClassDimStorage> can be constructed from some other Buffer type.
Buffer< not_const_T, Dims, InClassDimStorage > copy_to_interleaved(void *(*allocate_fn)(size_t)=nullptr, void(*deallocate_fn)(void *)=nullptr) const
Like copy(), but the copy is created in interleaved memory layout (vs.
int device_and_host_malloc(const struct halide_device_interface_t *device_interface, void *ctx=nullptr)
int device_sync(void *ctx=nullptr)
static Buffer< void, Dims, InClassDimStorage > make_interleaved(halide_type_t t, int width, int height, int channels)
If you use the (x, y, c) indexing convention, then Halide Buffers are stored planar by default.
Buffer(const std::vector< int > &sizes)
Allocate a new image of known type using a vector of ints as the size.
void embed(int d, int pos=0)
Embed a buffer in-place, increasing the dimensionality.
static constexpr halide_type_t static_halide_type()
Get the Halide type of T.
Buffer(T *data, int first, Args &&...rest)
Initialize an Buffer from a pointer and some sizes.
int copy_to_device(const struct halide_device_interface_t *device_interface, void *ctx=nullptr)
Buffer(Array(&vals)[N])
Make an Buffer that refers to a statically sized array.
const halide_buffer_t * raw_buffer() const
HALIDE_ALWAYS_INLINE not_void_T & operator()(int first, Args... rest)
static Buffer< T, Dims, InClassDimStorage > make_interleaved(int width, int height, int channels)
If you use the (x, y, c) indexing convention, then Halide Buffers are stored planar by default.
halide_type_t type() const
Get the type of the elements.
int device_and_host_free(const struct halide_device_interface_t *device_interface, void *ctx=nullptr)
Buffer(int first)
Allocate a new image of the given size.
halide_buffer_t * raw_buffer()
Get a pointer to the raw halide_buffer_t this wraps.
T * end() const
A pointer to one beyond the element with the highest address.
HALIDE_ALWAYS_INLINE bool device_dirty() const
Buffer< T, Dims, InClassDimStorage > cropped(const std::vector< std::pair< int, int > > &rect) const
Make an image that refers to a sub-rectangle of this image along the first N dimensions.
static constexpr int static_dimensions()
Callers should not use the result if has_static_dimensions is false.
void transpose(int d1, int d2)
Transpose a buffer in-place by changing how it is indexed.
void deallocate()
Drop reference to any owned host or device memory, possibly freeing it, if this buffer held the last ...
size_t size_in_bytes() const
The total number of bytes spanned by the data in memory.
bool has_device_allocation() const
void reset()
Reset the Buffer to be equivalent to a default-constructed Buffer of the same static type (if any); B...
Buffer(halide_type_t t, int first, Args... rest)
Allocate a new image of the given size with a runtime type.
int dimensions() const
Get the dimensionality of the buffer.
Buffer(halide_type_t t, add_const_if_T_is_const< void > *data, int d, const halide_dimension_t *shape)
Initialize an Buffer from a pointer to the min coordinate and an array describing the shape.
int min(int i) const
Access to the mins, strides, extents.
HALIDE_ALWAYS_INLINE const Buffer< T, Dims, InClassDimStorage > & for_each_element(Fn &&f) const
Call a function at each site in a buffer.
void device_deallocate()
Drop reference to any owned device memory, possibly freeing it if this buffer held the last reference...
HALIDE_ALWAYS_INLINE const not_void_T & operator()() const
static Buffer< T, Dims, InClassDimStorage > make_scalar()
Make a zero-dimensional Buffer.
void add_dimension_with_stride(int s)
Add a new dimension with a min of zero, an extent of one, and the specified stride.
Buffer(Buffer< T, Dims, InClassDimStorage > &&other) noexcept
Move constructor.
Buffer< T, Dims, InClassDimStorage > cropped(int d, int min, int extent) const
Make an image that refers to a sub-range of this image along the given dimension.
void crop(int d, int min, int extent)
Crop an image in-place along the given dimension.
Buffer< T, Dims, InClassDimStorage > & fill(Fn &&f)
Fill a buffer by evaluating a callable at every site.
static Buffer< T, Dims, InClassDimStorage > make_scalar(T *data)
Make a zero-dimensional Buffer that points to non-owned, existing data.
Buffer< T, Dims, InClassDimStorage > alias() const
Make a copy of the Buffer which shares the underlying host and/or device allocations as the existing ...
void set_min(Args... args)
size_t number_of_elements() const
The total number of elements this buffer represents.
static void assert_can_convert_from(const Buffer< T2, D2, S2 > &other)
Fail an assertion at runtime or compile-time if an Buffer<T, Dims, InClassDimStorage> cannot be const...
void translate(const std::vector< int > &delta)
Translate an image along the first N dimensions by changing how it is indexed.
Buffer(const Buffer< T, Dims, InClassDimStorage > &other)
Copy constructor.
HALIDE_ALWAYS_INLINE not_void_T & operator()(const int *pos)
T * data() const
Get a pointer to the address of the min coordinate.
Buffer< T, Dims, InClassDimStorage > & fill(not_void_T val)
Buffer(const std::vector< int > &sizes, const std::vector< int > &storage_order)
Buffer< T, Dims, InClassDimStorage > & operator=(Buffer< T2, D2, S2 > &&other)
Move from another Buffer of possibly-different dimensionality and type.
Buffer(halide_type_t t, const std::vector< int > &sizes, const std::vector< int > &storage_order)
Allocate a new image of unknown type using a vector of ints as the size and a vector of indices indic...
Buffer(halide_type_t t, add_const_if_T_is_const< void > *data, const std::vector< halide_dimension_t > &shape)
Initialize a Buffer from a pointer to the min coordinate and a vector describing the shape.
static constexpr bool has_static_dimensions
Buffer< T,(Dims==AnyDims ? AnyDims :Dims - 1)> sliced(int d, int pos) const
Make a lower-dimensional buffer that refers to one slice of this buffer.
static Buffer< add_const_if_T_is_const< void >, Dims, InClassDimStorage > make_interleaved(halide_type_t t, T *data, int width, int height, int channels)
Wrap an existing interleaved image.
HALIDE_ALWAYS_INLINE const Buffer< T, Dims, InClassDimStorage > & for_each_value(Fn &&f, Args &&...other_buffers) const
Call a function on every value in the buffer, and the corresponding values in some number of other bu...
bool is_bounds_query() const
Check if an input buffer passed extern stage is a querying bounds.
Buffer< T,(Dims==AnyDims ? AnyDims :Dims - 1)> sliced(int d) const
Make a lower-dimensional buffer that refers to one slice of this buffer at the dimension's minimum.
int left() const
Conventional names for the min and max value of each dimension.
void copy_from(Buffer< T2, D2, S2 > src)
Fill a Buffer with the values at the same coordinates in another Buffer.
Buffer< T, Dims, InClassDimStorage > translated(int d, int dx) const
Make an image which refers to the same data with using translated coordinates in the given dimension.
static Buffer< T, Dims, InClassDimStorage > make_interleaved(T *data, int width, int height, int channels)
Wrap an existing interleaved image.
static void set_default_deallocate_fn(void(*deallocate_fn)(void *))
static Buffer< T, Dims, InClassDimStorage > make_with_shape_of(Buffer< T2, D2, S2 > src, void *(*allocate_fn)(size_t)=nullptr, void(*deallocate_fn)(void *)=nullptr)
Make a buffer with the same shape and memory nesting order as another buffer.
Buffer(const Buffer< T2, D2, S2 > &other)
Construct a Buffer from a Buffer of different dimensionality and type.
bool contains(const std::vector< int > &coords) const
Test if a given coordinate is within the bounds of an image.
Buffer(T *data, const std::vector< halide_dimension_t > &shape)
Initialize a Buffer from a pointer to the min coordinate and a vector describing the shape.
Buffer(T *data, const std::vector< int > &sizes)
Initialize an Buffer from a pointer and a vector of sizes.
Buffer< T, Dims, InClassDimStorage > & operator=(const Buffer< T, Dims, InClassDimStorage > &other)
Standard assignment operator.
T * begin() const
A pointer to the element with the lowest address.
bool all_equal(not_void_T val) const
Tests that all values in this buffer are equal to val.
Buffer(halide_type_t t, add_const_if_T_is_const< void > *data, int first, Args &&...rest)
Initialize an Buffer of runtime type from a pointer and some sizes.
HALIDE_ALWAYS_INLINE Buffer< T2, D2, InClassDimStorage > & as() &
Return a typed reference to this Buffer.
HALIDE_ALWAYS_INLINE const Buffer< T2, D2, InClassDimStorage > & as() const &
Return a const typed reference to this Buffer.
static Buffer< add_const_if_T_is_const< void >, Dims, InClassDimStorage > make_scalar(halide_type_t t)
Make a zero-dimensional Buffer.
bool any_zero(const Container &c)
BufferDeviceOwnership
This indicates how to deallocate the device for a Halide::Runtime::Buffer.
This file defines the class FunctionDAG, which is our representation of a Halide pipeline,...
Expr min(const FuncRef &a, const FuncRef &b)
Explicit overloads of min and max for FuncRef.
Expr max(const FuncRef &a, const FuncRef &b)
unsigned __INT64_TYPE__ uint64_t
__UINTPTR_TYPE__ uintptr_t
ALWAYS_INLINE T align_up(T p, size_t alignment)
unsigned __INT8_TYPE__ uint8_t
__PTRDIFF_TYPE__ ptrdiff_t
unsigned __INT16_TYPE__ uint16_t
void * memcpy(void *s1, const void *s2, size_t n)
void * memset(void *s, int val, size_t n)
unsigned __INT32_TYPE__ uint32_t
An iterator class, so that you can iterate over coordinates in a dimensions using a range-based for l...
An iterator class, so that you can iterate over coordinates in a dimensions using a range-based for l...
bool operator!=(const iterator &other) const
A similar struct for managing device allocations.
BufferDeviceOwnership ownership
static void *(* default_allocate_fn)(size_t)
static void(* default_deallocate_fn)(void *)
The raw representation of an image passed around by generated Halide code.
int32_t dimensions
The dimensionality of the buffer.
halide_dimension_t * dim
The shape of the buffer.
uint64_t device
A device-handle for e.g.
uint8_t * host
A pointer to the start of the data in main memory.
Each GPU API provides a halide_device_interface_t struct pointing to the code that manages device all...
int(* device_and_host_malloc)(void *user_context, struct halide_buffer_t *buf, const struct halide_device_interface_t *device_interface)
int(* wrap_native)(void *user_context, struct halide_buffer_t *buf, uint64_t handle, const struct halide_device_interface_t *device_interface)
int(* copy_to_device)(void *user_context, struct halide_buffer_t *buf, const struct halide_device_interface_t *device_interface)
int(* device_malloc)(void *user_context, struct halide_buffer_t *buf, const struct halide_device_interface_t *device_interface)
A runtime tag for a type in the halide type system.