1#ifndef HALIDE_IR_MATCH_H
2#define HALIDE_IR_MATCH_H
139 typename =
typename std::remove_reference<T>::type::pattern_tag>
146 constexpr static uint32_t mask = std::remove_reference<T>::type::binds;
166 const int lanes = scalar_type.
lanes;
167 scalar_type.
lanes = 1;
170 switch (scalar_type.
code) {
204 template<u
int32_t bound>
232 template<u
int32_t bound>
234 static_assert(i >= 0 && i <
max_wild,
"Wild with out-of-range index");
237 op = ((
const Broadcast *)op)->value.get();
246 state.get_bound_const(i, val, type);
249 state.set_bound_const(i, value, e.type);
253 template<u
int32_t bound>
255 static_assert(i >= 0 && i <
max_wild,
"Wild with out-of-range index");
259 state.get_bound_const(i, val, type);
260 return type == i64_type && value == val.
u.
i64;
262 state.set_bound_const(i, value, i64_type);
298 template<u
int32_t bound>
300 static_assert(i >= 0 && i <
max_wild,
"Wild with out-of-range index");
303 op = ((
const Broadcast *)op)->value.get();
312 state.get_bound_const(i, val, type);
315 state.set_bound_const(i, value, e.type);
331 state.get_bound_const(i, val, ty);
351 template<u
int32_t bound>
353 static_assert(i >= 0 && i <
max_wild,
"Wild with out-of-range index");
356 op = ((
const Broadcast *)op)->value.get();
361 double value = ((
const FloatImm *)op)->value;
365 state.get_bound_const(i, val, type);
368 state.set_bound_const(i, value, e.type);
384 state.get_bound_const(i, val, ty);
405 template<u
int32_t bound>
407 static_assert(i >= 0 && i <
max_wild,
"Wild with out-of-range index");
410 op = ((
const Broadcast *)op)->value.get();
424 template<u
int32_t bound>
426 static_assert(i >= 0 && i <
max_wild,
"Wild with out-of-range index");
442 state.get_bound_const(i, val, ty);
463 template<u
int32_t bound>
466 return equal(*state.get_binding(i), e);
468 state.set_binding(i, e);
503 template<u
int32_t bound>
507 op = ((
const Broadcast *)op)->value.get();
515 return ((
const FloatImm *)op)->value == (double)
v;
521 template<u
int32_t bound>
526 template<u
int32_t bound>
550 val.u.f64 = (double)
v;
566 typename =
typename std::decay<T>::type::pattern_tag>
577 static_assert(!std::is_same<typename std::decay<T>::type,
Expr>::value || std::is_lvalue_reference<T>::value,
578 "Exprs are captured by reference by IRMatcher objects and so must be lvalues");
589 typename =
typename std::decay<T>::type::pattern_tag,
591 typename =
typename std::enable_if<!std::is_same<typename std::decay<T>::type, SpecificExpr>::value>::type>
627template<
typename Op,
typename A,
typename B>
642 A::canonical && B::canonical && (!
commutative(Op::_node_type) || (A::max_node_type >= B::min_node_type));
644 template<u
int32_t bound>
646 if (e.node_type != Op::_node_type) {
649 const Op &op = (
const Op &)e;
654 template<u
int32_t bound,
typename Op2,
typename A2,
typename B2>
656 return (std::is_same<Op, Op2>::value &&
661 constexpr static bool foldable = A::foldable && B::foldable;
666 if (std::is_same<A, IntLiteral>::value) {
667 b.make_folded_const(val_b, ty, state);
668 if ((std::is_same<Op, And>::value && val_b.
u.
u64 == 0) ||
669 (std::is_same<Op, Or>::value && val_b.
u.
u64 == 1)) {
675 a.make_folded_const(val_a, ty, state);
678 a.make_folded_const(val_a, ty, state);
679 if ((std::is_same<Op, And>::value && val_a.
u.
u64 == 0) ||
680 (std::is_same<Op, Or>::value && val_a.
u.
u64 == 1)) {
686 b.make_folded_const(val_b, ty, state);
709 if (std::is_same<A, IntLiteral>::value) {
710 eb =
b.make(state, type_hint);
711 ea =
a.make(state, eb.
type());
713 ea =
a.make(state, type_hint);
714 eb =
b.make(state, ea.
type());
716 return Op::make(std::move(ea), std::move(eb));
730template<
typename Op,
typename A,
typename B>
742 (!
commutative(Op::_node_type) || A::max_node_type >= B::min_node_type) &&
746 template<u
int32_t bound>
748 if (e.node_type != Op::_node_type) {
751 const Op &op = (
const Op &)e;
756 template<u
int32_t bound,
typename Op2,
typename A2,
typename B2>
758 return (std::is_same<Op, Op2>::value &&
763 constexpr static bool foldable = A::foldable && B::foldable;
769 if (std::is_same<A, IntLiteral>::value) {
770 b.make_folded_const(val_b, ty, state);
772 a.make_folded_const(val_a, ty, state);
775 a.make_folded_const(val_a, ty, state);
777 b.make_folded_const(val_b, ty, state);
803 if (std::is_same<A, IntLiteral>::value) {
804 eb =
b.make(state, {});
805 ea =
a.make(state, eb.
type());
807 ea =
a.make(state, {});
808 eb =
b.make(state, ea.
type());
810 return Op::make(std::move(ea), std::move(eb));
814template<
typename A,
typename B>
816 s <<
"(" << op.
a <<
" + " << op.
b <<
")";
820template<
typename A,
typename B>
822 s <<
"(" << op.
a <<
" - " << op.
b <<
")";
826template<
typename A,
typename B>
828 s <<
"(" << op.
a <<
" * " << op.
b <<
")";
832template<
typename A,
typename B>
834 s <<
"(" << op.
a <<
" / " << op.
b <<
")";
838template<
typename A,
typename B>
840 s <<
"(" << op.
a <<
" && " << op.
b <<
")";
844template<
typename A,
typename B>
846 s <<
"(" << op.
a <<
" || " << op.
b <<
")";
850template<
typename A,
typename B>
852 s <<
"min(" << op.
a <<
", " << op.
b <<
")";
856template<
typename A,
typename B>
858 s <<
"max(" << op.
a <<
", " << op.
b <<
")";
862template<
typename A,
typename B>
864 s <<
"(" << op.
a <<
" <= " << op.
b <<
")";
868template<
typename A,
typename B>
870 s <<
"(" << op.
a <<
" < " << op.
b <<
")";
874template<
typename A,
typename B>
876 s <<
"(" << op.
a <<
" >= " << op.
b <<
")";
880template<
typename A,
typename B>
882 s <<
"(" << op.
a <<
" > " << op.
b <<
")";
886template<
typename A,
typename B>
888 s <<
"(" << op.
a <<
" == " << op.
b <<
")";
892template<
typename A,
typename B>
894 s <<
"(" << op.
a <<
" != " << op.
b <<
")";
898template<
typename A,
typename B>
900 s <<
"(" << op.
a <<
" % " << op.
b <<
")";
904template<
typename A,
typename B>
911template<
typename A,
typename B>
921 int dead_bits = 64 - t.bits;
929 return (a + b) & (ones >> (64 - t.bits));
937template<
typename A,
typename B>
944template<
typename A,
typename B>
955 int dead_bits = 64 - t.bits;
962 return (a - b) & (ones >> (64 - t.bits));
970template<
typename A,
typename B>
977template<
typename A,
typename B>
987 int dead_bits = 64 - t.bits;
995 return (a * b) & (ones >> (64 - t.bits));
1003template<
typename A,
typename B>
1010template<
typename A,
typename B>
1030template<
typename A,
typename B>
1037template<
typename A,
typename B>
1059template<
typename A,
typename B>
1068 return std::min(a, b);
1073 return std::min(a, b);
1078 return std::min(a, b);
1081template<
typename A,
typename B>
1090 return std::max(a, b);
1095 return std::max(a, b);
1100 return std::max(a, b);
1103template<
typename A,
typename B>
1108template<
typename A,
typename B>
1128template<
typename A,
typename B>
1133template<
typename A,
typename B>
1153template<
typename A,
typename B>
1158template<
typename A,
typename B>
1178template<
typename A,
typename B>
1183template<
typename A,
typename B>
1203template<
typename A,
typename B>
1208template<
typename A,
typename B>
1228template<
typename A,
typename B>
1233template<
typename A,
typename B>
1253template<
typename A,
typename B>
1258template<
typename A,
typename B>
1279template<
typename A,
typename B>
1284template<
typename A,
typename B>
1309template<
typename... Args>
1318template<
typename... Args>
1323template<Call::IntrinsicOp
intrin>
1356 typename =
typename std::enable_if<(i <
sizeof...(Args))>::type>
1358 using T =
decltype(std::get<i>(
args));
1363 template<
int i, u
int32_t binds>
1368 template<u
int32_t bound>
1380 typename =
typename std::enable_if<(i <
sizeof...(Args))>::type>
1382 s << std::get<i>(
args);
1383 if (i + 1 <
sizeof...(Args)) {
1400 Expr arg0 = std::get<0>(
args).make(state, type_hint);
1402 return likely(std::move(arg0));
1406 return abs(std::move(arg0));
1411 Expr arg1 = std::get<std::min<size_t>(1,
sizeof...(Args) - 1)>(
args).
make(state, type_hint);
1413 return absd(std::move(arg0), std::move(arg1));
1431 return halving_add(std::move(arg0), std::move(arg1));
1433 return halving_sub(std::move(arg0), std::move(arg1));
1437 return std::move(arg0) << std::move(arg1);
1439 return std::move(arg0) >> std::move(arg1);
1446 Expr arg2 = std::get<std::min<size_t>(2,
sizeof...(Args) - 1)>(
args).
make(state, type_hint);
1448 return mul_shift_right(std::move(arg0), std::move(arg1), std::move(arg2));
1453 internal_error <<
"Unhandled intrinsic in IRMatcher: " << intrin;
1465 std::get<0>(
args).make_folded_const(val, ty, state);
1470 std::get<1>(
args).make_folded_const(arg1, signed_ty, state);
1473 if (arg1.
u.
i64 < 0) {
1476 val.u.i64 >>= -arg1.
u.
i64;
1479 val.u.u64 >>= -arg1.
u.
i64;
1482 val.u.u64 <<= arg1.
u.
i64;
1485 if (arg1.
u.
i64 > 0) {
1488 val.u.i64 >>= arg1.
u.
i64;
1491 val.u.u64 >>= arg1.
u.
i64;
1494 val.u.u64 <<= -arg1.
u.
i64;
1497 internal_error <<
"Folding not implemented for intrinsic: " << intrin;
1515template<
typename A,
typename B>
1519template<
typename A,
typename B>
1523template<
typename A,
typename B>
1528template<
typename A,
typename B>
1532template<
typename A,
typename B>
1536template<
typename A,
typename B>
1540template<
typename A,
typename B>
1544template<
typename A,
typename B>
1551 p.optional_type_hint.type = t;
1554template<
typename A,
typename B>
1558template<
typename A,
typename B>
1562template<
typename A,
typename B>
1566template<
typename A,
typename B>
1570template<
typename A,
typename B>
1574template<
typename A,
typename B>
1578template<
typename A,
typename B>
1582template<
typename A,
typename B,
typename C>
1586template<
typename A,
typename B,
typename C>
1596template<
typename A,
typename B>
1622 template<u
int32_t bound>
1627 const Not &op = (
const Not &)e;
1631 template<u
int32_t bound,
typename A2>
1643 template<
typename A1 = A>
1645 a.make_folded_const(val, ty, state);
1646 val.u.u64 = ~val.u.u64;
1665 s <<
"!(" << op.
a <<
")";
1669template<
typename C,
typename T,
typename F>
1681 constexpr static bool canonical = C::canonical && T::canonical && F::canonical;
1683 template<u
int32_t bound>
1693 template<u
int32_t bound,
typename C2,
typename T2,
typename F2>
1702 return Select::make(
c.make(state, {}),
t.make(state, type_hint),
f.make(state, type_hint));
1705 constexpr static bool foldable = C::foldable && T::foldable && F::foldable;
1707 template<
typename C1 = C>
1711 c.make_folded_const(c_val, c_ty, state);
1712 if ((c_val.
u.
u64 & 1) == 1) {
1713 t.make_folded_const(val, ty, state);
1715 f.make_folded_const(val, ty, state);
1721template<
typename C,
typename T,
typename F>
1723 s <<
"select(" << op.
c <<
", " << op.
t <<
", " << op.
f <<
")";
1727template<
typename C,
typename T,
typename F>
1735template<
typename A,
typename B>
1746 constexpr static bool canonical = A::canonical && B::canonical;
1748 template<u
int32_t bound>
1760 template<u
int32_t bound,
typename A2,
typename B2>
1770 lanes.make_folded_const(lanes_val, ty, state);
1772 type_hint.
lanes /= l;
1773 Expr val =
a.make(state, type_hint);
1783 template<
typename A1 = A>
1787 lanes.make_folded_const(lanes_val, lanes_ty, state);
1789 a.make_folded_const(val, ty, state);
1794template<
typename A,
typename B>
1796 s <<
"broadcast(" << op.
a <<
", " << op.
lanes <<
")";
1800template<
typename A,
typename B>
1806template<
typename A,
typename B,
typename C>
1818 constexpr static bool canonical = A::canonical && B::canonical && C::canonical;
1820 template<u
int32_t bound>
1835 template<u
int32_t bound,
typename A2,
typename B2,
typename C2>
1846 lanes.make_folded_const(lanes_val, ty, state);
1848 type_hint.
lanes /= l;
1850 eb =
b.make(state, type_hint);
1851 ea =
a.make(state, eb.
type());
1852 return Ramp::make(std::move(ea), std::move(eb), l);
1858template<
typename A,
typename B,
typename C>
1860 s <<
"ramp(" << op.
a <<
", " << op.
b <<
", " << op.
lanes <<
")";
1864template<
typename A,
typename B,
typename C>
1872template<
typename A,
typename B, VectorReduce::Operator reduce_op>
1884 template<u
int32_t bound>
1888 if (op.
op == reduce_op &&
1897 template<u
int32_t bound,
typename A2,
typename B2, VectorReduce::Operator reduce_op_2>
1899 return (reduce_op == reduce_op_2 &&
1908 lanes.make_folded_const(lanes_val, ty, state);
1909 int l = (int)lanes_val.
u.
i64;
1916template<
typename A,
typename B, VectorReduce::Operator reduce_op>
1918 s <<
"vector_reduce(" << reduce_op <<
", " << op.
a <<
", " << op.
lanes <<
")";
1922template<
typename A,
typename B>
1928template<
typename A,
typename B>
1934template<
typename A,
typename B>
1940template<
typename A,
typename B>
1946template<
typename A,
typename B>
1964 template<u
int32_t bound>
1969 const Sub &op = (
const Sub &)e;
1974 template<u
int32_t bound,
typename A2>
1981 Expr ea =
a.make(state, type_hint);
1983 return Sub::make(std::move(z), std::move(ea));
1988 template<
typename A1 = A>
1990 a.make_folded_const(val, ty, state);
1991 int dead_bits = 64 - ty.bits;
1994 if (ty.bits >= 32 && val.u.u64 && (val.u.u64 << (65 - ty.bits)) == 0) {
2003 val.u.u64 = ((-val.u.u64) << dead_bits) >> dead_bits;
2007 val.u.f64 = -val.u.f64;
2046 template<u
int32_t bound>
2052 return (e.type ==
t &&
2055 template<u
int32_t bound,
typename A2>
2062 return cast(
t,
a.make(state, {}));
2070 s <<
"cast(" << op.
t <<
", " << op.
a <<
")";
2091 template<u
int32_t bound>
2100 template<u
int32_t bound,
typename A2>
2107 Expr e =
a.make(state, {});
2109 return cast(w, std::move(e));
2117 s <<
"widen(" << op.
a <<
")";
2127template<
typename Vec,
typename Base,
typename Str
ide,
typename Lanes>
2135 static constexpr uint32_t binds = Vec::binds | Base::binds | Stride::binds | Lanes::binds;
2139 constexpr static bool canonical = Vec::canonical && Base::canonical && Stride::canonical && Lanes::canonical;
2141 template<u
int32_t bound>
2147 return v.
vectors.size() == 1 &&
2159 base.make_folded_const(base_val, ty, state);
2160 int b = (int)base_val.
u.
i64;
2161 stride.make_folded_const(stride_val, ty, state);
2162 int s = (int)stride_val.
u.
i64;
2163 lanes.make_folded_const(lanes_val, ty, state);
2164 int l = (int)lanes_val.
u.
i64;
2173 static_assert(Base::foldable,
"Base of slice should consist only of operations that constant-fold");
2174 static_assert(Stride::foldable,
"Stride of slice should consist only of operations that constant-fold");
2175 static_assert(Lanes::foldable,
"Lanes of slice should consist only of operations that constant-fold");
2179template<
typename Vec,
typename Base,
typename Str
ide,
typename Lanes>
2181 s <<
"slice(" << op.
vec <<
", " << op.
base <<
", " << op.
stride <<
", " << op.
lanes <<
")";
2185template<
typename Vec,
typename Base,
typename Str
ide,
typename Lanes>
2206 a.make_folded_const(c, ty, state);
2212 if (type_hint.bits) {
2216 c.
u.
f64 = (double)x;
2218 ty.
code = type_hint.code;
2219 ty.
bits = type_hint.bits;
2227 template<
typename A1 = A>
2229 a.make_folded_const(val, ty, state);
2241 s <<
"fold(" << op.
a <<
")";
2260 template<
typename A1 = A>
2262 a.make_folded_const(val, ty, state);
2278 s <<
"overflows(" << op.
a <<
")";
2292 template<u
int32_t bound>
2338 template<
typename A1 = A>
2340 Expr e =
a.make(state, {});
2367 s <<
"is_const(" << op.
a <<
")";
2369 s <<
"is_const(" << op.
a <<
", " << op.
v <<
")";
2374template<
typename A,
typename Prover>
2391 Expr condition =
a.make(state, {});
2392 condition =
prover->mutate(condition,
nullptr);
2400template<
typename A,
typename Prover>
2406template<
typename A,
typename Prover>
2408 s <<
"can_prove(" << op.
a <<
")";
2429 Type t =
a.make(state, {}).type();
2445 s <<
"is_float(" << op.
a <<
")";
2468 Type t =
a.make(state, {}).type();
2484 s <<
"is_int(" << op.
a;
2486 s <<
", " << op.
bits;
2489 s <<
", " << op.
lanes;
2514 Type t =
a.make(state, {}).type();
2530 s <<
"is_uint(" << op.
a;
2532 s <<
", " << op.
bits;
2535 s <<
", " << op.
lanes;
2558 Type t =
a.make(state, {}).type();
2574 s <<
"is_scalar(" << op.
a <<
")";
2595 a.make_folded_const(val, ty, state);
2598 val.
u.
u64 = (val.
u.
u64 == max_bits);
2615 s <<
"is_max_value(" << op.
a <<
")";
2636 a.make_folded_const(val, ty, state);
2639 val.
u.
u64 = (val.
u.
u64 == min_bits);
2658 s <<
"is_min_value(" << op.
a <<
")";
2679 Type t =
a.make(state, {}).type();
2695 s <<
"lanes_of(" << op.
a <<
")";
2700template<
typename Before,
2703 typename =
typename std::enable_if<std::decay<Before>::type::foldable &&
2704 std::decay<After>::type::foldable>::type>
2709 wildcard_type.lanes = output_type.lanes = 1;
2712 static std::set<uint32_t> tested;
2719 debug(0) <<
"validate('" << before <<
"', '" << after <<
"', '" << pred <<
"', " <<
Type(wildcard_type) <<
", " <<
Type(output_type) <<
")\n";
2724 static std::mt19937_64 rng(0);
2729 for (
int trials = 0; trials < 100; trials++) {
2733 int shift = (int)(rng() & (wildcard_type.bits - 1));
2735 for (
int i = 0; i <
max_wild; i++) {
2737 switch (wildcard_type.code) {
2757 double val = ((
int64_t)(rng() & 15) - 8) / 2.0;
2759 val = ((
int64_t)(rng() & 15) - 8) / 2.0;
2773 before.make_folded_const(val_before, type, state);
2775 after.make_folded_const(val_after, type, state);
2776 lanes |= type.
lanes;
2783 switch (output_type.code) {
2795 double error = std::abs(val_before.
u.
f64 - val_after.
u.
f64);
2798 ok &= (error < 0.01 ||
2799 val_before.
u.
u64 == val_after.
u.
u64 ||
2800 std::isnan(val_before.
u.
f64));
2808 debug(0) <<
"Fails with values:\n";
2809 for (
int i = 0; i <
max_wild; i++) {
2814 for (
int i = 0; i <
max_wild; i++) {
2819 debug(0) << val_before.
u.
u64 <<
" " << val_after.
u.
u64 <<
"\n";
2825template<
typename Before,
2828 typename =
typename std::enable_if<!(std::decay<Before>::type::foldable &&
2829 std::decay<After>::type::foldable)>::type>
2840template<
typename Pattern,
2841 typename =
typename enable_if_pattern<Pattern>::type>
2845 p.make_folded_const(c, ty, state);
2853#define HALIDE_DEBUG_MATCHED_RULES 0
2854#define HALIDE_DEBUG_UNMATCHED_RULES 0
2860#define HALIDE_FUZZ_TEST_RULES 0
2862template<
typename Instance>
2875 template<
typename After>
2877#if HALIDE_DEBUG_MATCHED_RULES
2883 template<
typename Before,
2888 static_assert((Before::binds & After::binds) == After::binds,
"Rule result uses unbound values");
2889 static_assert(Before::canonical,
"LHS of rewrite rule should be in canonical form");
2890 static_assert(After::canonical,
"RHS of rewrite rule should be in canonical form");
2891#if HALIDE_FUZZ_TEST_RULES
2896#if HALIDE_DEBUG_MATCHED_RULES
2901#if HALIDE_DEBUG_UNMATCHED_RULES
2902 debug(0) <<
instance <<
" does not match " << before <<
"\n";
2908 template<
typename Before,
2911 static_assert(Before::canonical,
"LHS of rewrite rule should be in canonical form");
2914#if HALIDE_DEBUG_MATCHED_RULES
2919#if HALIDE_DEBUG_UNMATCHED_RULES
2920 debug(0) <<
instance <<
" does not match " << before <<
"\n";
2926 template<
typename Before,
2929 static_assert(Before::canonical,
"LHS of rewrite rule should be in canonical form");
2930#if HALIDE_FUZZ_TEST_RULES
2935#if HALIDE_DEBUG_MATCHED_RULES
2940#if HALIDE_DEBUG_UNMATCHED_RULES
2941 debug(0) <<
instance <<
" does not match " << before <<
"\n";
2947 template<
typename Before,
2954 static_assert(Predicate::foldable,
"Predicates must consist only of operations that can constant-fold");
2955 static_assert((Before::binds & After::binds) == After::binds,
"Rule result uses unbound values");
2956 static_assert((Before::binds & Predicate::binds) == Predicate::binds,
"Rule predicate uses unbound values");
2957 static_assert(Before::canonical,
"LHS of rewrite rule should be in canonical form");
2958 static_assert(After::canonical,
"RHS of rewrite rule should be in canonical form");
2960#if HALIDE_FUZZ_TEST_RULES
2966#if HALIDE_DEBUG_MATCHED_RULES
2967 debug(0) <<
instance <<
" -> " <<
result <<
" via " << before <<
" -> " << after <<
" when " << pred <<
"\n";
2971#if HALIDE_DEBUG_UNMATCHED_RULES
2972 debug(0) <<
instance <<
" does not match " << before <<
"\n";
2978 template<
typename Before,
2983 static_assert(Predicate::foldable,
"Predicates must consist only of operations that can constant-fold");
2984 static_assert(Before::canonical,
"LHS of rewrite rule should be in canonical form");
2989#if HALIDE_DEBUG_MATCHED_RULES
2990 debug(0) <<
instance <<
" -> " <<
result <<
" via " << before <<
" -> " << after <<
" when " << pred <<
"\n";
2994#if HALIDE_DEBUG_UNMATCHED_RULES
2995 debug(0) <<
instance <<
" does not match " << before <<
"\n";
3001 template<
typename Before,
3006 static_assert(Predicate::foldable,
"Predicates must consist only of operations that can constant-fold");
3007 static_assert(Before::canonical,
"LHS of rewrite rule should be in canonical form");
3008#if HALIDE_FUZZ_TEST_RULES
3014#if HALIDE_DEBUG_MATCHED_RULES
3015 debug(0) <<
instance <<
" -> " <<
result <<
" via " << before <<
" -> " << after <<
" when " << pred <<
"\n";
3019#if HALIDE_DEBUG_UNMATCHED_RULES
3020 debug(0) <<
instance <<
" does not match " << before <<
"\n";
3044template<
typename Instance,
3045 typename =
typename enable_if_pattern<Instance>::type>
3047 return {
pattern_arg(instance), output_type, wildcard_type};
3050template<
typename Instance,
3051 typename =
typename enable_if_pattern<Instance>::type>
3053 return {
pattern_arg(instance), output_type, output_type};
#define debug(n)
For optional debugging during codegen, use the debug macro as follows:
@ halide_type_float
IEEE floating point numbers.
@ halide_type_bfloat
floating point numbers in the bfloat format
@ halide_type_int
signed integers
@ halide_type_uint
unsigned integers
#define HALIDE_NEVER_INLINE
#define HALIDE_ALWAYS_INLINE
Subtypes for Halide expressions (Halide::Expr) and statements (Halide::Internal::Stmt)
Methods to test Exprs and Stmts for equality of value.
Defines various operator overloads and utility functions that make it more pleasant to work with Hali...
An alternative template-metaprogramming approach to expression matching.
HALIDE_ALWAYS_INLINE auto rewriter(Instance instance, halide_type_t output_type, halide_type_t wildcard_type) noexcept -> Rewriter< decltype(pattern_arg(instance))>
Construct a rewriter for the given instance, which may be a pattern with concrete expressions as leav...
HALIDE_ALWAYS_INLINE T pattern_arg(T t)
auto rounding_halving_add(A &&a, B &&b) noexcept -> Intrin< Call::rounding_halving_add, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
HALIDE_ALWAYS_INLINE auto or_op(A &&a, B &&b) -> decltype(IRMatcher::operator||(a, b))
HALIDE_ALWAYS_INLINE auto operator!(A &&a) noexcept -> NotOp< decltype(pattern_arg(a))>
auto shift_right(A &&a, B &&b) noexcept -> Intrin< Call::shift_right, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
HALIDE_ALWAYS_INLINE auto min(A &&a, B &&b) noexcept -> BinOp< Min, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
auto widening_add(A &&a, B &&b) noexcept -> Intrin< Call::widening_add, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
HALIDE_ALWAYS_INLINE auto is_int(A &&a, uint8_t bits=0, uint16_t lanes=0) noexcept -> IsInt< decltype(pattern_arg(a))>
HALIDE_ALWAYS_INLINE bool evaluate_predicate(bool x, MatcherState &) noexcept
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< Div >(halide_type_t &t, int64_t a, int64_t b) noexcept
auto abs(A &&a) noexcept -> Intrin< Call::abs, decltype(pattern_arg(a))>
HALIDE_ALWAYS_INLINE auto ne(A &&a, B &&b) -> decltype(IRMatcher::operator!=(a, b))
HALIDE_ALWAYS_INLINE auto is_uint(A &&a, uint8_t bits=0, uint16_t lanes=0) noexcept -> IsUInt< decltype(pattern_arg(a))>
HALIDE_ALWAYS_INLINE auto negate(A &&a) -> decltype(IRMatcher::operator-(a))
uint64_t constant_fold_cmp_op(int64_t, int64_t) noexcept
HALIDE_ALWAYS_INLINE auto operator<=(A &&a, B &&b) noexcept -> CmpOp< LE, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
HALIDE_ALWAYS_INLINE auto operator+(A &&a, B &&b) noexcept -> BinOp< Add, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
HALIDE_ALWAYS_INLINE auto is_max_value(A &&a) noexcept -> IsMaxValue< decltype(pattern_arg(a))>
std::ostream & operator<<(std::ostream &s, const SpecificExpr &e)
HALIDE_ALWAYS_INLINE auto and_op(A &&a, B &&b) -> decltype(IRMatcher::operator&&(a, b))
HALIDE_ALWAYS_INLINE auto h_and(A &&a, B lanes) noexcept -> VectorReduceOp< decltype(pattern_arg(a)), decltype(pattern_arg(lanes)), VectorReduce::And >
HALIDE_ALWAYS_INLINE auto gt(A &&a, B &&b) -> decltype(IRMatcher::operator>(a, b))
HALIDE_ALWAYS_INLINE auto is_const(A &&a) noexcept -> IsConst< decltype(pattern_arg(a))>
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op< LE >(int64_t a, int64_t b) noexcept
HALIDE_ALWAYS_INLINE auto operator*(A &&a, B &&b) noexcept -> BinOp< Mul, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
HALIDE_ALWAYS_INLINE auto add(A &&a, B &&b) -> decltype(IRMatcher::operator+(a, b))
auto widen_right_add(A &&a, B &&b) noexcept -> Intrin< Call::widen_right_add, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
HALIDE_ALWAYS_INLINE auto div(A &&a, B &&b) -> decltype(IRMatcher::operator/(a, b))
auto widen_right_mul(A &&a, B &&b) noexcept -> Intrin< Call::widen_right_mul, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
HALIDE_ALWAYS_INLINE auto mul(A &&a, B &&b) -> decltype(IRMatcher::operator*(a, b))
HALIDE_ALWAYS_INLINE auto max(A &&a, B &&b) noexcept -> BinOp< Max, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
auto absd(A &&a, B &&b) noexcept -> Intrin< Call::absd, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
HALIDE_ALWAYS_INLINE auto slice(Vec vec, Base base, Stride stride, Lanes lanes) noexcept -> SliceOp< decltype(pattern_arg(vec)), decltype(pattern_arg(base)), decltype(pattern_arg(stride)), decltype(pattern_arg(lanes))>
HALIDE_ALWAYS_INLINE auto ramp(A &&a, B &&b, C &&c) noexcept -> RampOp< decltype(pattern_arg(a)), decltype(pattern_arg(b)), decltype(pattern_arg(c))>
HALIDE_ALWAYS_INLINE auto operator/(A &&a, B &&b) noexcept -> BinOp< Div, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
HALIDE_ALWAYS_INLINE auto widen(A &&a) noexcept -> WidenOp< decltype(pattern_arg(a))>
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< Mod >(halide_type_t &t, int64_t a, int64_t b) noexcept
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< And >(halide_type_t &t, int64_t a, int64_t b) noexcept
HALIDE_ALWAYS_INLINE int64_t unwrap(IntLiteral t)
auto widening_mul(A &&a, B &&b) noexcept -> Intrin< Call::widening_mul, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
HALIDE_ALWAYS_INLINE auto operator>(A &&a, B &&b) noexcept -> CmpOp< GT, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
HALIDE_ALWAYS_INLINE auto cast(halide_type_t t, A &&a) noexcept -> CastOp< decltype(pattern_arg(a))>
HALIDE_ALWAYS_INLINE auto overflows(A &&a) noexcept -> Overflows< decltype(pattern_arg(a))>
auto saturating_cast(const Type &t, A &&a) noexcept -> Intrin< Call::saturating_cast, decltype(pattern_arg(a))>
HALIDE_ALWAYS_INLINE void assert_is_lvalue_if_expr()
HALIDE_ALWAYS_INLINE auto operator%(A &&a, B &&b) noexcept -> BinOp< Mod, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< Sub >(halide_type_t &t, int64_t a, int64_t b) noexcept
auto rounding_shift_left(A &&a, B &&b) noexcept -> Intrin< Call::rounding_shift_left, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
HALIDE_ALWAYS_INLINE auto is_scalar(A &&a) noexcept -> IsScalar< decltype(pattern_arg(a))>
HALIDE_ALWAYS_INLINE auto fold(A &&a) noexcept -> Fold< decltype(pattern_arg(a))>
HALIDE_ALWAYS_INLINE auto not_op(A &&a) -> decltype(IRMatcher::operator!(a))
auto likely(A &&a) noexcept -> Intrin< Call::likely, decltype(pattern_arg(a))>
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< Max >(halide_type_t &t, int64_t a, int64_t b) noexcept
constexpr bool and_reduce()
HALIDE_ALWAYS_INLINE auto operator||(A &&a, B &&b) noexcept -> BinOp< Or, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
HALIDE_ALWAYS_INLINE auto operator!=(A &&a, B &&b) noexcept -> CmpOp< NE, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
auto halving_add(A &&a, B &&b) noexcept -> Intrin< Call::halving_add, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
auto mul_shift_right(A &&a, B &&b, C &&c) noexcept -> Intrin< Call::mul_shift_right, decltype(pattern_arg(a)), decltype(pattern_arg(b)), decltype(pattern_arg(c))>
HALIDE_ALWAYS_INLINE auto is_float(A &&a) noexcept -> IsFloat< decltype(pattern_arg(a))>
auto widening_sub(A &&a, B &&b) noexcept -> Intrin< Call::widening_sub, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
HALIDE_ALWAYS_INLINE auto operator>=(A &&a, B &&b) noexcept -> CmpOp< GE, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
HALIDE_ALWAYS_INLINE auto operator<(A &&a, B &&b) noexcept -> CmpOp< LT, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
HALIDE_ALWAYS_INLINE auto operator&&(A &&a, B &&b) noexcept -> BinOp< And, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
HALIDE_ALWAYS_INLINE auto h_or(A &&a, B lanes) noexcept -> VectorReduceOp< decltype(pattern_arg(a)), decltype(pattern_arg(lanes)), VectorReduce::Or >
constexpr bool commutative(IRNodeType t)
HALIDE_ALWAYS_INLINE auto sub(A &&a, B &&b) -> decltype(IRMatcher::operator-(a, b))
auto likely_if_innermost(A &&a) noexcept -> Intrin< Call::likely_if_innermost, decltype(pattern_arg(a))>
HALIDE_ALWAYS_INLINE auto h_max(A &&a, B lanes) noexcept -> VectorReduceOp< decltype(pattern_arg(a)), decltype(pattern_arg(lanes)), VectorReduce::Max >
HALIDE_ALWAYS_INLINE auto broadcast(A &&a, B lanes) noexcept -> BroadcastOp< decltype(pattern_arg(a)), decltype(pattern_arg(lanes))>
HALIDE_ALWAYS_INLINE auto select(C &&c, T &&t, F &&f) noexcept -> SelectOp< decltype(pattern_arg(c)), decltype(pattern_arg(t)), decltype(pattern_arg(f))>
auto saturating_sub(A &&a, B &&b) noexcept -> Intrin< Call::saturating_sub, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
HALIDE_ALWAYS_INLINE auto is_min_value(A &&a) noexcept -> IsMinValue< decltype(pattern_arg(a))>
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< Min >(halide_type_t &t, int64_t a, int64_t b) noexcept
HALIDE_NEVER_INLINE void fuzz_test_rule(Before &&before, After &&after, Predicate &&pred, halide_type_t wildcard_type, halide_type_t output_type) noexcept
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op< GT >(int64_t a, int64_t b) noexcept
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< Mul >(halide_type_t &t, int64_t a, int64_t b) noexcept
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op< GE >(int64_t a, int64_t b) noexcept
HALIDE_ALWAYS_INLINE auto operator-(A &&a, B &&b) noexcept -> BinOp< Sub, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
auto rounding_mul_shift_right(A &&a, B &&b, C &&c) noexcept -> Intrin< Call::rounding_mul_shift_right, decltype(pattern_arg(a)), decltype(pattern_arg(b)), decltype(pattern_arg(c))>
auto saturating_add(A &&a, B &&b) noexcept -> Intrin< Call::saturating_add, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
HALIDE_ALWAYS_INLINE auto le(A &&a, B &&b) -> decltype(IRMatcher::operator<=(a, b))
HALIDE_ALWAYS_INLINE auto lt(A &&a, B &&b) -> decltype(IRMatcher::operator<(a, b))
auto shift_left(A &&a, B &&b) noexcept -> Intrin< Call::shift_left, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
HALIDE_ALWAYS_INLINE auto lanes_of(A &&a) noexcept -> LanesOf< decltype(pattern_arg(a))>
auto rounding_shift_right(A &&a, B &&b) noexcept -> Intrin< Call::rounding_shift_right, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op< LT >(int64_t a, int64_t b) noexcept
HALIDE_ALWAYS_INLINE auto h_min(A &&a, B lanes) noexcept -> VectorReduceOp< decltype(pattern_arg(a)), decltype(pattern_arg(lanes)), VectorReduce::Min >
HALIDE_ALWAYS_INLINE auto h_add(A &&a, B lanes) noexcept -> VectorReduceOp< decltype(pattern_arg(a)), decltype(pattern_arg(lanes)), VectorReduce::Add >
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< Or >(halide_type_t &t, int64_t a, int64_t b) noexcept
HALIDE_ALWAYS_INLINE Expr make_const_expr(halide_scalar_value_t val, halide_type_t ty)
constexpr uint32_t bitwise_or_reduce()
int64_t constant_fold_bin_op(halide_type_t &, int64_t, int64_t) noexcept
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op< EQ >(int64_t a, int64_t b) noexcept
HALIDE_NEVER_INLINE Expr make_const_special_expr(halide_type_t ty)
auto halving_sub(A &&a, B &&b) noexcept -> Intrin< Call::halving_sub, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
HALIDE_ALWAYS_INLINE auto ge(A &&a, B &&b) -> decltype(IRMatcher::operator>=(a, b))
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op< NE >(int64_t a, int64_t b) noexcept
HALIDE_ALWAYS_INLINE auto mod(A &&a, B &&b) -> decltype(IRMatcher::operator%(a, b))
HALIDE_ALWAYS_INLINE auto operator==(A &&a, B &&b) noexcept -> CmpOp< EQ, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
auto widen_right_sub(A &&a, B &&b) noexcept -> Intrin< Call::widen_right_sub, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< Add >(halide_type_t &t, int64_t a, int64_t b) noexcept
HALIDE_ALWAYS_INLINE auto can_prove(A &&a, Prover *p) noexcept -> CanProve< decltype(pattern_arg(a)), Prover >
HALIDE_ALWAYS_INLINE auto eq(A &&a, B &&b) -> decltype(IRMatcher::operator==(a, b))
bool is_const_zero(const Expr &e)
Is the expression a const (as defined by is_const), and also equal to zero (in all lanes,...
Expr make_zero(Type t)
Construct the representation of zero in the given type.
bool is_const_one(const Expr &e)
Is the expression a const (as defined by is_const), and also equal to one (in all lanes,...
bool equal(const RDom &bounds0, const RDom &bounds1)
Return true if bounds0 and bounds1 represent the same bounds.
constexpr IRNodeType StrongestExprNodeType
Expr make_const(Type t, int64_t val)
Construct an immediate of the given type from any numeric C++ type.
T mod_imp(T a, T b)
Implementations of division and mod that are specific to Halide.
bool sub_would_overflow(int bits, int64_t a, int64_t b)
bool add_would_overflow(int bits, int64_t a, int64_t b)
Routines to test if math would overflow for signed integers with the given number of bits.
DstType reinterpret_bits(const SrcType &src)
An aggressive form of reinterpret cast used for correct type-punning.
bool mul_would_overflow(int bits, int64_t a, int64_t b)
Expr with_lanes(const Expr &x, int lanes)
Rewrite the expression x to have lanes lanes.
bool expr_match(const Expr &pattern, const Expr &expr, std::vector< Expr > &result)
Does the first expression have the same structure as the second?
Expr make_signed_integer_overflow(Type type)
Construct a unique signed_integer_overflow Expr.
IRNodeType
All our IR node types get unique IDs for the purposes of RTTI.
bool is_const(const Expr &e)
Is the expression either an IntImm, a FloatImm, a StringImm, or a Cast of the same,...
This file defines the class FunctionDAG, which is our representation of a Halide pipeline,...
@ Predicate
Guard the loads and stores in the loop with an if statement that prevents evaluation beyond the origi...
unsigned __INT64_TYPE__ uint64_t
signed __INT64_TYPE__ int64_t
signed __INT32_TYPE__ int32_t
unsigned __INT8_TYPE__ uint8_t
unsigned __INT16_TYPE__ uint16_t
unsigned __INT32_TYPE__ uint32_t
A fragment of Halide syntax.
HALIDE_ALWAYS_INLINE Type type() const
Get the type of this expression node.
HALIDE_ALWAYS_INLINE const Internal::BaseExprNode * get() const
Override get() to return a BaseExprNode * instead of an IRNode *.
The sum of two expressions.
Logical and - are both expressions true.
A base class for expression nodes.
A vector with 'lanes' elements, in which every element is 'value'.
static Expr make(Expr value, int lanes)
static const IRNodeType _node_type
@ signed_integer_overflow
@ rounding_mul_shift_right
bool is_intrinsic() const
static const IRNodeType _node_type
The actual IR nodes begin here.
static const IRNodeType _node_type
The ratio of two expressions.
Is the first expression equal to the second.
Floating point constants.
static const FloatImm * make(Type t, double value)
Is the first expression greater than or equal to the second.
Is the first expression greater than the second.
static constexpr bool canonical
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
static constexpr uint32_t binds
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
static constexpr bool foldable
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const noexcept
HALIDE_ALWAYS_INLINE bool match(const BinOp< Op2, A2, B2 > &op, MatcherState &state) const noexcept
static constexpr IRNodeType max_node_type
static constexpr IRNodeType min_node_type
static constexpr IRNodeType min_node_type
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
HALIDE_ALWAYS_INLINE bool match(const BroadcastOp< A2, B2 > &op, MatcherState &state) const noexcept
static constexpr bool foldable
static constexpr uint32_t binds
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
static constexpr bool canonical
static constexpr IRNodeType max_node_type
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
HALIDE_NEVER_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const
static constexpr uint32_t binds
static constexpr IRNodeType min_node_type
static constexpr IRNodeType max_node_type
static constexpr bool foldable
static constexpr bool canonical
static constexpr IRNodeType max_node_type
static constexpr bool foldable
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
static constexpr IRNodeType min_node_type
static constexpr uint32_t binds
static constexpr bool canonical
HALIDE_ALWAYS_INLINE bool match(const CastOp< A2 > &op, MatcherState &state) const noexcept
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
static constexpr IRNodeType max_node_type
static constexpr uint32_t binds
static constexpr bool canonical
static constexpr bool foldable
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
static constexpr IRNodeType min_node_type
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
HALIDE_ALWAYS_INLINE bool match(const CmpOp< Op2, A2, B2 > &op, MatcherState &state) const noexcept
static constexpr IRNodeType max_node_type
static constexpr uint32_t binds
static constexpr IRNodeType min_node_type
static constexpr bool canonical
static constexpr bool foldable
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const noexcept
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
static constexpr IRNodeType max_node_type
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
HALIDE_ALWAYS_INLINE IntLiteral(int64_t v)
HALIDE_ALWAYS_INLINE bool match(const IntLiteral &b, MatcherState &state) const noexcept
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
static constexpr IRNodeType min_node_type
static constexpr bool canonical
static constexpr bool foldable
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
HALIDE_ALWAYS_INLINE bool match(int64_t val, MatcherState &state) const noexcept
static constexpr uint32_t binds
HALIDE_ALWAYS_INLINE Intrin(Args... args) noexcept
HALIDE_ALWAYS_INLINE void print_args(std::ostream &s) const
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
static constexpr bool foldable
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
std::tuple< Args... > args
static constexpr uint32_t binds
HALIDE_ALWAYS_INLINE void print_args(double, std::ostream &s) const
HALIDE_ALWAYS_INLINE bool match_args(int, const Call &c, MatcherState &state) const noexcept
static constexpr bool canonical
HALIDE_ALWAYS_INLINE bool match_args(double, const Call &c, MatcherState &state) const noexcept
HALIDE_ALWAYS_INLINE void print_args(int, std::ostream &s) const
static constexpr IRNodeType max_node_type
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
static constexpr IRNodeType min_node_type
OptionalIntrinType< intrin > optional_type_hint
static constexpr IRNodeType min_node_type
static constexpr bool canonical
static constexpr bool foldable
static constexpr IRNodeType max_node_type
static constexpr uint32_t binds
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
static constexpr bool foldable
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const
static constexpr bool canonical
static constexpr IRNodeType min_node_type
static constexpr IRNodeType max_node_type
static constexpr uint32_t binds
static constexpr IRNodeType max_node_type
static constexpr bool foldable
static constexpr uint32_t binds
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const
static constexpr IRNodeType min_node_type
static constexpr bool canonical
static constexpr bool foldable
static constexpr IRNodeType min_node_type
static constexpr bool canonical
static constexpr IRNodeType max_node_type
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const
static constexpr uint32_t binds
static constexpr IRNodeType min_node_type
static constexpr bool canonical
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const
static constexpr bool foldable
static constexpr IRNodeType max_node_type
static constexpr uint32_t binds
static constexpr IRNodeType max_node_type
static constexpr uint32_t binds
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const
static constexpr IRNodeType min_node_type
static constexpr bool foldable
static constexpr bool canonical
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const
static constexpr bool foldable
static constexpr IRNodeType min_node_type
static constexpr bool canonical
static constexpr uint32_t binds
static constexpr IRNodeType max_node_type
static constexpr IRNodeType max_node_type
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const
static constexpr IRNodeType min_node_type
static constexpr bool foldable
static constexpr uint32_t binds
static constexpr bool canonical
To save stack space, the matcher objects are largely stateless and immutable.
HALIDE_ALWAYS_INLINE void get_bound_const(int i, halide_scalar_value_t &val, halide_type_t &type) const noexcept
HALIDE_ALWAYS_INLINE void set_bound_const(int i, int64_t s, halide_type_t t) noexcept
HALIDE_ALWAYS_INLINE void set_bound_const(int i, double f, halide_type_t t) noexcept
static constexpr uint16_t special_values_mask
HALIDE_ALWAYS_INLINE void set_bound_const(int i, halide_scalar_value_t val, halide_type_t t) noexcept
halide_type_t bound_const_type[max_wild]
HALIDE_ALWAYS_INLINE void set_binding(int i, const BaseExprNode &n) noexcept
HALIDE_ALWAYS_INLINE MatcherState() noexcept
HALIDE_ALWAYS_INLINE const BaseExprNode * get_binding(int i) const noexcept
halide_scalar_value_t bound_const[max_wild]
const BaseExprNode * bindings[max_wild]
HALIDE_ALWAYS_INLINE void set_bound_const(int i, uint64_t u, halide_type_t t) noexcept
static constexpr uint16_t signed_integer_overflow
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
HALIDE_ALWAYS_INLINE bool match(NegateOp< A2 > &&p, MatcherState &state) const noexcept
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
static constexpr uint32_t binds
static constexpr bool canonical
static constexpr bool foldable
static constexpr IRNodeType max_node_type
static constexpr IRNodeType min_node_type
static constexpr uint32_t binds
static constexpr bool foldable
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
static constexpr IRNodeType max_node_type
static constexpr bool canonical
HALIDE_ALWAYS_INLINE bool match(const NotOp< A2 > &op, MatcherState &state) const noexcept
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
static constexpr IRNodeType min_node_type
bool check(const Type &t) const
bool check(const Type &) const
static constexpr uint32_t binds
static constexpr IRNodeType max_node_type
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
static constexpr bool canonical
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
static constexpr bool foldable
static constexpr IRNodeType min_node_type
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
static constexpr IRNodeType min_node_type
static constexpr uint32_t binds
static constexpr bool canonical
static constexpr IRNodeType max_node_type
static constexpr bool foldable
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
static constexpr bool canonical
static constexpr IRNodeType max_node_type
static constexpr IRNodeType min_node_type
static constexpr uint32_t binds
HALIDE_ALWAYS_INLINE bool match(const RampOp< A2, B2, C2 > &op, MatcherState &state) const noexcept
static constexpr bool foldable
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
HALIDE_NEVER_INLINE void build_replacement(After after)
HALIDE_ALWAYS_INLINE bool operator()(Before before, After after, Predicate pred)
HALIDE_ALWAYS_INLINE bool operator()(Before before, int64_t after) noexcept
HALIDE_ALWAYS_INLINE Rewriter(Instance instance, halide_type_t ot, halide_type_t wt)
HALIDE_ALWAYS_INLINE bool operator()(Before before, const Expr &after, Predicate pred)
HALIDE_ALWAYS_INLINE bool operator()(Before before, const Expr &after) noexcept
HALIDE_ALWAYS_INLINE bool operator()(Before before, int64_t after, Predicate pred)
HALIDE_ALWAYS_INLINE bool operator()(Before before, After after)
halide_type_t wildcard_type
halide_type_t output_type
static constexpr uint32_t binds
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
static constexpr bool foldable
static constexpr bool canonical
HALIDE_ALWAYS_INLINE bool match(const SelectOp< C2, T2, F2 > &instance, MatcherState &state) const noexcept
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
static constexpr IRNodeType max_node_type
static constexpr IRNodeType min_node_type
static constexpr bool canonical
static constexpr IRNodeType max_node_type
static constexpr bool foldable
HALIDE_ALWAYS_INLINE SliceOp(Vec v, Base b, Stride s, Lanes l)
static constexpr IRNodeType min_node_type
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
static constexpr uint32_t binds
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
static constexpr IRNodeType min_node_type
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
static constexpr bool canonical
static constexpr IRNodeType max_node_type
const BaseExprNode & expr
static constexpr uint32_t binds
static constexpr bool foldable
static constexpr bool canonical
HALIDE_ALWAYS_INLINE bool match(const VectorReduceOp< A2, B2, reduce_op_2 > &op, MatcherState &state) const noexcept
static constexpr uint32_t binds
static constexpr IRNodeType min_node_type
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
static constexpr IRNodeType max_node_type
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
static constexpr bool foldable
static constexpr uint32_t binds
static constexpr bool canonical
static constexpr bool foldable
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
static constexpr IRNodeType max_node_type
HALIDE_ALWAYS_INLINE bool match(const WidenOp< A2 > &op, MatcherState &state) const noexcept
static constexpr IRNodeType min_node_type
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
static constexpr bool canonical
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
static constexpr IRNodeType max_node_type
static constexpr IRNodeType min_node_type
static constexpr uint32_t binds
static constexpr bool foldable
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
static constexpr bool canonical
static constexpr IRNodeType max_node_type
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
static constexpr uint32_t binds
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
HALIDE_ALWAYS_INLINE bool match(int64_t e, MatcherState &state) const noexcept
static constexpr IRNodeType min_node_type
static constexpr bool foldable
static constexpr bool canonical
static constexpr bool foldable
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
static constexpr uint32_t binds
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
HALIDE_ALWAYS_INLINE bool match(int64_t value, MatcherState &state) const noexcept
static constexpr IRNodeType min_node_type
static constexpr IRNodeType max_node_type
static constexpr uint32_t binds
static constexpr bool foldable
static constexpr IRNodeType max_node_type
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
static constexpr IRNodeType min_node_type
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
static constexpr bool canonical
static constexpr IRNodeType min_node_type
static constexpr uint32_t binds
static constexpr IRNodeType max_node_type
static constexpr bool canonical
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
static constexpr bool foldable
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
static constexpr uint32_t mask
IRNodeType node_type
Each IR node subclass has a unique identifier.
static const IntImm * make(Type t, int64_t value)
Is the first expression less than or equal to the second.
Is the first expression less than the second.
The greater of two values.
The lesser of two values.
The product of two expressions.
Is the first expression not equal to the second.
Logical not - true if the expression false.
Logical or - is at least one of the expression true.
A linear ramp vector node.
static const IRNodeType _node_type
static Expr make(Expr base, Expr stride, int lanes)
static Expr make(Expr condition, Expr true_value, Expr false_value)
static const IRNodeType _node_type
Construct a new vector by taking elements from another sequence of vectors.
static Expr make_slice(Expr vector, int begin, int stride, int size)
Convenience constructor for making a shuffle representing a contiguous subset of a vector.
std::vector< Expr > vectors
bool is_slice() const
Check if this shuffle is a contiguous strict subset of the vector arguments, and if so,...
int slice_stride() const
Check if this shuffle is a contiguous strict subset of the vector arguments, and if so,...
int slice_begin() const
Check if this shuffle is a contiguous strict subset of the vector arguments, and if so,...
The difference of two expressions.
static const IRNodeType _node_type
static Expr make(Expr a, Expr b)
Unsigned integer constants.
static const UIntImm * make(Type t, uint64_t value)
Horizontally reduce a vector to a scalar or narrower vector using the given commutative and associati...
static const IRNodeType _node_type
static Expr make(Operator op, Expr vec, int lanes)
Type widen() const
Return Type with the same type code and number of lanes, but with at least twice as many bits.
HALIDE_ALWAYS_INLINE bool is_int() const
Is this type a signed integer type?
HALIDE_ALWAYS_INLINE int lanes() const
Return the number of vector elements in this type.
HALIDE_ALWAYS_INLINE bool is_uint() const
Is this type an unsigned integer type?
HALIDE_ALWAYS_INLINE int bits() const
Return the bit size of a single element of this type.
HALIDE_ALWAYS_INLINE bool is_scalar() const
Is this type a scalar type?
HALIDE_ALWAYS_INLINE bool is_float() const
Is this type a floating point type (float or double).
halide_scalar_value_t is a simple union able to represent all the well-known scalar values in a filte...
union halide_scalar_value_t::@314322015121151262135054202130057122113055355347 u
A runtime tag for a type in the halide type system.
uint8_t bits
The number of bits of precision of a single scalar value of this type.
uint16_t lanes
How many elements in a vector.
uint8_t code
The basic type code: signed integer, unsigned integer, or floating point.