Halide
IRMatch.h
Go to the documentation of this file.
1 #ifndef HALIDE_IR_MATCH_H
2 #define HALIDE_IR_MATCH_H
3 
4 /** \file
5  * Defines a method to match a fragment of IR against a pattern containing wildcards
6  */
7 
8 #include <map>
9 #include <random>
10 #include <set>
11 #include <vector>
12 
13 #include "IR.h"
14 #include "IREquality.h"
15 #include "IROperator.h"
16 
17 namespace Halide {
18 namespace Internal {
19 
20 /** Does the first expression have the same structure as the second?
21  * Variables in the first expression with the name * are interpreted
22  * as wildcards, and their matching equivalent in the second
23  * expression is placed in the vector give as the third argument.
24  * Wildcards require the types to match. For the type bits and width,
25  * a 0 indicates "match anything". So an Int(8, 0) will match 8-bit
26  * integer vectors of any width (including scalars), and a UInt(0, 0)
27  * will match any unsigned integer type.
28  *
29  * For example:
30  \code
31  Expr x = Variable::make(Int(32), "*");
32  match(x + x, 3 + (2*k), result)
33  \endcode
34  * should return true, and set result[0] to 3 and
35  * result[1] to 2*k.
36  */
37 bool expr_match(const Expr &pattern, const Expr &expr, std::vector<Expr> &result);
38 
39 /** Does the first expression have the same structure as the second?
40  * Variables are matched consistently. The first time a variable is
41  * matched, it assumes the value of the matching part of the second
42  * expression. Subsequent matches must be equal to the first match.
43  *
44  * For example:
45  \code
46  Var x("x"), y("y");
47  match(x*(x + y), a*(a + b), result)
48  \endcode
49  * should return true, and set result["x"] = a, and result["y"] = b.
50  */
51 bool expr_match(const Expr &pattern, const Expr &expr, std::map<std::string, Expr> &result);
52 
53 void expr_match_test();
54 
55 /** An alternative template-metaprogramming approach to expression
56  * matching. Potentially more efficient. We lift the expression
57  * pattern into a type, and then use force-inlined functions to
58  * generate efficient matching and reconstruction code for any
59  * pattern. Pattern elements are either one of the classes in the
60  * namespace IRMatcher, or are non-null Exprs (represented as
61  * BaseExprNode &).
62  *
63  * Pattern elements that are fully specified by their pattern can be
64  * built into an expression using the make method. Some patterns,
65  * such as a broadcast that matches any number of lanes, don't have
66  * enough information to recreate an Expr.
67  */
68 namespace IRMatcher {
69 
70 constexpr int max_wild = 6;
71 
72 /** To save stack space, the matcher objects are largely stateless and
73  * immutable. This state object is built up during matching and then
74  * consumed when constructing a replacement Expr.
75  */
76 struct MatcherState {
79 
80  // values of the lanes field with special meaning.
81  static constexpr uint16_t signed_integer_overflow = 0x8000;
82  static constexpr uint16_t special_values_mask = 0x8000; // currently only one
83 
85 
87  void set_binding(int i, const BaseExprNode &n) noexcept {
88  bindings[i] = &n;
89  }
90 
92  const BaseExprNode *get_binding(int i) const noexcept {
93  return bindings[i];
94  }
95 
97  void set_bound_const(int i, int64_t s, halide_type_t t) noexcept {
98  bound_const[i].u.i64 = s;
99  bound_const_type[i] = t;
100  }
101 
103  void set_bound_const(int i, uint64_t u, halide_type_t t) noexcept {
104  bound_const[i].u.u64 = u;
105  bound_const_type[i] = t;
106  }
107 
109  void set_bound_const(int i, double f, halide_type_t t) noexcept {
110  bound_const[i].u.f64 = f;
111  bound_const_type[i] = t;
112  }
113 
115  void set_bound_const(int i, halide_scalar_value_t val, halide_type_t t) noexcept {
116  bound_const[i] = val;
117  bound_const_type[i] = t;
118  }
119 
121  void get_bound_const(int i, halide_scalar_value_t &val, halide_type_t &type) const noexcept {
122  val = bound_const[i];
123  type = bound_const_type[i];
124  }
125 
127  MatcherState() noexcept {
128  }
129 };
130 
131 template<typename T,
132  typename = typename std::remove_reference<T>::type::pattern_tag>
134  struct type {};
135 };
136 
137 template<typename T>
138 struct bindings {
139  constexpr static uint32_t mask = std::remove_reference<T>::type::binds;
140 };
141 
144  ty.lanes &= ~MatcherState::special_values_mask;
146  return make_signed_integer_overflow(ty);
147  }
148  // unreachable
149  return Expr();
150 }
151 
154  halide_type_t scalar_type = ty;
155  if (scalar_type.lanes & MatcherState::special_values_mask) {
156  return make_const_special_expr(scalar_type);
157  }
158 
159  const int lanes = scalar_type.lanes;
160  scalar_type.lanes = 1;
161 
162  Expr e;
163  switch (scalar_type.code) {
164  case halide_type_int:
165  e = IntImm::make(scalar_type, val.u.i64);
166  break;
167  case halide_type_uint:
168  e = UIntImm::make(scalar_type, val.u.u64);
169  break;
170  case halide_type_float:
171  case halide_type_bfloat:
172  e = FloatImm::make(scalar_type, val.u.f64);
173  break;
174  default:
175  // Unreachable
176  return Expr();
177  }
178  if (lanes > 1) {
179  e = Broadcast::make(e, lanes);
180  }
181  return e;
182 }
183 
184 bool equal_helper(const BaseExprNode &a, const BaseExprNode &b) noexcept;
185 
186 // A fast version of expression equality that assumes a well-typed non-null expression tree.
188 bool equal(const BaseExprNode &a, const BaseExprNode &b) noexcept {
189  // Early out
190  return (&a == &b) ||
191  ((a.type == b.type) &&
192  (a.node_type == b.node_type) &&
193  equal_helper(a, b));
194 }
195 
196 // A pattern that matches a specific expression
197 struct SpecificExpr {
198  struct pattern_tag {};
199 
200  constexpr static uint32_t binds = 0;
201 
202  // What is the weakest and strongest IR node this could possibly be
205  constexpr static bool canonical = true;
206 
208 
209  template<uint32_t bound>
210  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
211  return equal(*expr.get(), e);
212  }
213 
215  Expr make(MatcherState &state, halide_type_t type_hint) const {
216  return expr;
217  }
218 
219  constexpr static bool foldable = false;
220 };
221 
222 inline std::ostream &operator<<(std::ostream &s, const SpecificExpr &e) {
223  s << e.expr;
224  return s;
225 }
226 
227 template<int i>
228 struct WildConstInt {
229  struct pattern_tag {};
230 
231  constexpr static uint32_t binds = 1 << i;
232 
235  constexpr static bool canonical = true;
236 
237  template<uint32_t bound>
238  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
239  static_assert(i >= 0 && i < max_wild, "Wild with out-of-range index");
240  const BaseExprNode *op = &e;
241  if (op->node_type == IRNodeType::Broadcast) {
242  op = ((const Broadcast *)op)->value.get();
243  }
244  if (op->node_type != IRNodeType::IntImm) {
245  return false;
246  }
247  int64_t value = ((const IntImm *)op)->value;
248  if (bound & binds) {
250  halide_type_t type;
251  state.get_bound_const(i, val, type);
252  return e.type == type && value == val.u.i64;
253  }
254  state.set_bound_const(i, value, e.type);
255  return true;
256  }
257 
259  Expr make(MatcherState &state, halide_type_t type_hint) const {
261  halide_type_t type;
262  state.get_bound_const(i, val, type);
263  return make_const_expr(val, type);
264  }
265 
266  constexpr static bool foldable = true;
267 
270  state.get_bound_const(i, val, ty);
271  }
272 };
273 
274 template<int i>
275 std::ostream &operator<<(std::ostream &s, const WildConstInt<i> &c) {
276  s << "ci" << i;
277  return s;
278 }
279 
280 template<int i>
282  struct pattern_tag {};
283 
284  constexpr static uint32_t binds = 1 << i;
285 
288  constexpr static bool canonical = true;
289 
290  template<uint32_t bound>
291  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
292  static_assert(i >= 0 && i < max_wild, "Wild with out-of-range index");
293  const BaseExprNode *op = &e;
294  if (op->node_type == IRNodeType::Broadcast) {
295  op = ((const Broadcast *)op)->value.get();
296  }
297  if (op->node_type != IRNodeType::UIntImm) {
298  return false;
299  }
300  uint64_t value = ((const UIntImm *)op)->value;
301  if (bound & binds) {
303  halide_type_t type;
304  state.get_bound_const(i, val, type);
305  return e.type == type && value == val.u.u64;
306  }
307  state.set_bound_const(i, value, e.type);
308  return true;
309  }
310 
312  Expr make(MatcherState &state, halide_type_t type_hint) const {
314  halide_type_t type;
315  state.get_bound_const(i, val, type);
316  return make_const_expr(val, type);
317  }
318 
319  constexpr static bool foldable = true;
320 
322  void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept {
323  state.get_bound_const(i, val, ty);
324  }
325 };
326 
327 template<int i>
328 std::ostream &operator<<(std::ostream &s, const WildConstUInt<i> &c) {
329  s << "cu" << i;
330  return s;
331 }
332 
333 template<int i>
335  struct pattern_tag {};
336 
337  constexpr static uint32_t binds = 1 << i;
338 
341  constexpr static bool canonical = true;
342 
343  template<uint32_t bound>
344  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
345  static_assert(i >= 0 && i < max_wild, "Wild with out-of-range index");
346  const BaseExprNode *op = &e;
347  if (op->node_type == IRNodeType::Broadcast) {
348  op = ((const Broadcast *)op)->value.get();
349  }
350  if (op->node_type != IRNodeType::FloatImm) {
351  return false;
352  }
353  double value = ((const FloatImm *)op)->value;
354  if (bound & binds) {
356  halide_type_t type;
357  state.get_bound_const(i, val, type);
358  return e.type == type && value == val.u.f64;
359  }
360  state.set_bound_const(i, value, e.type);
361  return true;
362  }
363 
365  Expr make(MatcherState &state, halide_type_t type_hint) const {
367  halide_type_t type;
368  state.get_bound_const(i, val, type);
369  return make_const_expr(val, type);
370  }
371 
372  constexpr static bool foldable = true;
373 
375  void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept {
376  state.get_bound_const(i, val, ty);
377  }
378 };
379 
380 template<int i>
381 std::ostream &operator<<(std::ostream &s, const WildConstFloat<i> &c) {
382  s << "cf" << i;
383  return s;
384 }
385 
386 // Matches and binds to any constant Expr. Does not support constant-folding.
387 template<int i>
388 struct WildConst {
389  struct pattern_tag {};
390 
391  constexpr static uint32_t binds = 1 << i;
392 
395  constexpr static bool canonical = true;
396 
397  template<uint32_t bound>
398  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
399  static_assert(i >= 0 && i < max_wild, "Wild with out-of-range index");
400  const BaseExprNode *op = &e;
401  if (op->node_type == IRNodeType::Broadcast) {
402  op = ((const Broadcast *)op)->value.get();
403  }
404  switch (op->node_type) {
405  case IRNodeType::IntImm:
406  return WildConstInt<i>().template match<bound>(e, state);
407  case IRNodeType::UIntImm:
408  return WildConstUInt<i>().template match<bound>(e, state);
410  return WildConstFloat<i>().template match<bound>(e, state);
411  default:
412  return false;
413  }
414  }
415 
417  Expr make(MatcherState &state, halide_type_t type_hint) const {
419  halide_type_t type;
420  state.get_bound_const(i, val, type);
421  return make_const_expr(val, type);
422  }
423 
424  constexpr static bool foldable = true;
425 
427  void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept {
428  state.get_bound_const(i, val, ty);
429  }
430 };
431 
432 template<int i>
433 std::ostream &operator<<(std::ostream &s, const WildConst<i> &c) {
434  s << "c" << i;
435  return s;
436 }
437 
438 // Matches and binds to any Expr
439 template<int i>
440 struct Wild {
441  struct pattern_tag {};
442 
443  constexpr static uint32_t binds = 1 << (i + 16);
444 
447  constexpr static bool canonical = true;
448 
449  template<uint32_t bound>
450  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
451  if (bound & binds) {
452  return equal(*state.get_binding(i), e);
453  }
454  state.set_binding(i, e);
455  return true;
456  }
457 
459  Expr make(MatcherState &state, halide_type_t type_hint) const {
460  return state.get_binding(i);
461  }
462 
463  constexpr static bool foldable = true;
465  void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept {
466  auto e = state.get_binding(i);
467  ty = e->type;
468  switch (e->node_type) {
469  case IRNodeType::UIntImm:
470  val.u.u64 = ((const UIntImm *)e)->value;
471  return;
472  case IRNodeType::IntImm:
473  val.u.i64 = ((const IntImm *)e)->value;
474  return;
476  val.u.f64 = ((const FloatImm *)e)->value;
477  return;
478  default:
479  // The function is noexcept, so silent failure. You
480  // shouldn't be calling this if you haven't already
481  // checked it's going to be a constant (e.g. with
482  // is_const, or because you manually bound a constant Expr
483  // to the state).
484  val.u.u64 = 0;
485  }
486  }
487 };
488 
489 template<int i>
490 std::ostream &operator<<(std::ostream &s, const Wild<i> &op) {
491  s << "_" << i;
492  return s;
493 }
494 
495 // Matches a specific constant or broadcast of that constant. The
496 // constant must be representable as an int64_t.
497 struct Const {
498  struct pattern_tag {};
500 
501  constexpr static uint32_t binds = 0;
502 
505  constexpr static bool canonical = true;
506 
509  : v(v) {
510  }
511 
512  template<uint32_t bound>
513  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
514  const BaseExprNode *op = &e;
515  if (e.node_type == IRNodeType::Broadcast) {
516  op = ((const Broadcast *)op)->value.get();
517  }
518  switch (op->node_type) {
519  case IRNodeType::IntImm:
520  return ((const IntImm *)op)->value == (int64_t)v;
521  case IRNodeType::UIntImm:
522  return ((const UIntImm *)op)->value == (uint64_t)v;
524  return ((const FloatImm *)op)->value == (double)v;
525  default:
526  return false;
527  }
528  }
529 
530  template<uint32_t bound>
531  HALIDE_ALWAYS_INLINE bool match(const Const &b, MatcherState &state) const noexcept {
532  return v == b.v;
533  }
534 
536  Expr make(MatcherState &state, halide_type_t type_hint) const {
537  return make_const(type_hint, v);
538  }
539 
540  constexpr static bool foldable = true;
541 
543  void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept {
544  // Assume type is already correct
545  switch (ty.code) {
546  case halide_type_int:
547  val.u.i64 = v;
548  break;
549  case halide_type_uint:
550  val.u.u64 = (uint64_t)v;
551  break;
552  case halide_type_float:
553  case halide_type_bfloat:
554  val.u.f64 = (double)v;
555  break;
556  default:
557  // Unreachable
558  ;
559  }
560  }
561 };
562 
563 // Convert a provided pattern, expr, or constant int into the internal
564 // representation we use in the matcher trees.
565 template<typename T,
566  typename = typename std::decay<T>::type::pattern_tag>
568  return t;
569 }
572  return {x};
573 }
575 const SpecificExpr pattern_arg(const Expr &e) {
576  return {e};
577 }
578 
579 // Helpers to deref SpecificExprs to const BaseExprNode & rather than
580 // passing them by value anywhere (incurring lots of refcounting)
581 template<typename T,
582  // T must be a pattern node
583  typename = typename std::decay<T>::type::pattern_tag,
584  // But T may not be SpecificExpr
585  typename = typename std::enable_if<!std::is_same<typename std::decay<T>::type, SpecificExpr>::value>::type>
587  return t;
588 }
589 
591 const BaseExprNode &unwrap(const SpecificExpr &e) {
592  return *e.expr.get();
593 }
594 
595 inline std::ostream &operator<<(std::ostream &s, const Const &op) {
596  s << op.v;
597  return s;
598 }
599 
600 template<typename Op>
602 
603 template<typename Op>
605 
606 template<typename Op>
607 double constant_fold_bin_op(halide_type_t &, double, double) noexcept;
608 
609 constexpr bool commutative(IRNodeType t) {
610  return (t == IRNodeType::Add ||
611  t == IRNodeType::Mul ||
612  t == IRNodeType::And ||
613  t == IRNodeType::Or ||
614  t == IRNodeType::Min ||
615  t == IRNodeType::Max ||
616  t == IRNodeType::EQ ||
617  t == IRNodeType::NE);
618 }
619 
620 // Matches one of the binary operators
621 template<typename Op, typename A, typename B>
622 struct BinOp {
623  struct pattern_tag {};
624  A a;
625  B b;
626 
628 
629  constexpr static IRNodeType min_node_type = Op::_node_type;
630  constexpr static IRNodeType max_node_type = Op::_node_type;
631 
632  // For commutative bin ops, we expect the weaker IR node type on
633  // the right. That is, for the rule to be canonical it must be
634  // possible that A is at least as strong as B.
635  constexpr static bool canonical =
636  A::canonical && B::canonical && (!commutative(Op::_node_type) || (A::max_node_type >= B::min_node_type));
637 
638  template<uint32_t bound>
639  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
640  if (e.node_type != Op::_node_type) {
641  return false;
642  }
643  const Op &op = (const Op &)e;
644  return (a.template match<bound>(*op.a.get(), state) &&
645  b.template match<bound | bindings<A>::mask>(*op.b.get(), state));
646  }
647 
648  template<uint32_t bound, typename Op2, typename A2, typename B2>
649  HALIDE_ALWAYS_INLINE bool match(const BinOp<Op2, A2, B2> &op, MatcherState &state) const noexcept {
650  return (std::is_same<Op, Op2>::value &&
651  a.template match<bound>(unwrap(op.a), state) &&
652  b.template match<bound | bindings<A>::mask>(unwrap(op.b), state));
653  }
654 
655  constexpr static bool foldable = A::foldable && B::foldable;
656 
658  void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept {
659  halide_scalar_value_t val_a, val_b;
660  if (std::is_same<A, Const>::value) {
661  b.make_folded_const(val_b, ty, state);
662  if ((std::is_same<Op, And>::value && val_b.u.u64 == 0) ||
663  (std::is_same<Op, Or>::value && val_b.u.u64 == 1)) {
664  // Short circuit
665  val = val_b;
666  return;
667  }
668  const uint16_t l = ty.lanes;
669  a.make_folded_const(val_a, ty, state);
670  ty.lanes |= l; // Make sure the overflow bits are sticky
671  } else {
672  a.make_folded_const(val_a, ty, state);
673  if ((std::is_same<Op, And>::value && val_a.u.u64 == 0) ||
674  (std::is_same<Op, Or>::value && val_a.u.u64 == 1)) {
675  // Short circuit
676  val = val_a;
677  return;
678  }
679  const uint16_t l = ty.lanes;
680  b.make_folded_const(val_b, ty, state);
681  ty.lanes |= l;
682  }
683  switch (ty.code) {
684  case halide_type_int:
685  val.u.i64 = constant_fold_bin_op<Op>(ty, val_a.u.i64, val_b.u.i64);
686  break;
687  case halide_type_uint:
688  val.u.u64 = constant_fold_bin_op<Op>(ty, val_a.u.u64, val_b.u.u64);
689  break;
690  case halide_type_float:
691  case halide_type_bfloat:
692  val.u.f64 = constant_fold_bin_op<Op>(ty, val_a.u.f64, val_b.u.f64);
693  break;
694  default:
695  // unreachable
696  ;
697  }
698  }
699 
701  Expr make(MatcherState &state, halide_type_t type_hint) const noexcept {
702  Expr ea, eb;
703  if (std::is_same<A, Const>::value) {
704  eb = b.make(state, type_hint);
705  ea = a.make(state, eb.type());
706  } else {
707  ea = a.make(state, type_hint);
708  eb = b.make(state, ea.type());
709  }
710  // We sometimes mix vectors and scalars in the rewrite rules,
711  // so insert a broadcast if necessary.
712  if (ea.type().is_vector() && !eb.type().is_vector()) {
713  eb = Broadcast::make(eb, ea.type().lanes());
714  }
715  if (eb.type().is_vector() && !ea.type().is_vector()) {
716  ea = Broadcast::make(ea, eb.type().lanes());
717  }
718  return Op::make(std::move(ea), std::move(eb));
719  }
720 };
721 
722 template<typename Op>
724 
725 template<typename Op>
727 
728 template<typename Op>
729 uint64_t constant_fold_cmp_op(double, double) noexcept;
730 
731 // Matches one of the comparison operators
732 template<typename Op, typename A, typename B>
733 struct CmpOp {
734  struct pattern_tag {};
735  A a;
736  B b;
737 
739 
740  constexpr static IRNodeType min_node_type = Op::_node_type;
741  constexpr static IRNodeType max_node_type = Op::_node_type;
742  constexpr static bool canonical = (A::canonical &&
743  B::canonical &&
744  (!commutative(Op::_node_type) || A::max_node_type >= B::min_node_type) &&
745  (Op::_node_type != IRNodeType::GE) &&
746  (Op::_node_type != IRNodeType::GT));
747 
748  template<uint32_t bound>
749  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
750  if (e.node_type != Op::_node_type) {
751  return false;
752  }
753  const Op &op = (const Op &)e;
754  return (a.template match<bound>(*op.a.get(), state) &&
755  b.template match<bound | bindings<A>::mask>(*op.b.get(), state));
756  }
757 
758  template<uint32_t bound, typename Op2, typename A2, typename B2>
759  HALIDE_ALWAYS_INLINE bool match(const CmpOp<Op2, A2, B2> &op, MatcherState &state) const noexcept {
760  return (std::is_same<Op, Op2>::value &&
761  a.template match<bound>(unwrap(op.a), state) &&
762  b.template match<bound | bindings<A>::mask>(unwrap(op.b), state));
763  }
764 
765  constexpr static bool foldable = A::foldable && B::foldable;
766 
768  void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept {
769  halide_scalar_value_t val_a, val_b;
770  // If one side is an untyped const, evaluate the other side first to get a type hint.
771  if (std::is_same<A, Const>::value) {
772  b.make_folded_const(val_b, ty, state);
773  const uint16_t l = ty.lanes;
774  a.make_folded_const(val_a, ty, state);
775  ty.lanes |= l;
776  } else {
777  a.make_folded_const(val_a, ty, state);
778  const uint16_t l = ty.lanes;
779  b.make_folded_const(val_b, ty, state);
780  ty.lanes |= l;
781  }
782  switch (ty.code) {
783  case halide_type_int:
784  val.u.u64 = constant_fold_cmp_op<Op>(val_a.u.i64, val_b.u.i64);
785  break;
786  case halide_type_uint:
787  val.u.u64 = constant_fold_cmp_op<Op>(val_a.u.u64, val_b.u.u64);
788  break;
789  case halide_type_float:
790  case halide_type_bfloat:
791  val.u.u64 = constant_fold_cmp_op<Op>(val_a.u.f64, val_b.u.f64);
792  break;
793  default:
794  // unreachable
795  ;
796  }
797  ty.code = halide_type_uint;
798  ty.bits = 1;
799  }
800 
802  Expr make(MatcherState &state, halide_type_t type_hint) const {
803  // If one side is an untyped const, evaluate the other side first to get a type hint.
804  Expr ea, eb;
805  if (std::is_same<A, Const>::value) {
806  eb = b.make(state, {});
807  ea = a.make(state, eb.type());
808  } else {
809  ea = a.make(state, {});
810  eb = b.make(state, ea.type());
811  }
812  // We sometimes mix vectors and scalars in the rewrite rules,
813  // so insert a broadcast if necessary.
814  if (ea.type().is_vector() && !eb.type().is_vector()) {
815  eb = Broadcast::make(eb, ea.type().lanes());
816  }
817  if (eb.type().is_vector() && !ea.type().is_vector()) {
818  ea = Broadcast::make(ea, eb.type().lanes());
819  }
820  return Op::make(std::move(ea), std::move(eb));
821  }
822 };
823 
824 template<typename A, typename B>
825 std::ostream &operator<<(std::ostream &s, const BinOp<Add, A, B> &op) {
826  s << "(" << op.a << " + " << op.b << ")";
827  return s;
828 }
829 
830 template<typename A, typename B>
831 std::ostream &operator<<(std::ostream &s, const BinOp<Sub, A, B> &op) {
832  s << "(" << op.a << " - " << op.b << ")";
833  return s;
834 }
835 
836 template<typename A, typename B>
837 std::ostream &operator<<(std::ostream &s, const BinOp<Mul, A, B> &op) {
838  s << "(" << op.a << " * " << op.b << ")";
839  return s;
840 }
841 
842 template<typename A, typename B>
843 std::ostream &operator<<(std::ostream &s, const BinOp<Div, A, B> &op) {
844  s << "(" << op.a << " / " << op.b << ")";
845  return s;
846 }
847 
848 template<typename A, typename B>
849 std::ostream &operator<<(std::ostream &s, const BinOp<And, A, B> &op) {
850  s << "(" << op.a << " && " << op.b << ")";
851  return s;
852 }
853 
854 template<typename A, typename B>
855 std::ostream &operator<<(std::ostream &s, const BinOp<Or, A, B> &op) {
856  s << "(" << op.a << " || " << op.b << ")";
857  return s;
858 }
859 
860 template<typename A, typename B>
861 std::ostream &operator<<(std::ostream &s, const BinOp<Min, A, B> &op) {
862  s << "min(" << op.a << ", " << op.b << ")";
863  return s;
864 }
865 
866 template<typename A, typename B>
867 std::ostream &operator<<(std::ostream &s, const BinOp<Max, A, B> &op) {
868  s << "max(" << op.a << ", " << op.b << ")";
869  return s;
870 }
871 
872 template<typename A, typename B>
873 std::ostream &operator<<(std::ostream &s, const CmpOp<LE, A, B> &op) {
874  s << "(" << op.a << " <= " << op.b << ")";
875  return s;
876 }
877 
878 template<typename A, typename B>
879 std::ostream &operator<<(std::ostream &s, const CmpOp<LT, A, B> &op) {
880  s << "(" << op.a << " < " << op.b << ")";
881  return s;
882 }
883 
884 template<typename A, typename B>
885 std::ostream &operator<<(std::ostream &s, const CmpOp<GE, A, B> &op) {
886  s << "(" << op.a << " >= " << op.b << ")";
887  return s;
888 }
889 
890 template<typename A, typename B>
891 std::ostream &operator<<(std::ostream &s, const CmpOp<GT, A, B> &op) {
892  s << "(" << op.a << " > " << op.b << ")";
893  return s;
894 }
895 
896 template<typename A, typename B>
897 std::ostream &operator<<(std::ostream &s, const CmpOp<EQ, A, B> &op) {
898  s << "(" << op.a << " == " << op.b << ")";
899  return s;
900 }
901 
902 template<typename A, typename B>
903 std::ostream &operator<<(std::ostream &s, const CmpOp<NE, A, B> &op) {
904  s << "(" << op.a << " != " << op.b << ")";
905  return s;
906 }
907 
908 template<typename A, typename B>
909 std::ostream &operator<<(std::ostream &s, const BinOp<Mod, A, B> &op) {
910  s << "(" << op.a << " % " << op.b << ")";
911  return s;
912 }
913 
914 template<typename A, typename B>
915 HALIDE_ALWAYS_INLINE auto operator+(A a, B b) noexcept -> BinOp<Add, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
916  return {pattern_arg(a), pattern_arg(b)};
917 }
918 
919 template<typename A, typename B>
920 HALIDE_ALWAYS_INLINE auto add(A a, B b) -> decltype(IRMatcher::operator+(a, b)) {
921  return IRMatcher::operator+(a, b);
922 }
923 
924 template<>
926  t.lanes |= ((t.bits >= 32) && add_would_overflow(t.bits, a, b)) ? MatcherState::signed_integer_overflow : 0;
927  int dead_bits = 64 - t.bits;
928  // Drop the high bits then sign-extend them back
929  return int64_t((uint64_t(a) + uint64_t(b)) << dead_bits) >> dead_bits;
930 }
931 
932 template<>
934  uint64_t ones = (uint64_t)(-1);
935  return (a + b) & (ones >> (64 - t.bits));
936 }
937 
938 template<>
939 HALIDE_ALWAYS_INLINE double constant_fold_bin_op<Add>(halide_type_t &t, double a, double b) noexcept {
940  return a + b;
941 }
942 
943 template<typename A, typename B>
944 HALIDE_ALWAYS_INLINE auto operator-(A a, B b) noexcept -> BinOp<Sub, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
945  return {pattern_arg(a), pattern_arg(b)};
946 }
947 
948 template<typename A, typename B>
949 HALIDE_ALWAYS_INLINE auto sub(A a, B b) -> decltype(IRMatcher::operator-(a, b)) {
950  return IRMatcher::operator-(a, b);
951 }
952 
953 template<>
955  t.lanes |= ((t.bits >= 32) && sub_would_overflow(t.bits, a, b)) ? MatcherState::signed_integer_overflow : 0;
956  // Drop the high bits then sign-extend them back
957  int dead_bits = 64 - t.bits;
958  return int64_t((uint64_t(a) - uint64_t(b)) << dead_bits) >> dead_bits;
959 }
960 
961 template<>
963  uint64_t ones = (uint64_t)(-1);
964  return (a - b) & (ones >> (64 - t.bits));
965 }
966 
967 template<>
968 HALIDE_ALWAYS_INLINE double constant_fold_bin_op<Sub>(halide_type_t &t, double a, double b) noexcept {
969  return a - b;
970 }
971 
972 template<typename A, typename B>
973 HALIDE_ALWAYS_INLINE auto operator*(A a, B b) noexcept -> BinOp<Mul, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
974  return {pattern_arg(a), pattern_arg(b)};
975 }
976 
977 template<typename A, typename B>
978 HALIDE_ALWAYS_INLINE auto mul(A a, B b) -> decltype(IRMatcher::operator*(a, b)) {
979  return IRMatcher::operator*(a, b);
980 }
981 
982 template<>
984  t.lanes |= ((t.bits >= 32) && mul_would_overflow(t.bits, a, b)) ? MatcherState::signed_integer_overflow : 0;
985  int dead_bits = 64 - t.bits;
986  // Drop the high bits then sign-extend them back
987  return int64_t((uint64_t(a) * uint64_t(b)) << dead_bits) >> dead_bits;
988 }
989 
990 template<>
992  uint64_t ones = (uint64_t)(-1);
993  return (a * b) & (ones >> (64 - t.bits));
994 }
995 
996 template<>
997 HALIDE_ALWAYS_INLINE double constant_fold_bin_op<Mul>(halide_type_t &t, double a, double b) noexcept {
998  return a * b;
999 }
1000 
1001 template<typename A, typename B>
1002 HALIDE_ALWAYS_INLINE auto operator/(A a, B b) noexcept -> BinOp<Div, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1003  return {pattern_arg(a), pattern_arg(b)};
1004 }
1005 
1006 template<typename A, typename B>
1007 HALIDE_ALWAYS_INLINE auto div(A a, B b) -> decltype(IRMatcher::operator/(a, b)) {
1008  return IRMatcher::operator/(a, b);
1009 }
1010 
1011 template<>
1013  return div_imp(a, b);
1014 }
1015 
1016 template<>
1018  return div_imp(a, b);
1019 }
1020 
1021 template<>
1022 HALIDE_ALWAYS_INLINE double constant_fold_bin_op<Div>(halide_type_t &t, double a, double b) noexcept {
1023  return div_imp(a, b);
1024 }
1025 
1026 template<typename A, typename B>
1027 HALIDE_ALWAYS_INLINE auto operator%(A a, B b) noexcept -> BinOp<Mod, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1028  return {pattern_arg(a), pattern_arg(b)};
1029 }
1030 
1031 template<typename A, typename B>
1032 HALIDE_ALWAYS_INLINE auto mod(A a, B b) -> decltype(IRMatcher::operator%(a, b)) {
1033  return IRMatcher::operator%(a, b);
1034 }
1035 
1036 template<>
1038  return mod_imp(a, b);
1039 }
1040 
1041 template<>
1043  return mod_imp(a, b);
1044 }
1045 
1046 template<>
1047 HALIDE_ALWAYS_INLINE double constant_fold_bin_op<Mod>(halide_type_t &t, double a, double b) noexcept {
1048  return mod_imp(a, b);
1049 }
1050 
1051 template<typename A, typename B>
1052 HALIDE_ALWAYS_INLINE auto min(A a, B b) noexcept -> BinOp<Min, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1053  return {pattern_arg(a), pattern_arg(b)};
1054 }
1055 
1056 template<>
1058  return std::min(a, b);
1059 }
1060 
1061 template<>
1063  return std::min(a, b);
1064 }
1065 
1066 template<>
1067 HALIDE_ALWAYS_INLINE double constant_fold_bin_op<Min>(halide_type_t &t, double a, double b) noexcept {
1068  return std::min(a, b);
1069 }
1070 
1071 template<typename A, typename B>
1072 HALIDE_ALWAYS_INLINE auto max(A a, B b) noexcept -> BinOp<Max, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1073  return {pattern_arg(a), pattern_arg(b)};
1074 }
1075 
1076 template<>
1078  return std::max(a, b);
1079 }
1080 
1081 template<>
1083  return std::max(a, b);
1084 }
1085 
1086 template<>
1087 HALIDE_ALWAYS_INLINE double constant_fold_bin_op<Max>(halide_type_t &t, double a, double b) noexcept {
1088  return std::max(a, b);
1089 }
1090 
1091 template<typename A, typename B>
1092 HALIDE_ALWAYS_INLINE auto operator<(A a, B b) noexcept -> CmpOp<LT, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1093  return {pattern_arg(a), pattern_arg(b)};
1094 }
1095 
1096 template<typename A, typename B>
1097 HALIDE_ALWAYS_INLINE auto lt(A a, B b) -> decltype(IRMatcher::operator<(a, b)) {
1098  return IRMatcher::operator<(a, b);
1099 }
1100 
1101 template<>
1103  return a < b;
1104 }
1105 
1106 template<>
1108  return a < b;
1109 }
1110 
1111 template<>
1113  return a < b;
1114 }
1115 
1116 template<typename A, typename B>
1117 HALIDE_ALWAYS_INLINE auto operator>(A a, B b) noexcept -> CmpOp<GT, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1118  return {pattern_arg(a), pattern_arg(b)};
1119 }
1120 
1121 template<typename A, typename B>
1122 HALIDE_ALWAYS_INLINE auto gt(A a, B b) -> decltype(IRMatcher::operator>(a, b)) {
1123  return IRMatcher::operator>(a, b);
1124 }
1125 
1126 template<>
1128  return a > b;
1129 }
1130 
1131 template<>
1133  return a > b;
1134 }
1135 
1136 template<>
1138  return a > b;
1139 }
1140 
1141 template<typename A, typename B>
1142 HALIDE_ALWAYS_INLINE auto operator<=(A a, B b) noexcept -> CmpOp<LE, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1143  return {pattern_arg(a), pattern_arg(b)};
1144 }
1145 
1146 template<typename A, typename B>
1147 HALIDE_ALWAYS_INLINE auto le(A a, B b) -> decltype(IRMatcher::operator<=(a, b)) {
1148  return IRMatcher::operator<=(a, b);
1149 }
1150 
1151 template<>
1153  return a <= b;
1154 }
1155 
1156 template<>
1158  return a <= b;
1159 }
1160 
1161 template<>
1163  return a <= b;
1164 }
1165 
1166 template<typename A, typename B>
1167 HALIDE_ALWAYS_INLINE auto operator>=(A a, B b) noexcept -> CmpOp<GE, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1168  return {pattern_arg(a), pattern_arg(b)};
1169 }
1170 
1171 template<typename A, typename B>
1172 HALIDE_ALWAYS_INLINE auto ge(A a, B b) -> decltype(IRMatcher::operator>=(a, b)) {
1173  return IRMatcher::operator>=(a, b);
1174 }
1175 
1176 template<>
1178  return a >= b;
1179 }
1180 
1181 template<>
1183  return a >= b;
1184 }
1185 
1186 template<>
1188  return a >= b;
1189 }
1190 
1191 template<typename A, typename B>
1192 HALIDE_ALWAYS_INLINE auto operator==(A a, B b) noexcept -> CmpOp<EQ, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1193  return {pattern_arg(a), pattern_arg(b)};
1194 }
1195 
1196 template<typename A, typename B>
1197 HALIDE_ALWAYS_INLINE auto eq(A a, B b) -> decltype(IRMatcher::operator==(a, b)) {
1198  return IRMatcher::operator==(a, b);
1199 }
1200 
1201 template<>
1203  return a == b;
1204 }
1205 
1206 template<>
1208  return a == b;
1209 }
1210 
1211 template<>
1213  return a == b;
1214 }
1215 
1216 template<typename A, typename B>
1217 HALIDE_ALWAYS_INLINE auto operator!=(A a, B b) noexcept -> CmpOp<NE, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1218  return {pattern_arg(a), pattern_arg(b)};
1219 }
1220 
1221 template<typename A, typename B>
1222 HALIDE_ALWAYS_INLINE auto ne(A a, B b) -> decltype(IRMatcher::operator!=(a, b)) {
1223  return IRMatcher::operator!=(a, b);
1224 }
1225 
1226 template<>
1228  return a != b;
1229 }
1230 
1231 template<>
1233  return a != b;
1234 }
1235 
1236 template<>
1238  return a != b;
1239 }
1240 
1241 template<typename A, typename B>
1242 HALIDE_ALWAYS_INLINE auto operator||(A a, B b) noexcept -> BinOp<Or, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1243  return {pattern_arg(a), pattern_arg(b)};
1244 }
1245 
1246 template<typename A, typename B>
1247 HALIDE_ALWAYS_INLINE auto or_op(A a, B b) -> decltype(IRMatcher::operator||(a, b)) {
1248  return IRMatcher::operator||(a, b);
1249 }
1250 
1251 template<>
1253  return (a | b) & 1;
1254 }
1255 
1256 template<>
1258  return (a | b) & 1;
1259 }
1260 
1261 template<>
1262 HALIDE_ALWAYS_INLINE double constant_fold_bin_op<Or>(halide_type_t &t, double a, double b) noexcept {
1263  // Unreachable, as it would be a type mismatch.
1264  return 0;
1265 }
1266 
1267 template<typename A, typename B>
1268 HALIDE_ALWAYS_INLINE auto operator&&(A a, B b) noexcept -> BinOp<And, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1269  return {pattern_arg(a), pattern_arg(b)};
1270 }
1271 
1272 template<typename A, typename B>
1273 HALIDE_ALWAYS_INLINE auto and_op(A a, B b) -> decltype(IRMatcher::operator&&(a, b)) {
1274  return IRMatcher::operator&&(a, b);
1275 }
1276 
1277 template<>
1279  return a & b & 1;
1280 }
1281 
1282 template<>
1284  return a & b & 1;
1285 }
1286 
1287 template<>
1288 HALIDE_ALWAYS_INLINE double constant_fold_bin_op<And>(halide_type_t &t, double a, double b) noexcept {
1289  // Unreachable
1290  return 0;
1291 }
1292 
1293 constexpr inline uint32_t bitwise_or_reduce() {
1294  return 0;
1295 }
1296 
1297 template<typename... Args>
1298 constexpr uint32_t bitwise_or_reduce(uint32_t first, Args... rest) {
1299  return first | bitwise_or_reduce(rest...);
1300 }
1301 
1302 constexpr inline bool and_reduce() {
1303  return true;
1304 }
1305 
1306 template<typename... Args>
1307 constexpr bool and_reduce(bool first, Args... rest) {
1308  return first && and_reduce(rest...);
1309 }
1310 
1311 template<typename... Args>
1312 struct Intrin {
1313  struct pattern_tag {};
1315  std::tuple<Args...> args;
1316 
1318 
1321  constexpr static bool canonical = and_reduce((Args::canonical)...);
1322 
1323  template<int i,
1324  uint32_t bound,
1325  typename = typename std::enable_if<(i < sizeof...(Args))>::type>
1326  HALIDE_ALWAYS_INLINE bool match_args(int, const Call &c, MatcherState &state) const noexcept {
1327  using T = decltype(std::get<i>(args));
1328  return (std::get<i>(args).template match<bound>(*c.args[i].get(), state) &&
1329  match_args<i + 1, bound | bindings<T>::mask>(0, c, state));
1330  }
1331 
1332  template<int i, uint32_t binds>
1333  HALIDE_ALWAYS_INLINE bool match_args(double, const Call &c, MatcherState &state) const noexcept {
1334  return true;
1335  }
1336 
1337  template<uint32_t bound>
1338  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
1339  if (e.node_type != IRNodeType::Call) {
1340  return false;
1341  }
1342  const Call &c = (const Call &)e;
1343  return (c.is_intrinsic(intrin) && match_args<0, bound>(0, c, state));
1344  }
1345 
1346  template<int i,
1347  typename = typename std::enable_if<(i < sizeof...(Args))>::type>
1348  HALIDE_ALWAYS_INLINE void print_args(int, std::ostream &s) const {
1349  s << std::get<i>(args);
1350  if (i + 1 < sizeof...(Args)) {
1351  s << ", ";
1352  }
1353  print_args<i + 1>(0, s);
1354  }
1355 
1356  template<int i>
1357  HALIDE_ALWAYS_INLINE void print_args(double, std::ostream &s) const {
1358  }
1359 
1361  void print_args(std::ostream &s) const {
1362  print_args<0>(0, s);
1363  }
1364 
1366  Expr make(MatcherState &state, halide_type_t type_hint) const {
1367  if (intrin == Call::likely) {
1368  return likely(std::get<0>(args).make(state, type_hint));
1369  } else if (intrin == Call::likely_if_innermost) {
1370  return likely_if_innermost(std::get<0>(args).make(state, type_hint));
1371  }
1372  internal_error << "Unhandled intrinsic in IRMatcher: " << intrin;
1373  return Expr();
1374  }
1375 
1376  constexpr static bool foldable = false;
1377 
1380  : intrin(intrin), args(args...) {
1381  }
1382 };
1383 
1384 template<typename... Args>
1385 std::ostream &operator<<(std::ostream &s, const Intrin<Args...> &op) {
1386  s << op.intrin << "(";
1387  op.print_args(s);
1388  s << ")";
1389  return s;
1390 }
1391 
1392 template<typename... Args>
1393 HALIDE_ALWAYS_INLINE auto intrin(Call::IntrinsicOp intrinsic_op, Args... args) noexcept -> Intrin<decltype(pattern_arg(args))...> {
1394  return {intrinsic_op, pattern_arg(args)...};
1395 }
1396 
1397 template<typename A>
1398 struct NotOp {
1399  struct pattern_tag {};
1400  A a;
1401 
1402  constexpr static uint32_t binds = bindings<A>::mask;
1403 
1406  constexpr static bool canonical = A::canonical;
1407 
1408  template<uint32_t bound>
1409  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
1410  if (e.node_type != IRNodeType::Not) {
1411  return false;
1412  }
1413  const Not &op = (const Not &)e;
1414  return (a.template match<bound>(*op.a.get(), state));
1415  }
1416 
1417  template<uint32_t bound, typename A2>
1418  HALIDE_ALWAYS_INLINE bool match(const NotOp<A2> &op, MatcherState &state) const noexcept {
1419  return a.template match<bound>(unwrap(op.a), state);
1420  }
1421 
1423  Expr make(MatcherState &state, halide_type_t type_hint) const {
1424  return Not::make(a.make(state, type_hint));
1425  }
1426 
1427  constexpr static bool foldable = A::foldable;
1428 
1429  template<typename A1 = A>
1431  a.make_folded_const(val, ty, state);
1432  val.u.u64 = ~val.u.u64;
1433  val.u.u64 &= 1;
1434  }
1435 };
1436 
1437 template<typename A>
1438 HALIDE_ALWAYS_INLINE auto operator!(A a) noexcept -> NotOp<decltype(pattern_arg(a))> {
1439  return {pattern_arg(a)};
1440 }
1441 
1442 template<typename A>
1443 HALIDE_ALWAYS_INLINE auto not_op(A a) -> decltype(IRMatcher::operator!(a)) {
1444  return IRMatcher::operator!(a);
1445 }
1446 
1447 template<typename A>
1448 inline std::ostream &operator<<(std::ostream &s, const NotOp<A> &op) {
1449  s << "!(" << op.a << ")";
1450  return s;
1451 }
1452 
1453 template<typename C, typename T, typename F>
1454 struct SelectOp {
1455  struct pattern_tag {};
1456  C c;
1457  T t;
1458  F f;
1459 
1461 
1464 
1465  constexpr static bool canonical = C::canonical && T::canonical && F::canonical;
1466 
1467  template<uint32_t bound>
1468  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
1469  if (e.node_type != Select::_node_type) {
1470  return false;
1471  }
1472  const Select &op = (const Select &)e;
1473  return (c.template match<bound>(*op.condition.get(), state) &&
1474  t.template match<bound | bindings<C>::mask>(*op.true_value.get(), state) &&
1475  f.template match<bound | bindings<C>::mask | bindings<T>::mask>(*op.false_value.get(), state));
1476  }
1477  template<uint32_t bound, typename C2, typename T2, typename F2>
1478  HALIDE_ALWAYS_INLINE bool match(const SelectOp<C2, T2, F2> &instance, MatcherState &state) const noexcept {
1479  return (c.template match<bound>(unwrap(instance.c), state) &&
1480  t.template match<bound | bindings<C>::mask>(unwrap(instance.t), state) &&
1481  f.template match<bound | bindings<C>::mask | bindings<T>::mask>(unwrap(instance.f), state));
1482  }
1483 
1485  Expr make(MatcherState &state, halide_type_t type_hint) const {
1486  return Select::make(c.make(state, {}), t.make(state, type_hint), f.make(state, type_hint));
1487  }
1488 
1489  constexpr static bool foldable = C::foldable && T::foldable && F::foldable;
1490 
1491  template<typename C1 = C>
1493  halide_scalar_value_t c_val, t_val, f_val;
1494  halide_type_t c_ty;
1495  c.make_folded_const(c_val, c_ty, state);
1496  if ((c_val.u.u64 & 1) == 1) {
1497  t.make_folded_const(val, ty, state);
1498  } else {
1499  f.make_folded_const(val, ty, state);
1500  }
1501  ty.lanes |= c_ty.lanes & MatcherState::special_values_mask;
1502  }
1503 };
1504 
1505 template<typename C, typename T, typename F>
1506 std::ostream &operator<<(std::ostream &s, const SelectOp<C, T, F> &op) {
1507  s << "select(" << op.c << ", " << op.t << ", " << op.f << ")";
1508  return s;
1509 }
1510 
1511 template<typename C, typename T, typename F>
1512 HALIDE_ALWAYS_INLINE auto select(C c, T t, F f) noexcept -> SelectOp<decltype(pattern_arg(c)), decltype(pattern_arg(t)), decltype(pattern_arg(f))> {
1513  return {pattern_arg(c), pattern_arg(t), pattern_arg(f)};
1514 }
1515 
1516 template<typename A, bool known_lanes>
1517 struct BroadcastOp {
1518  struct pattern_tag {};
1519  A a;
1520  int lanes;
1521 
1522  constexpr static uint32_t binds = bindings<A>::mask;
1523 
1526 
1527  constexpr static bool canonical = A::canonical;
1528 
1529  template<uint32_t bound>
1530  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
1531  if (e.node_type == Broadcast::_node_type) {
1532  const Broadcast &op = (const Broadcast &)e;
1533  if ((!known_lanes || lanes == op.lanes) &&
1534  a.template match<bound>(*op.value.get(), state)) {
1535  return true;
1536  }
1537  }
1538  return false;
1539  }
1540 
1541  template<uint32_t bound, typename A2, bool known_lanes_2>
1543  return (a.template match<bound>(unwrap(op.a), state) &&
1544  (lanes == op.lanes || !known_lanes || !known_lanes_2));
1545  }
1546 
1548  Expr make(MatcherState &state, halide_type_t type_hint) const {
1549  const int l = known_lanes ? lanes : type_hint.lanes;
1550  type_hint.lanes = 1;
1551  Expr val = a.make(state, type_hint);
1552  if (l == 1) {
1553  return val;
1554  } else {
1555  return Broadcast::make(std::move(val), l);
1556  }
1557  }
1558 
1559  constexpr static bool foldable = false;
1560 
1561  template<typename A1 = A>
1563  uint16_t l = known_lanes ? lanes : ty.lanes;
1564  a.make_folded_const(val, ty, state);
1565  ty.lanes = l | (ty.lanes & MatcherState::special_values_mask);
1566  }
1567 };
1568 
1569 template<typename A>
1570 inline std::ostream &operator<<(std::ostream &s, const BroadcastOp<A, true> &op) {
1571  s << "broadcast(" << op.a << ", " << op.lanes << ")";
1572  return s;
1573 }
1574 
1575 template<typename A>
1576 inline std::ostream &operator<<(std::ostream &s, const BroadcastOp<A, false> &op) {
1577  s << "broadcast(" << op.a << ")";
1578  return s;
1579 }
1580 
1581 template<typename A>
1582 HALIDE_ALWAYS_INLINE auto broadcast(A a, int lanes) noexcept -> BroadcastOp<decltype(pattern_arg(a)), true> {
1583  return {pattern_arg(a), lanes};
1584 }
1585 
1586 template<typename A>
1587 HALIDE_ALWAYS_INLINE auto broadcast(A a) noexcept -> BroadcastOp<decltype(pattern_arg(a)), false> {
1588  return {pattern_arg(a), 0};
1589 }
1590 
1591 template<typename A, typename B, bool known_lanes>
1592 struct RampOp {
1593  struct pattern_tag {};
1594  A a;
1595  B b;
1596  int lanes;
1597 
1599 
1602 
1603  constexpr static bool canonical = A::canonical && B::canonical;
1604 
1605  template<uint32_t bound>
1606  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
1607  if (e.node_type != Ramp::_node_type) {
1608  return false;
1609  }
1610  const Ramp &op = (const Ramp &)e;
1611  if ((lanes == op.type.lanes() || !known_lanes) &&
1612  a.template match<bound>(*op.base.get(), state) &&
1613  b.template match<bound | bindings<A>::mask>(*op.stride.get(), state)) {
1614  return true;
1615  } else {
1616  return false;
1617  }
1618  }
1619 
1620  template<uint32_t bound, typename A2, typename B2, bool known_lanes_2>
1622  return ((lanes == op.lanes || !known_lanes || !known_lanes_2) &&
1623  a.template match<bound>(unwrap(op.a), state) &&
1624  b.template match<bound | bindings<A>::mask>(unwrap(op.b), state));
1625  }
1626 
1628  Expr make(MatcherState &state, halide_type_t type_hint) const {
1629  const int l = known_lanes ? lanes : type_hint.lanes;
1630  type_hint.lanes = 1;
1631  Expr ea, eb;
1632  if (std::is_same<A, Const>::value) {
1633  eb = b.make(state, type_hint);
1634  ea = a.make(state, eb.type());
1635  } else {
1636  ea = a.make(state, type_hint);
1637  eb = b.make(state, ea.type());
1638  }
1639  return Ramp::make(ea, eb, l);
1640  }
1641 
1642  constexpr static bool foldable = false;
1643 };
1644 
1645 template<typename A, typename B>
1646 std::ostream &operator<<(std::ostream &s, const RampOp<A, B, true> &op) {
1647  s << "ramp(" << op.a << ", " << op.b << ", " << op.lanes << ")";
1648  return s;
1649 }
1650 
1651 template<typename A, typename B>
1652 std::ostream &operator<<(std::ostream &s, const RampOp<A, B, false> &op) {
1653  s << "ramp(" << op.a << ", " << op.b << ")";
1654  return s;
1655 }
1656 
1657 template<typename A, typename B>
1658 HALIDE_ALWAYS_INLINE auto ramp(A a, B b, int lanes) noexcept -> RampOp<decltype(pattern_arg(a)), decltype(pattern_arg(b)), true> {
1659  return {pattern_arg(a), pattern_arg(b), lanes};
1660 }
1661 
1662 template<typename A, typename B>
1663 HALIDE_ALWAYS_INLINE auto ramp(A a, B b) noexcept -> RampOp<decltype(pattern_arg(a)), decltype(pattern_arg(b)), false> {
1664  return {pattern_arg(a), pattern_arg(b), 0};
1665 }
1666 
1667 template<typename A, bool known_lanes, VectorReduce::Operator reduce_op>
1669  struct pattern_tag {};
1670  A a;
1671  int lanes;
1672 
1673  constexpr static uint32_t binds = bindings<A>::mask;
1674 
1677  constexpr static bool canonical = A::canonical;
1678 
1679  template<uint32_t bound>
1680  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
1681  if (e.node_type == VectorReduce::_node_type) {
1682  const VectorReduce &op = (const VectorReduce &)e;
1683  if (op.op == reduce_op &&
1684  (!known_lanes || lanes == op.type.lanes()) &&
1685  a.template match<bound>(*op.value.get(), state)) {
1686  return true;
1687  }
1688  }
1689  return false;
1690  }
1691 
1692  template<uint32_t bound, typename A2, bool known_lanes_2, VectorReduce::Operator reduce_op_2>
1694  return (reduce_op == reduce_op_2 &&
1695  a.template match<bound>(unwrap(op.a), state) &&
1696  (lanes == op.lanes || !known_lanes || !known_lanes_2));
1697  }
1698 
1700  Expr make(MatcherState &state, halide_type_t type_hint) const {
1701  const int l = known_lanes ? lanes : type_hint.lanes;
1702  return VectorReduce::make(reduce_op, a.make(state, type_hint), l);
1703  }
1704 
1705  constexpr static bool foldable = false;
1706 };
1707 
1708 template<typename A, VectorReduce::Operator reduce_op>
1709 inline std::ostream &operator<<(std::ostream &s, const VectorReduceOp<A, true, reduce_op> &op) {
1710  s << "vector_reduce(" << reduce_op << ", " << op.a << ", " << op.lanes << ")";
1711  return s;
1712 }
1713 
1714 template<typename A, VectorReduce::Operator reduce_op>
1715 inline std::ostream &operator<<(std::ostream &s, const VectorReduceOp<A, false, reduce_op> &op) {
1716  s << "vector_reduce(" << reduce_op << ", " << op.a << ")";
1717  return s;
1718 }
1719 
1720 template<typename A>
1721 HALIDE_ALWAYS_INLINE auto h_add(A a, int lanes) noexcept -> VectorReduceOp<decltype(pattern_arg(a)), true, VectorReduce::Add> {
1722  return {pattern_arg(a), lanes};
1723 }
1724 
1725 template<typename A>
1726 HALIDE_ALWAYS_INLINE auto h_add(A a) noexcept -> VectorReduceOp<decltype(pattern_arg(a)), false, VectorReduce::Add> {
1727  return {pattern_arg(a), 0};
1728 }
1729 
1730 template<typename A>
1731 HALIDE_ALWAYS_INLINE auto h_min(A a, int lanes) noexcept -> VectorReduceOp<decltype(pattern_arg(a)), true, VectorReduce::Min> {
1732  return {pattern_arg(a), lanes};
1733 }
1734 
1735 template<typename A>
1736 HALIDE_ALWAYS_INLINE auto h_min(A a) noexcept -> VectorReduceOp<decltype(pattern_arg(a)), false, VectorReduce::Min> {
1737  return {pattern_arg(a), 0};
1738 }
1739 
1740 template<typename A>
1741 HALIDE_ALWAYS_INLINE auto h_max(A a, int lanes) noexcept -> VectorReduceOp<decltype(pattern_arg(a)), true, VectorReduce::Max> {
1742  return {pattern_arg(a), lanes};
1743 }
1744 
1745 template<typename A>
1746 HALIDE_ALWAYS_INLINE auto h_max(A a) noexcept -> VectorReduceOp<decltype(pattern_arg(a)), false, VectorReduce::Max> {
1747  return {pattern_arg(a), 0};
1748 }
1749 
1750 template<typename A>
1751 HALIDE_ALWAYS_INLINE auto h_and(A a, int lanes) noexcept -> VectorReduceOp<decltype(pattern_arg(a)), true, VectorReduce::And> {
1752  return {pattern_arg(a), lanes};
1753 }
1754 
1755 template<typename A>
1756 HALIDE_ALWAYS_INLINE auto h_and(A a) noexcept -> VectorReduceOp<decltype(pattern_arg(a)), false, VectorReduce::And> {
1757  return {pattern_arg(a), 0};
1758 }
1759 
1760 template<typename A>
1761 HALIDE_ALWAYS_INLINE auto h_or(A a, int lanes) noexcept -> VectorReduceOp<decltype(pattern_arg(a)), true, VectorReduce::Or> {
1762  return {pattern_arg(a), lanes};
1763 }
1764 
1765 template<typename A>
1766 HALIDE_ALWAYS_INLINE auto h_or(A a) noexcept -> VectorReduceOp<decltype(pattern_arg(a)), false, VectorReduce::Or> {
1767  return {pattern_arg(a), 0};
1768 }
1769 
1770 template<typename A>
1771 struct NegateOp {
1772  struct pattern_tag {};
1773  A a;
1774 
1775  constexpr static uint32_t binds = bindings<A>::mask;
1776 
1779 
1780  constexpr static bool canonical = A::canonical;
1781 
1782  template<uint32_t bound>
1783  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
1784  if (e.node_type != Sub::_node_type) {
1785  return false;
1786  }
1787  const Sub &op = (const Sub &)e;
1788  return (a.template match<bound>(*op.b.get(), state) &&
1789  is_zero(op.a));
1790  }
1791 
1792  template<uint32_t bound, typename A2>
1793  HALIDE_ALWAYS_INLINE bool match(NegateOp<A2> &&p, MatcherState &state) const noexcept {
1794  return a.template match<bound>(unwrap(p.a), state);
1795  }
1796 
1798  Expr make(MatcherState &state, halide_type_t type_hint) const {
1799  Expr ea = a.make(state, type_hint);
1800  Expr z = make_zero(ea.type());
1801  return Sub::make(std::move(z), std::move(ea));
1802  }
1803 
1804  constexpr static bool foldable = A::foldable;
1805 
1806  template<typename A1 = A>
1808  a.make_folded_const(val, ty, state);
1809  int dead_bits = 64 - ty.bits;
1810  switch (ty.code) {
1811  case halide_type_int:
1812  if (ty.bits >= 32 && val.u.u64 && (val.u.u64 << (65 - ty.bits)) == 0) {
1813  // Trying to negate the most negative signed int for a no-overflow type.
1815  } else {
1816  // Negate, drop the high bits, and then sign-extend them back
1817  val.u.i64 = int64_t(uint64_t(-val.u.i64) << dead_bits) >> dead_bits;
1818  }
1819  break;
1820  case halide_type_uint:
1821  val.u.u64 = ((-val.u.u64) << dead_bits) >> dead_bits;
1822  break;
1823  case halide_type_float:
1824  case halide_type_bfloat:
1825  val.u.f64 = -val.u.f64;
1826  break;
1827  default:
1828  // unreachable
1829  ;
1830  }
1831  }
1832 };
1833 
1834 template<typename A>
1835 std::ostream &operator<<(std::ostream &s, const NegateOp<A> &op) {
1836  s << "-" << op.a;
1837  return s;
1838 }
1839 
1840 template<typename A>
1841 HALIDE_ALWAYS_INLINE auto operator-(A a) noexcept -> NegateOp<decltype(pattern_arg(a))> {
1842  return {pattern_arg(a)};
1843 }
1844 
1845 template<typename A>
1846 HALIDE_ALWAYS_INLINE auto negate(A a) -> decltype(IRMatcher::operator-(a)) {
1847  return IRMatcher::operator-(a);
1848 }
1849 
1850 template<typename A>
1851 struct CastOp {
1852  struct pattern_tag {};
1854  A a;
1855 
1856  constexpr static uint32_t binds = bindings<A>::mask;
1857 
1860  constexpr static bool canonical = A::canonical;
1861 
1862  template<uint32_t bound>
1863  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
1864  if (e.node_type != Cast::_node_type) {
1865  return false;
1866  }
1867  const Cast &op = (const Cast &)e;
1868  return (e.type == t &&
1869  a.template match<bound>(*op.value.get(), state));
1870  }
1871  template<uint32_t bound, typename A2>
1872  HALIDE_ALWAYS_INLINE bool match(const CastOp<A2> &op, MatcherState &state) const noexcept {
1873  return t == op.t && a.template match<bound>(unwrap(op.a), state);
1874  }
1875 
1877  Expr make(MatcherState &state, halide_type_t type_hint) const {
1878  return cast(t, a.make(state, {}));
1879  }
1880 
1881  constexpr static bool foldable = false; // TODO
1882 };
1883 
1884 template<typename A>
1885 std::ostream &operator<<(std::ostream &s, const CastOp<A> &op) {
1886  s << "cast(" << op.t << ", " << op.a << ")";
1887  return s;
1888 }
1889 
1890 template<typename A>
1891 HALIDE_ALWAYS_INLINE auto cast(halide_type_t t, A a) noexcept -> CastOp<decltype(pattern_arg(a))> {
1892  return {t, pattern_arg(a)};
1893 }
1894 
1895 template<typename A>
1896 struct Fold {
1897  struct pattern_tag {};
1898  A a;
1899 
1900  constexpr static uint32_t binds = bindings<A>::mask;
1901 
1904  constexpr static bool canonical = true;
1905 
1907  Expr make(MatcherState &state, halide_type_t type_hint) const noexcept {
1909  halide_type_t ty = type_hint;
1910  a.make_folded_const(c, ty, state);
1911  return make_const_expr(c, ty);
1912  }
1913 
1914  constexpr static bool foldable = A::foldable;
1915 
1916  template<typename A1 = A>
1918  a.make_folded_const(val, ty, state);
1919  }
1920 };
1921 
1922 template<typename A>
1923 HALIDE_ALWAYS_INLINE auto fold(A a) noexcept -> Fold<decltype(pattern_arg(a))> {
1924  return {pattern_arg(a)};
1925 }
1926 
1927 template<typename A>
1928 std::ostream &operator<<(std::ostream &s, const Fold<A> &op) {
1929  s << "fold(" << op.a << ")";
1930  return s;
1931 }
1932 
1933 template<typename A>
1934 struct Overflows {
1935  struct pattern_tag {};
1936  A a;
1937 
1938  constexpr static uint32_t binds = bindings<A>::mask;
1939 
1940  // This rule is a predicate, so it always evaluates to a boolean,
1941  // which has IRNodeType UIntImm
1944  constexpr static bool canonical = true;
1945 
1946  constexpr static bool foldable = A::foldable;
1947 
1948  template<typename A1 = A>
1950  a.make_folded_const(val, ty, state);
1951  ty.code = halide_type_uint;
1952  ty.bits = 64;
1953  val.u.u64 = (ty.lanes & MatcherState::special_values_mask) != 0;
1954  ty.lanes = 1;
1955  }
1956 };
1957 
1958 template<typename A>
1959 HALIDE_ALWAYS_INLINE auto overflows(A a) noexcept -> Overflows<decltype(pattern_arg(a))> {
1960  return {pattern_arg(a)};
1961 }
1962 
1963 template<typename A>
1964 std::ostream &operator<<(std::ostream &s, const Overflows<A> &op) {
1965  s << "overflows(" << op.a << ")";
1966  return s;
1967 }
1968 
1969 struct Overflow {
1970  struct pattern_tag {};
1971 
1972  constexpr static uint32_t binds = 0;
1973 
1974  // Overflow is an intrinsic, represented as a Call node
1977  constexpr static bool canonical = true;
1978 
1979  template<uint32_t bound>
1980  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
1981  if (e.node_type != Call::_node_type) {
1982  return false;
1983  }
1984  const Call &op = (const Call &)e;
1986  }
1987 
1989  Expr make(MatcherState &state, halide_type_t type_hint) const {
1991  return make_const_special_expr(type_hint);
1992  }
1993 
1994  constexpr static bool foldable = true;
1995 
1997  void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept {
1998  val.u.u64 = 0;
2000  }
2001 };
2002 
2003 inline std::ostream &operator<<(std::ostream &s, const Overflow &op) {
2004  s << "overflow()";
2005  return s;
2006 }
2007 
2008 template<typename A>
2009 struct IsConst {
2010  struct pattern_tag {};
2011 
2012  constexpr static uint32_t binds = bindings<A>::mask;
2013 
2014  // This rule is a boolean-valued predicate. Bools have type UIntImm.
2017  constexpr static bool canonical = true;
2018 
2019  A a;
2020 
2021  constexpr static bool foldable = true;
2022 
2023  template<typename A1 = A>
2025  Expr e = a.make(state, {});
2026  ty.code = halide_type_uint;
2027  ty.bits = 64;
2028  ty.lanes = 1;
2029  val.u.u64 = is_const(e) ? 1 : 0;
2030  }
2031 };
2032 
2033 template<typename A>
2034 HALIDE_ALWAYS_INLINE auto is_const(A a) noexcept -> IsConst<decltype(pattern_arg(a))> {
2035  return {pattern_arg(a)};
2036 }
2037 
2038 template<typename A>
2039 std::ostream &operator<<(std::ostream &s, const IsConst<A> &op) {
2040  s << "is_const(" << op.a << ")";
2041  return s;
2042 }
2043 
2044 template<typename A, typename Prover>
2045 struct CanProve {
2046  struct pattern_tag {};
2047  A a;
2048  Prover *prover; // An existing simplifying mutator
2049 
2050  constexpr static uint32_t binds = bindings<A>::mask;
2051 
2052  // This rule is a boolean-valued predicate. Bools have type UIntImm.
2055  constexpr static bool canonical = true;
2056 
2057  constexpr static bool foldable = true;
2058 
2059  // Includes a raw call to an inlined make method, so don't inline.
2061  Expr condition = a.make(state, {});
2062  condition = prover->mutate(condition, nullptr);
2063  val.u.u64 = is_one(condition);
2064  ty.code = halide_type_uint;
2065  ty.bits = 1;
2066  ty.lanes = condition.type().lanes();
2067  };
2068 };
2069 
2070 template<typename A, typename Prover>
2071 HALIDE_ALWAYS_INLINE auto can_prove(A a, Prover *p) noexcept -> CanProve<decltype(pattern_arg(a)), Prover> {
2072  return {pattern_arg(a), p};
2073 }
2074 
2075 template<typename A, typename Prover>
2076 std::ostream &operator<<(std::ostream &s, const CanProve<A, Prover> &op) {
2077  s << "can_prove(" << op.a << ")";
2078  return s;
2079 }
2080 
2081 template<typename A>
2082 struct IsFloat {
2083  struct pattern_tag {};
2084  A a;
2085 
2086  constexpr static uint32_t binds = bindings<A>::mask;
2087 
2088  // This rule is a boolean-valued predicate. Bools have type UIntImm.
2091  constexpr static bool canonical = true;
2092 
2093  constexpr static bool foldable = true;
2094 
2097  // a is almost certainly a very simple pattern (e.g. a wild), so just inline the make method.
2098  Type t = a.make(state, {}).type();
2099  val.u.u64 = t.is_float();
2100  ty.code = halide_type_uint;
2101  ty.bits = 1;
2102  ty.lanes = t.lanes();
2103  };
2104 };
2105 
2106 template<typename A>
2107 HALIDE_ALWAYS_INLINE auto is_float(A a) noexcept -> IsFloat<decltype(pattern_arg(a))> {
2108  return {pattern_arg(a)};
2109 }
2110 
2111 template<typename A>
2112 std::ostream &operator<<(std::ostream &s, const IsFloat<A> &op) {
2113  s << "is_float(" << op.a << ")";
2114  return s;
2115 }
2116 
2117 // Verify properties of each rewrite rule. Currently just fuzz tests them.
2118 template<typename Before,
2119  typename After,
2120  typename Predicate,
2121  typename = typename std::enable_if<std::decay<Before>::type::foldable &&
2122  std::decay<After>::type::foldable>::type>
2123 HALIDE_NEVER_INLINE void fuzz_test_rule(Before &&before, After &&after, Predicate &&pred,
2124  halide_type_t wildcard_type, halide_type_t output_type) noexcept {
2125 
2126  // We only validate the rules in the scalar case
2127  wildcard_type.lanes = output_type.lanes = 1;
2128 
2129  // Track which types this rule has been tested for before
2130  static std::set<uint32_t> tested;
2131 
2132  if (!tested.insert(reinterpret_bits<uint32_t>(wildcard_type)).second) return;
2133 
2134  // Print it in a form where it can be piped into a python/z3 validator
2135  debug(0) << "validate('" << before << "', '" << after << "', '" << pred << "', " << Type(wildcard_type) << ", " << Type(output_type) << ")\n";
2136 
2137  // Substitute some random constants into the before and after
2138  // expressions and see if the rule holds true. This should catch
2139  // silly errors, but not necessarily corner cases.
2140  static std::mt19937_64 rng(0);
2141  MatcherState state;
2142 
2143  Expr exprs[max_wild];
2144 
2145  for (int trials = 0; trials < 100; trials++) {
2146  // We want to test small constants more frequently than
2147  // large ones, otherwise we'll just get coverage of
2148  // overflow rules.
2149  int shift = (int)(rng() & (wildcard_type.bits - 1));
2150 
2151  for (int i = 0; i < max_wild; i++) {
2152  // Bind all the exprs and constants
2153  switch (wildcard_type.code) {
2154  case halide_type_uint: {
2155  // Normalize to the type's range by adding zero
2156  uint64_t val = constant_fold_bin_op<Add>(wildcard_type, (uint64_t)rng() >> shift, 0);
2157  state.set_bound_const(i, val, wildcard_type);
2158  val = constant_fold_bin_op<Add>(wildcard_type, (uint64_t)rng() >> shift, 0);
2159  exprs[i] = make_const(wildcard_type, val);
2160  state.set_binding(i, *exprs[i].get());
2161  } break;
2162  case halide_type_int: {
2163  int64_t val = constant_fold_bin_op<Add>(wildcard_type, (int64_t)rng() >> shift, 0);
2164  state.set_bound_const(i, val, wildcard_type);
2165  val = constant_fold_bin_op<Add>(wildcard_type, (int64_t)rng() >> shift, 0);
2166  exprs[i] = make_const(wildcard_type, val);
2167  } break;
2168  case halide_type_float:
2169  case halide_type_bfloat: {
2170  // Use a very narrow range of precise floats, so
2171  // that none of the rules a human is likely to
2172  // write have instabilities.
2173  double val = ((int64_t)(rng() & 15) - 8) / 2.0;
2174  state.set_bound_const(i, val, wildcard_type);
2175  val = ((int64_t)(rng() & 15) - 8) / 2.0;
2176  exprs[i] = make_const(wildcard_type, val);
2177  } break;
2178  default:
2179  return; // Don't care about handles
2180  }
2181  state.set_binding(i, *exprs[i].get());
2182  }
2183 
2184  halide_scalar_value_t val_pred, val_before, val_after;
2185  halide_type_t type = output_type;
2186  if (!evaluate_predicate(pred, state)) continue;
2187  before.make_folded_const(val_before, type, state);
2188  uint16_t lanes = type.lanes;
2189  after.make_folded_const(val_after, type, state);
2190  lanes |= type.lanes;
2191 
2192  if (lanes & MatcherState::special_values_mask) continue;
2193 
2194  bool ok = true;
2195  switch (output_type.code) {
2196  case halide_type_uint:
2197  // Compare normalized representations
2198  ok &= (constant_fold_bin_op<Add>(output_type, val_before.u.u64, 0) ==
2199  constant_fold_bin_op<Add>(output_type, val_after.u.u64, 0));
2200  break;
2201  case halide_type_int:
2202  ok &= (constant_fold_bin_op<Add>(output_type, val_before.u.i64, 0) ==
2203  constant_fold_bin_op<Add>(output_type, val_after.u.i64, 0));
2204  break;
2205  case halide_type_float:
2206  case halide_type_bfloat: {
2207  double error = std::abs(val_before.u.f64 - val_after.u.f64);
2208  // We accept an equal bit pattern (e.g. inf vs inf),
2209  // a small floating point difference, or turning a nan into not-a-nan.
2210  ok &= (error < 0.01 ||
2211  val_before.u.u64 == val_after.u.u64 ||
2212  std::isnan(val_before.u.f64));
2213  break;
2214  }
2215  default:
2216  return;
2217  }
2218 
2219  if (!ok) {
2220  debug(0) << "Fails with values:\n";
2221  for (int i = 0; i < max_wild; i++) {
2223  state.get_bound_const(i, val, wildcard_type);
2224  debug(0) << " c" << i << ": " << make_const_expr(val, wildcard_type) << "\n";
2225  }
2226  for (int i = 0; i < max_wild; i++) {
2227  debug(0) << " _" << i << ": " << Expr(state.get_binding(i)) << "\n";
2228  }
2229  debug(0) << " Before: " << make_const_expr(val_before, output_type) << "\n";
2230  debug(0) << " After: " << make_const_expr(val_after, output_type) << "\n";
2231  debug(0) << val_before.u.u64 << " " << val_after.u.u64 << "\n";
2233  }
2234  }
2235 }
2236 
2237 template<typename Before,
2238  typename After,
2239  typename Predicate,
2240  typename = typename std::enable_if<!(std::decay<Before>::type::foldable &&
2241  std::decay<After>::type::foldable)>::type>
2242 HALIDE_ALWAYS_INLINE void fuzz_test_rule(Before &&before, After &&after, Predicate &&pred,
2243  halide_type_t, halide_type_t, int dummy = 0) noexcept {
2244  // We can't verify rewrite rules that can't be constant-folded.
2245 }
2246 
2248 bool evaluate_predicate(bool x, MatcherState &) noexcept {
2249  return x;
2250 }
2251 
2252 template<typename Pattern,
2253  typename = typename enable_if_pattern<Pattern>::type>
2256  halide_type_t ty = halide_type_of<bool>();
2257  p.make_folded_const(c, ty, state);
2258  // Overflow counts as a failed predicate
2259  return (c.u.u64 != 0) && ((ty.lanes & MatcherState::special_values_mask) == 0);
2260 }
2261 
2262 // #defines for testing
2263 
2264 // Print all successful or failed matches
2265 #define HALIDE_DEBUG_MATCHED_RULES 0
2266 #define HALIDE_DEBUG_UNMATCHED_RULES 0
2267 
2268 // Set to true if you want to fuzz test every rewrite passed to
2269 // operator() to ensure the input and the output have the same value
2270 // for lots of random values of the wildcards. Run
2271 // correctness_simplify with this on.
2272 #define HALIDE_FUZZ_TEST_RULES 0
2273 
2274 template<typename Instance>
2275 struct Rewriter {
2276  Instance instance;
2280  bool validate;
2281 
2284  : instance(std::forward<Instance>(instance)), output_type(ot), wildcard_type(wt) {
2285  }
2286 
2287  template<typename After>
2289  result = after.make(state, output_type);
2290  }
2291 
2292  template<typename Before,
2293  typename After,
2294  typename = typename enable_if_pattern<Before>::type,
2295  typename = typename enable_if_pattern<After>::type>
2296  HALIDE_ALWAYS_INLINE bool operator()(Before before, After after) {
2297  static_assert((Before::binds & After::binds) == After::binds, "Rule result uses unbound values");
2298  static_assert(Before::canonical, "LHS of rewrite rule should be in canonical form");
2299  static_assert(After::canonical, "RHS of rewrite rule should be in canonical form");
2300 #if HALIDE_FUZZ_TEST_RULES
2301  fuzz_test_rule(before, after, true, wildcard_type, output_type);
2302 #endif
2303  if (before.template match<0>(instance, state)) {
2304  build_replacement(after);
2305 #if HALIDE_DEBUG_MATCHED_RULES
2306  debug(0) << instance << " -> " << result << " via " << before << " -> " << after << "\n";
2307 #endif
2308  return true;
2309  } else {
2310 #if HALIDE_DEBUG_UNMATCHED_RULES
2311  debug(0) << instance << " does not match " << before << "\n";
2312 #endif
2313  return false;
2314  }
2315  }
2316 
2317  template<typename Before,
2318  typename = typename enable_if_pattern<Before>::type>
2319  HALIDE_ALWAYS_INLINE bool operator()(Before before, const Expr &after) noexcept {
2320  static_assert(Before::canonical, "LHS of rewrite rule should be in canonical form");
2321  if (before.template match<0>(instance, state)) {
2322  result = after;
2323 #if HALIDE_DEBUG_MATCHED_RULES
2324  debug(0) << instance << " -> " << result << " via " << before << " -> " << after << "\n";
2325 #endif
2326  return true;
2327  } else {
2328 #if HALIDE_DEBUG_UNMATCHED_RULES
2329  debug(0) << instance << " does not match " << before << "\n";
2330 #endif
2331  return false;
2332  }
2333  }
2334 
2335  template<typename Before,
2336  typename = typename enable_if_pattern<Before>::type>
2337  HALIDE_ALWAYS_INLINE bool operator()(Before before, int64_t after) noexcept {
2338  static_assert(Before::canonical, "LHS of rewrite rule should be in canonical form");
2339 #if HALIDE_FUZZ_TEST_RULES
2340  fuzz_test_rule(before, Const(after), true, wildcard_type, output_type);
2341 #endif
2342  if (before.template match<0>(instance, state)) {
2343  result = make_const(output_type, after);
2344 #if HALIDE_DEBUG_MATCHED_RULES
2345  debug(0) << instance << " -> " << result << " via " << before << " -> " << after << "\n";
2346 #endif
2347  return true;
2348  } else {
2349 #if HALIDE_DEBUG_UNMATCHED_RULES
2350  debug(0) << instance << " does not match " << before << "\n";
2351 #endif
2352  return false;
2353  }
2354  }
2355 
2356  template<typename Before,
2357  typename After,
2358  typename Predicate,
2359  typename = typename enable_if_pattern<Before>::type,
2360  typename = typename enable_if_pattern<After>::type,
2361  typename = typename enable_if_pattern<Predicate>::type>
2362  HALIDE_ALWAYS_INLINE bool operator()(Before before, After after, Predicate pred) {
2363  static_assert(Predicate::foldable, "Predicates must consist only of operations that can constant-fold");
2364  static_assert((Before::binds & After::binds) == After::binds, "Rule result uses unbound values");
2365  static_assert((Before::binds & Predicate::binds) == Predicate::binds, "Rule predicate uses unbound values");
2366  static_assert(Before::canonical, "LHS of rewrite rule should be in canonical form");
2367  static_assert(After::canonical, "RHS of rewrite rule should be in canonical form");
2368 
2369 #if HALIDE_FUZZ_TEST_RULES
2370  fuzz_test_rule(before, after, pred, wildcard_type, output_type);
2371 #endif
2372  if (before.template match<0>(instance, state) &&
2373  evaluate_predicate(pred, state)) {
2374  build_replacement(after);
2375 #if HALIDE_DEBUG_MATCHED_RULES
2376  debug(0) << instance << " -> " << result << " via " << before << " -> " << after << " when " << pred << "\n";
2377 #endif
2378  return true;
2379  } else {
2380 #if HALIDE_DEBUG_UNMATCHED_RULES
2381  debug(0) << instance << " does not match " << before << "\n";
2382 #endif
2383  return false;
2384  }
2385  }
2386 
2387  template<typename Before,
2388  typename Predicate,
2389  typename = typename enable_if_pattern<Before>::type,
2390  typename = typename enable_if_pattern<Predicate>::type>
2391  HALIDE_ALWAYS_INLINE bool operator()(Before before, const Expr &after, Predicate pred) {
2392  static_assert(Predicate::foldable, "Predicates must consist only of operations that can constant-fold");
2393  static_assert(Before::canonical, "LHS of rewrite rule should be in canonical form");
2394  if (before.template match<0>(instance, state) &&
2395  evaluate_predicate(pred, state)) {
2396  result = after;
2397 #if HALIDE_DEBUG_MATCHED_RULES
2398  debug(0) << instance << " -> " << result << " via " << before << " -> " << after << " when " << pred << "\n";
2399 #endif
2400  return true;
2401  } else {
2402 #if HALIDE_DEBUG_UNMATCHED_RULES
2403  debug(0) << instance << " does not match " << before << "\n";
2404 #endif
2405  return false;
2406  }
2407  }
2408 
2409  template<typename Before,
2410  typename Predicate,
2411  typename = typename enable_if_pattern<Before>::type,
2412  typename = typename enable_if_pattern<Predicate>::type>
2413  HALIDE_ALWAYS_INLINE bool operator()(Before before, int64_t after, Predicate pred) {
2414  static_assert(Predicate::foldable, "Predicates must consist only of operations that can constant-fold");
2415  static_assert(Before::canonical, "LHS of rewrite rule should be in canonical form");
2416 #if HALIDE_FUZZ_TEST_RULES
2417  fuzz_test_rule(before, Const(after), pred, wildcard_type, output_type);
2418 #endif
2419  if (before.template match<0>(instance, state) &&
2420  evaluate_predicate(pred, state)) {
2421  result = make_const(output_type, after);
2422 #if HALIDE_DEBUG_MATCHED_RULES
2423  debug(0) << instance << " -> " << result << " via " << before << " -> " << after << " when " << pred << "\n";
2424 #endif
2425  return true;
2426  } else {
2427 #if HALIDE_DEBUG_UNMATCHED_RULES
2428  debug(0) << instance << " does not match " << before << "\n";
2429 #endif
2430  return false;
2431  }
2432  }
2433 };
2434 
2435 /** Construct a rewriter for the given instance, which may be a pattern
2436  * with concrete expressions as leaves, or just an expression. The
2437  * second optional argument (wildcard_type) is a hint as to what the
2438  * type of the wildcards is likely to be. If omitted it uses the same
2439  * type as the expression itself. They are not required to be this
2440  * type, but the rule will only be tested for wildcards of that type
2441  * when testing is enabled.
2442  *
2443  * The rewriter can be used to check to see if the instance is one of
2444  * some number of patterns and if so rewrite it into another form,
2445  * using its operator() method. See Simplify.cpp for a bunch of
2446  * example usage.
2447  */
2448 // @{
2449 template<typename Instance,
2450  typename = typename enable_if_pattern<Instance>::type>
2451 HALIDE_ALWAYS_INLINE auto rewriter(Instance instance, halide_type_t output_type, halide_type_t wildcard_type) noexcept -> Rewriter<decltype(pattern_arg(instance))> {
2452  return {pattern_arg(instance), output_type, wildcard_type};
2453 }
2454 
2455 template<typename Instance,
2456  typename = typename enable_if_pattern<Instance>::type>
2457 HALIDE_ALWAYS_INLINE auto rewriter(Instance instance, halide_type_t output_type) noexcept -> Rewriter<decltype(pattern_arg(instance))> {
2458  return {pattern_arg(instance), output_type, output_type};
2459 }
2460 
2462 auto rewriter(const Expr &e, halide_type_t wildcard_type) noexcept -> Rewriter<decltype(pattern_arg(e))> {
2463  return {pattern_arg(e), e.type(), wildcard_type};
2464 }
2465 
2467 auto rewriter(const Expr &e) noexcept -> Rewriter<decltype(pattern_arg(e))> {
2468  return {pattern_arg(e), e.type(), e.type()};
2469 }
2470 // @}
2471 
2472 } // namespace IRMatcher
2473 
2474 } // namespace Internal
2475 } // namespace Halide
2476 
2477 #endif
Halide::Internal::IRMatcher::Overflow::pattern_tag
Definition: IRMatch.h:1970
Halide::Internal::IRMatcher::h_and
HALIDE_ALWAYS_INLINE auto h_and(A a, int lanes) noexcept -> VectorReduceOp< decltype(pattern_arg(a)), true, VectorReduce::And >
Definition: IRMatch.h:1751
Halide::Internal::IRMatcher::Intrin::binds
static constexpr uint32_t binds
Definition: IRMatch.h:1317
Halide::Internal::IRMatcher::h_add
HALIDE_ALWAYS_INLINE auto h_add(A a, int lanes) noexcept -> VectorReduceOp< decltype(pattern_arg(a)), true, VectorReduce::Add >
Definition: IRMatch.h:1721
Halide::Internal::IRMatcher::VectorReduceOp::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:1700
Halide::Internal::IRMatcher::WildConst::foldable
constexpr static bool foldable
Definition: IRMatch.h:424
Halide::Internal::Cast::value
Expr value
Definition: IR.h:30
Halide::Internal::IRMatcher::Const::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition: IRMatch.h:543
Halide::Internal::IRMatcher::Intrin::pattern_tag
Definition: IRMatch.h:1313
Halide::Internal::IRMatcher::WildConstUInt::binds
constexpr static uint32_t binds
Definition: IRMatch.h:284
Halide::Internal::IRMatcher::cast
HALIDE_ALWAYS_INLINE auto cast(halide_type_t t, A a) noexcept -> CastOp< decltype(pattern_arg(a))>
Definition: IRMatch.h:1891
Halide::Internal::Broadcast::value
Expr value
Definition: IR.h:242
Halide::Internal::Call::signed_integer_overflow
@ signed_integer_overflow
Definition: IR.h:544
Halide::Internal::IRMatcher::SelectOp::pattern_tag
Definition: IRMatch.h:1455
Halide::Internal::IRMatcher::intrin
HALIDE_ALWAYS_INLINE auto intrin(Call::IntrinsicOp intrinsic_op, Args... args) noexcept -> Intrin< decltype(pattern_arg(args))... >
Definition: IRMatch.h:1393
Halide::Internal::Add
The sum of two expressions.
Definition: IR.h:38
Halide::Internal::IRNodeType::Cast
@ Cast
Halide::Internal::IRMatcher::constant_fold_bin_op< Div >
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< Div >(halide_type_t &t, int64_t a, int64_t b) noexcept
Definition: IRMatch.h:1012
Halide::Internal::IRMatcher::VectorReduceOp::foldable
constexpr static bool foldable
Definition: IRMatch.h:1705
Halide::Internal::IRMatcher::constant_fold_cmp_op< LE >
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op< LE >(int64_t a, int64_t b) noexcept
Definition: IRMatch.h:1152
Halide::Internal::IRMatcher::SelectOp::canonical
constexpr static bool canonical
Definition: IRMatch.h:1465
Halide::Internal::IRMatcher::can_prove
HALIDE_ALWAYS_INLINE auto can_prove(A a, Prover *p) noexcept -> CanProve< decltype(pattern_arg(a)), Prover >
Definition: IRMatch.h:2071
Halide::Internal::IRMatcher::NegateOp::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:1798
Halide::Internal::IRMatcher::Rewriter::validate
bool validate
Definition: IRMatch.h:2280
Halide::Internal::IRMatcher::WildConstUInt::foldable
constexpr static bool foldable
Definition: IRMatch.h:319
Halide::Internal::IRMatcher::bitwise_or_reduce
constexpr uint32_t bitwise_or_reduce()
Definition: IRMatch.h:1293
Halide::Internal::IRMatcher::ramp
HALIDE_ALWAYS_INLINE auto ramp(A a, B b, int lanes) noexcept -> RampOp< decltype(pattern_arg(a)), decltype(pattern_arg(b)), true >
Definition: IRMatch.h:1658
Halide::Internal::IRMatcher::MatcherState::signed_integer_overflow
static constexpr uint16_t signed_integer_overflow
Definition: IRMatch.h:81
Halide::Internal::IRMatcher::SpecificExpr::canonical
constexpr static bool canonical
Definition: IRMatch.h:205
Halide::Internal::IRMatcher::Const::match
HALIDE_ALWAYS_INLINE bool match(const Const &b, MatcherState &state) const noexcept
Definition: IRMatch.h:531
Halide::Internal::IRMatcher::Overflow::canonical
constexpr static bool canonical
Definition: IRMatch.h:1977
halide_scalar_value_t::u
union halide_scalar_value_t::@3 u
Halide::Internal::IRMatcher::BroadcastOp::lanes
int lanes
Definition: IRMatch.h:1520
Halide::Internal::IRMatcher::IsFloat::a
A a
Definition: IRMatch.h:2084
Halide::Internal::IRMatcher::WildConstInt::canonical
constexpr static bool canonical
Definition: IRMatch.h:235
Halide::Internal::VectorReduce::op
Operator op
Definition: IR.h:841
Halide::Internal::IRMatcher::Intrin::match_args
HALIDE_ALWAYS_INLINE bool match_args(double, const Call &c, MatcherState &state) const noexcept
Definition: IRMatch.h:1333
Halide::Internal::IRMatcher::Overflow::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:1975
Halide::Internal::IRMatcher::bindings
Definition: IRMatch.h:138
Halide::Internal::StrongestExprNodeType
constexpr IRNodeType StrongestExprNodeType
Definition: Expr.h:79
Halide::Internal::IRMatcher::WildConstInt
Definition: IRMatch.h:228
Halide::Internal::IRMatcher::WildConstInt::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const
Definition: IRMatch.h:269
Halide::Internal::IRMatcher::NotOp::canonical
constexpr static bool canonical
Definition: IRMatch.h:1406
Halide::Internal::IRMatcher::Rewriter::instance
Instance instance
Definition: IRMatch.h:2276
Halide::Internal::IRMatcher::SelectOp::f
F f
Definition: IRMatch.h:1458
Halide::Internal::IRMatcher::WildConstFloat::canonical
constexpr static bool canonical
Definition: IRMatch.h:341
Halide::Internal::IRMatcher::constant_fold_cmp_op< LT >
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op< LT >(int64_t a, int64_t b) noexcept
Definition: IRMatch.h:1102
Halide::Internal::IRMatcher::IsConst
Definition: IRMatch.h:2009
Halide::Internal::mod_imp
T mod_imp(T a, T b)
Implementations of division and mod that are specific to Halide.
Definition: IROperator.h:244
Halide::Internal::IRMatcher::CastOp::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:1863
Halide::Internal::IRMatcher::operator>=
HALIDE_ALWAYS_INLINE auto operator>=(A a, B b) noexcept -> CmpOp< GE, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1167
Halide::Internal::IRMatcher::SpecificExpr
Definition: IRMatch.h:197
halide_type_bfloat
@ halide_type_bfloat
floating point numbers in the bfloat format
Definition: HalideRuntime.h:408
Halide::Internal::IRMatcher::lt
HALIDE_ALWAYS_INLINE auto lt(A a, B b) -> decltype(IRMatcher::operator<(a, b))
Definition: IRMatch.h:1097
Halide::Internal::IRNodeType::Not
@ Not
Halide::Internal::IRMatcher::Overflow::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:1989
Halide::Internal::IRNodeType::Select
@ Select
Halide::Internal::IRMatcher::SpecificExpr::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:215
Halide::Internal::IRMatcher::CmpOp::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition: IRMatch.h:768
Halide::Internal::IRMatcher::VectorReduceOp
Definition: IRMatch.h:1668
Halide::Internal::IRMatcher::IsFloat::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const
Definition: IRMatch.h:2096
Halide::Internal::IRMatcher::CanProve::make_folded_const
HALIDE_NEVER_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const
Definition: IRMatch.h:2060
Halide::Internal::VectorReduce
Horizontally reduce a vector to a scalar or narrower vector using the given commutative and associati...
Definition: IR.h:827
uint16_t
unsigned __INT16_TYPE__ uint16_t
Definition: runtime_internal.h:23
Halide::Internal::IRMatcher::CanProve
Definition: IRMatch.h:2045
Halide::Internal::IRMatcher::Fold::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition: IRMatch.h:1917
Halide::Internal::Select::condition
Expr condition
Definition: IR.h:187
Halide::Internal::IRMatcher::IsFloat::foldable
constexpr static bool foldable
Definition: IRMatch.h:2093
Halide::Internal::BaseExprNode::type
Type type
Definition: Expr.h:146
Halide::Internal::sub_would_overflow
bool sub_would_overflow(int bits, int64_t a, int64_t b)
Halide::Internal::IRMatcher::IsConst::a
A a
Definition: IRMatch.h:2019
Halide::Internal::IRMatcher::Rewriter::operator()
HALIDE_ALWAYS_INLINE bool operator()(Before before, const Expr &after, Predicate pred)
Definition: IRMatch.h:2391
Halide::Internal::IRMatcher::CastOp::foldable
constexpr static bool foldable
Definition: IRMatch.h:1881
Halide::Internal::IRMatcher::Fold::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const noexcept
Definition: IRMatch.h:1907
Halide::Internal::IRMatcher::or_op
HALIDE_ALWAYS_INLINE auto or_op(A a, B b) -> decltype(IRMatcher::operator||(a, b))
Definition: IRMatch.h:1247
Halide::Internal::IRMatcher::CmpOp::a
A a
Definition: IRMatch.h:735
Halide::Internal::IRMatcher::make_const_expr
HALIDE_ALWAYS_INLINE Expr make_const_expr(halide_scalar_value_t val, halide_type_t ty)
Definition: IRMatch.h:153
Halide::Internal::VectorReduce::value
Expr value
Definition: IR.h:840
Halide::Internal::IRMatcher::MatcherState::set_bound_const
HALIDE_ALWAYS_INLINE void set_bound_const(int i, halide_scalar_value_t val, halide_type_t t) noexcept
Definition: IRMatch.h:115
Halide::Internal::IRMatcher::constant_fold_bin_op< Mul >
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< Mul >(halide_type_t &t, int64_t a, int64_t b) noexcept
Definition: IRMatch.h:983
Halide::Internal::IRMatcher::CastOp::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:1877
Halide::Internal::IRMatcher::NegateOp::a
A a
Definition: IRMatch.h:1773
Halide::Internal::IRMatcher::Wild::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition: IRMatch.h:465
Halide::Internal::GE
Is the first expression greater than or equal to the second.
Definition: IR.h:148
Halide::Internal::IRMatcher::MatcherState::MatcherState
HALIDE_ALWAYS_INLINE MatcherState() noexcept
Definition: IRMatch.h:127
Halide::Internal::IRMatcher::IsConst::foldable
constexpr static bool foldable
Definition: IRMatch.h:2021
halide_type_float
@ halide_type_float
IEEE floating point numbers.
Definition: HalideRuntime.h:406
Halide::Internal::IRMatcher::max_wild
constexpr int max_wild
Definition: IRMatch.h:70
Halide::Internal::IRMatcher::BroadcastOp::pattern_tag
Definition: IRMatch.h:1518
Halide::Internal::IRMatcher::le
HALIDE_ALWAYS_INLINE auto le(A a, B b) -> decltype(IRMatcher::operator<=(a, b))
Definition: IRMatch.h:1147
Halide::Internal::IRNodeType
IRNodeType
All our IR node types get unique IDs for the purposes of RTTI.
Definition: Expr.h:25
Halide::Internal::IRMatcher::fuzz_test_rule
HALIDE_NEVER_INLINE void fuzz_test_rule(Before &&before, After &&after, Predicate &&pred, halide_type_t wildcard_type, halide_type_t output_type) noexcept
Definition: IRMatch.h:2123
Halide::Internal::IRMatcher::IsConst::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:2015
Halide::Internal::IRNodeType::EQ
@ EQ
Halide::Internal::IRMatcher::IsFloat::canonical
constexpr static bool canonical
Definition: IRMatch.h:2091
Halide::Internal::IRMatcher::Overflow::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:1976
Halide::Internal::IRMatcher::Rewriter::operator()
HALIDE_ALWAYS_INLINE bool operator()(Before before, const Expr &after) noexcept
Definition: IRMatch.h:2319
Halide::Internal::IRMatcher::Intrin::canonical
constexpr static bool canonical
Definition: IRMatch.h:1321
Halide::Internal::IRMatcher::Fold::foldable
constexpr static bool foldable
Definition: IRMatch.h:1914
IROperator.h
Halide::Internal::IRMatcher::SelectOp::t
T t
Definition: IRMatch.h:1457
Halide::Internal::IRMatcher::BroadcastOp::a
A a
Definition: IRMatch.h:1519
Halide::Internal::IRMatcher::IsFloat::pattern_tag
Definition: IRMatch.h:2083
Halide::Internal::IRMatcher::CastOp::pattern_tag
Definition: IRMatch.h:1852
Halide::Internal::IRMatcher::constant_fold_bin_op
int64_t constant_fold_bin_op(halide_type_t &, int64_t, int64_t) noexcept
Halide::Internal::IRMatcher::Const::Const
HALIDE_ALWAYS_INLINE Const(int64_t v)
Definition: IRMatch.h:508
Halide::Internal::FloatImm
Floating point constants.
Definition: Expr.h:234
Halide::Internal::IRMatcher::SpecificExpr::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:204
Halide::Type::lanes
HALIDE_ALWAYS_INLINE int lanes() const
Return the number of vector elements in this type.
Definition: Type.h:337
Halide::Internal::IRMatcher::CanProve::pattern_tag
Definition: IRMatch.h:2046
Halide::Internal::IRMatcher::WildConstUInt::pattern_tag
Definition: IRMatch.h:282
Halide::Internal::IRNodeType::Or
@ Or
Halide::Internal::IRMatcher::WildConstFloat::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition: IRMatch.h:375
Halide::Internal::IRMatcher::Rewriter::output_type
halide_type_t output_type
Definition: IRMatch.h:2279
Halide::Internal::UIntImm::make
static const UIntImm * make(Type t, uint64_t value)
Halide::Internal::IRMatcher::CmpOp::binds
constexpr static uint32_t binds
Definition: IRMatch.h:738
Halide::Internal::IRMatcher::SpecificExpr::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:203
Halide::Internal::Broadcast
A vector with 'lanes' elements, in which every element is 'value'.
Definition: IR.h:241
Halide::Internal::Div
The ratio of two expressions.
Definition: IR.h:65
Halide::Type::is_float
HALIDE_ALWAYS_INLINE bool is_float() const
Is this type a floating point type (float or double).
Definition: Type.h:384
Halide::Internal::IRMatcher::commutative
constexpr bool commutative(IRNodeType t)
Definition: IRMatch.h:609
Halide::Internal::IRMatcher::BinOp::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:629
Halide::Internal::IRMatcher::CmpOp::b
B b
Definition: IRMatch.h:736
Halide::Internal::IRMatcher::SelectOp::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition: IRMatch.h:1492
Halide::Internal::IRMatcher::SelectOp
Definition: IRMatch.h:1454
Halide::Internal::IRMatcher::bindings::mask
constexpr static uint32_t mask
Definition: IRMatch.h:139
Halide::Internal::IntImm
Integer constants.
Definition: Expr.h:216
Halide::Internal::IRMatcher::BinOp::binds
constexpr static uint32_t binds
Definition: IRMatch.h:627
Halide::Internal::IRMatcher::SelectOp::binds
constexpr static uint32_t binds
Definition: IRMatch.h:1460
halide_type_t::bits
uint8_t bits
The number of bits of precision of a single scalar value of this type.
Definition: HalideRuntime.h:438
Halide::Internal::IRMatcher::RampOp::b
B b
Definition: IRMatch.h:1595
Halide::Internal::IRMatcher::negate
HALIDE_ALWAYS_INLINE auto negate(A a) -> decltype(IRMatcher::operator-(a))
Definition: IRMatch.h:1846
Halide::Internal::IRMatcher::not_op
HALIDE_ALWAYS_INLINE auto not_op(A a) -> decltype(IRMatcher::operator!(a))
Definition: IRMatch.h:1443
Halide::Internal::IRMatcher::Const::canonical
constexpr static bool canonical
Definition: IRMatch.h:505
Halide::Internal::IRMatcher::Intrin::match_args
HALIDE_ALWAYS_INLINE bool match_args(int, const Call &c, MatcherState &state) const noexcept
Definition: IRMatch.h:1326
Halide::Internal::IRMatcher::NotOp::match
HALIDE_ALWAYS_INLINE bool match(const NotOp< A2 > &op, MatcherState &state) const noexcept
Definition: IRMatch.h:1418
Halide::Internal::IRMatcher::Wild::pattern_tag
Definition: IRMatch.h:441
Halide::Internal::IRMatcher::WildConstInt::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:259
Halide::Internal::IRMatcher::Wild
Definition: IRMatch.h:440
Halide::Internal::IRMatcher::RampOp::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:1606
Halide::Internal::IRMatcher::WildConstInt::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:238
Halide::Internal::IRMatcher::MatcherState::set_binding
HALIDE_ALWAYS_INLINE void set_binding(int i, const BaseExprNode &n) noexcept
Definition: IRMatch.h:87
Halide::Internal::is_one
bool is_one(const Expr &e)
Is the expression a const (as defined by is_const), and also equal to one (in all lanes,...
Halide::Internal::IRMatcher::CastOp::binds
constexpr static uint32_t binds
Definition: IRMatch.h:1856
Halide::Internal::IRMatcher::operator<<
std::ostream & operator<<(std::ostream &s, const SpecificExpr &e)
Definition: IRMatch.h:222
Halide::Internal::IRNodeType::Min
@ Min
IR.h
Halide::Internal::IRMatcher::NegateOp::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:1777
Halide::Internal::IRMatcher::VectorReduceOp::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:1680
Halide::Internal::IRMatcher::Intrin::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:1320
Halide::Internal::IRMatcher::RampOp::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:1600
Halide::Internal::IRMatcher::Rewriter::operator()
HALIDE_ALWAYS_INLINE bool operator()(Before before, After after, Predicate pred)
Definition: IRMatch.h:2362
Halide::Internal::Cast
The actual IR nodes begin here.
Definition: IR.h:29
Halide::Internal::LE
Is the first expression less than or equal to the second.
Definition: IR.h:130
Halide::Internal::make_signed_integer_overflow
Expr make_signed_integer_overflow(Type type)
Construct a unique signed_integer_overflow Expr.
Halide::Internal::Sub::_node_type
static const IRNodeType _node_type
Definition: IR.h:52
halide_type_t
A runtime tag for a type in the halide type system.
Definition: HalideRuntime.h:426
Halide::Internal::IRNodeType::GE
@ GE
Halide::Internal::IRMatcher::SpecificExpr::pattern_tag
Definition: IRMatch.h:198
Halide::Internal::IRMatcher::equal_helper
bool equal_helper(const BaseExprNode &a, const BaseExprNode &b) noexcept
Halide::Internal::IRMatcher::Intrin::intrin
Call::IntrinsicOp intrin
Definition: IRMatch.h:1314
Halide::Internal::IRMatcher::CmpOp::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:749
Halide::Internal::NE
Is the first expression not equal to the second.
Definition: IR.h:112
Halide::Internal::IRMatcher::VectorReduceOp::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:1675
Halide::Internal::IRMatcher::Overflow::foldable
constexpr static bool foldable
Definition: IRMatch.h:1994
Halide::Internal::IRMatcher::overflows
HALIDE_ALWAYS_INLINE auto overflows(A a) noexcept -> Overflows< decltype(pattern_arg(a))>
Definition: IRMatch.h:1959
Halide::Internal::IRMatcher::WildConst::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition: IRMatch.h:427
Halide::Internal::IRMatcher::NegateOp::match
HALIDE_ALWAYS_INLINE bool match(NegateOp< A2 > &&p, MatcherState &state) const noexcept
Definition: IRMatch.h:1793
Halide::Internal::IRMatcher::IsFloat
Definition: IRMatch.h:2082
Halide::Internal::IRMatcher::CastOp::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:1858
Halide::Internal::IRMatcher::SelectOp::c
C c
Definition: IRMatch.h:1456
Halide::Internal::IRMatcher::BinOp
Definition: IRMatch.h:622
Halide::Internal::IRMatcher::Wild::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:445
Halide::Internal::IRMatcher::RampOp::pattern_tag
Definition: IRMatch.h:1593
Halide::Internal::IRMatcher::WildConstFloat::binds
constexpr static uint32_t binds
Definition: IRMatch.h:337
Halide::NameMangling::C
@ C
No name mangling.
Halide::Internal::IRMatcher::Fold::canonical
constexpr static bool canonical
Definition: IRMatch.h:1904
Halide::Internal::IRMatcher::WildConstUInt
Definition: IRMatch.h:281
Halide::Internal::IRMatcher::CmpOp::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:802
Halide::Internal::IRMatcher::Overflows::a
A a
Definition: IRMatch.h:1936
Halide::Internal::IRMatcher::Fold::pattern_tag
Definition: IRMatch.h:1897
uint64_t
unsigned __INT64_TYPE__ uint64_t
Definition: runtime_internal.h:19
Halide::Internal::IRMatcher::VectorReduceOp::binds
constexpr static uint32_t binds
Definition: IRMatch.h:1673
Halide::Internal::IRMatcher::constant_fold_cmp_op
uint64_t constant_fold_cmp_op(int64_t, int64_t) noexcept
Halide::Type::is_vector
HALIDE_ALWAYS_INLINE bool is_vector() const
Is this type a vector type? (lanes() != 1).
Definition: Type.h:371
Halide::Internal::IRMatcher::MatcherState::get_binding
const HALIDE_ALWAYS_INLINE BaseExprNode * get_binding(int i) const noexcept
Definition: IRMatch.h:92
Halide::Internal::IRMatcher::Rewriter::operator()
HALIDE_ALWAYS_INLINE bool operator()(Before before, After after)
Definition: IRMatch.h:2296
Halide::Type
Types in the halide type system.
Definition: Type.h:269
Halide::Internal::IRMatcher::SpecificExpr::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:210
HALIDE_NEVER_INLINE
#define HALIDE_NEVER_INLINE
Definition: HalideRuntime.h:33
Halide::Internal::IRMatcher::constant_fold_bin_op< Mod >
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< Mod >(halide_type_t &t, int64_t a, int64_t b) noexcept
Definition: IRMatch.h:1037
Halide::Internal::IRMatcher::CmpOp
Definition: IRMatch.h:733
Halide::Internal::IRMatcher::RampOp::foldable
constexpr static bool foldable
Definition: IRMatch.h:1642
Halide::Internal::Broadcast::make
static Expr make(Expr value, int lanes)
Halide::Internal::expr_match
bool expr_match(const Expr &pattern, const Expr &expr, std::vector< Expr > &result)
Does the first expression have the same structure as the second? Variables in the first expression wi...
Halide::Internal::add_would_overflow
bool add_would_overflow(int bits, int64_t a, int64_t b)
Routines to test if math would overflow for signed integers with the given number of bits.
Halide::Internal::IRMatcher::constant_fold_cmp_op< GT >
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op< GT >(int64_t a, int64_t b) noexcept
Definition: IRMatch.h:1127
Halide::Internal::Call::likely
@ likely
Definition: IR.h:525
Halide::Internal::IRMatcher::Overflows::foldable
constexpr static bool foldable
Definition: IRMatch.h:1946
Halide
This file defines the class FunctionDAG, which is our representation of a Halide pipeline,...
Definition: AddAtomicMutex.h:21
Halide::Internal::Or
Logical or - is at least one of the expression true.
Definition: IR.h:166
Halide::Internal::EQ
Is the first expression equal to the second.
Definition: IR.h:103
Halide::Internal::IRMatcher::mul
HALIDE_ALWAYS_INLINE auto mul(A a, B b) -> decltype(IRMatcher::operator*(a, b))
Definition: IRMatch.h:978
Halide::Internal::IRMatcher::IsFloat::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:2090
Halide::Internal::IRMatcher::NotOp::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:1404
Halide::Internal::IRMatcher::CastOp::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:1859
Halide::Internal::IRMatcher::WildConst::canonical
constexpr static bool canonical
Definition: IRMatch.h:395
Halide::Internal::IRMatcher::CanProve::a
A a
Definition: IRMatch.h:2047
Halide::Internal::IRMatcher::IsConst::canonical
constexpr static bool canonical
Definition: IRMatch.h:2017
Halide::Internal::IRMatcher::WildConstInt::pattern_tag
Definition: IRMatch.h:229
Halide::Internal::IRMatcher::WildConstFloat::foldable
constexpr static bool foldable
Definition: IRMatch.h:372
halide_scalar_value_t
halide_scalar_value_t is a simple union able to represent all the well-known scalar values in a filte...
Definition: HalideRuntime.h:1557
Halide::Internal::IRMatcher::WildConstFloat::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:365
Halide::Internal::IRMatcher::fold
HALIDE_ALWAYS_INLINE auto fold(A a) noexcept -> Fold< decltype(pattern_arg(a))>
Definition: IRMatch.h:1923
Halide::Internal::IRMatcher::Const::binds
constexpr static uint32_t binds
Definition: IRMatch.h:501
Halide::Internal::IRMatcher::IsFloat::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:2089
Halide::Internal::IRMatcher::pattern_arg
HALIDE_ALWAYS_INLINE T pattern_arg(T t)
Definition: IRMatch.h:567
Halide::Internal::IRMatcher::SelectOp::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:1463
Halide::LinkageType::Internal
@ Internal
Not visible externally, similar to 'static' linkage in C.
Halide::Internal::Max
The greater of two values.
Definition: IR.h:94
Halide::Internal::IRMatcher::WildConst::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:393
Halide::Internal::IRMatcher::Const::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:504
Halide::Internal::Sub::a
Expr a
Definition: IR.h:48
Halide::Internal::IRMatcher::operator*
HALIDE_ALWAYS_INLINE auto operator*(A a, B b) noexcept -> BinOp< Mul, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:973
Halide::Internal::IRMatcher::Const::pattern_tag
Definition: IRMatch.h:498
Halide::Internal::IRMatcher::Overflows::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:1942
Halide::Internal::IRMatcher::VectorReduceOp::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:1676
Halide::Internal::IRMatcher::WildConstFloat::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:340
Halide::Internal::make_zero
Expr make_zero(Type t)
Construct the representation of zero in the given type.
Halide::Internal::IRMatcher::make_const_special_expr
HALIDE_NEVER_INLINE Expr make_const_special_expr(halide_type_t ty)
Definition: IRMatch.h:142
Halide::Internal::IRMatcher::NegateOp::binds
constexpr static uint32_t binds
Definition: IRMatch.h:1775
Halide::Internal::IRMatcher::VectorReduceOp::canonical
constexpr static bool canonical
Definition: IRMatch.h:1677
Halide::Internal::VectorReduce::Min
@ Min
Definition: IR.h:834
Halide::Internal::IRMatcher::Const::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:503
Halide::Internal::IRMatcher::BinOp::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:630
Halide::Internal::IRMatcher::CastOp::a
A a
Definition: IRMatch.h:1854
Halide::Internal::IRMatcher::Fold::a
A a
Definition: IRMatch.h:1898
Halide::Internal::IRMatcher::NotOp
Definition: IRMatch.h:1398
Halide::Internal::IRNodeType::Broadcast
@ Broadcast
halide_scalar_value_t::i64
int64_t i64
Definition: HalideRuntime.h:1563
Halide::Internal::IRMatcher::WildConstFloat::pattern_tag
Definition: IRMatch.h:335
Halide::Internal::IRNodeType::Add
@ Add
Halide::Internal::VectorReduce::make
static Expr make(Operator op, Expr vec, int lanes)
Halide::Internal::IRMatcher::NotOp::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:1405
Halide::Internal::IRMatcher::operator<=
HALIDE_ALWAYS_INLINE auto operator<=(A a, B b) noexcept -> CmpOp< LE, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1142
Halide::Internal::IRMatcher::RampOp
Definition: IRMatch.h:1592
Halide::Internal::IRMatcher::WildConst::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:417
Halide::Internal::IRMatcher::operator!=
HALIDE_ALWAYS_INLINE auto operator!=(A a, B b) noexcept -> CmpOp< NE, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1217
Halide::Internal::IRMatcher::WildConstInt::binds
constexpr static uint32_t binds
Definition: IRMatch.h:231
Halide::Internal::Select::make
static Expr make(Expr condition, Expr true_value, Expr false_value)
Halide::Internal::IRMatcher::BinOp::foldable
constexpr static bool foldable
Definition: IRMatch.h:655
Halide::Internal::IRMatcher::SpecificExpr::binds
constexpr static uint32_t binds
Definition: IRMatch.h:200
Halide::Internal::IRMatcher::RampOp::lanes
int lanes
Definition: IRMatch.h:1596
Halide::Internal::IRMatcher::WildConstUInt::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:287
Halide::Internal::IRMatcher::RampOp::match
HALIDE_ALWAYS_INLINE bool match(const RampOp< A2, B2, known_lanes_2 > &op, MatcherState &state) const noexcept
Definition: IRMatch.h:1621
Halide::Internal::IRMatcher::BinOp::b
B b
Definition: IRMatch.h:625
Halide::Internal::IRMatcher::SpecificExpr::foldable
constexpr static bool foldable
Definition: IRMatch.h:219
Halide::Internal::IRMatcher::Const
Definition: IRMatch.h:497
Halide::Internal::IRMatcher::Wild::canonical
constexpr static bool canonical
Definition: IRMatch.h:447
Halide::Internal::IRNodeType::Sub
@ Sub
Halide::Internal::IRMatcher::Intrin
Definition: IRMatch.h:1312
Halide::Internal::IRMatcher::SelectOp::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:1485
Halide::Internal::VectorReduce::Max
@ Max
Definition: IR.h:835
Halide::Internal::IRNodeType::FloatImm
@ FloatImm
Halide::Internal::IRMatcher::mod
HALIDE_ALWAYS_INLINE auto mod(A a, B b) -> decltype(IRMatcher::operator%(a, b))
Definition: IRMatch.h:1032
Halide::Internal::Ramp
A linear ramp vector node.
Definition: IR.h:229
Halide::Internal::Ramp::make
static Expr make(Expr base, Expr stride, int lanes)
HALIDE_ALWAYS_INLINE
#define HALIDE_ALWAYS_INLINE
Definition: HalideRuntime.h:32
Halide::Internal::IRMatcher::and_reduce
constexpr bool and_reduce()
Definition: IRMatch.h:1302
Halide::Internal::Select::true_value
Expr true_value
Definition: IR.h:187
Halide::Internal::Broadcast::lanes
int lanes
Definition: IR.h:243
Halide::Expr::type
HALIDE_ALWAYS_INLINE Type type() const
Get the type of this expression node.
Definition: Expr.h:320
Halide::Internal::expr_match_test
void expr_match_test()
halide_type_t::lanes
uint16_t lanes
How many elements in a vector.
Definition: HalideRuntime.h:442
Halide::Internal::IRMatcher::BroadcastOp::binds
constexpr static uint32_t binds
Definition: IRMatch.h:1522
Halide::Internal::IRMatcher::VectorReduceOp::pattern_tag
Definition: IRMatch.h:1669
Halide::Internal::IRMatcher::Overflow
Definition: IRMatch.h:1969
Halide::Internal::IRMatcher::IsConst::pattern_tag
Definition: IRMatch.h:2010
Halide::Internal::IRNodeType::IntImm
@ IntImm
Halide::Internal::IRMatcher::NegateOp::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition: IRMatch.h:1807
Halide::Internal::IRMatcher::eq
HALIDE_ALWAYS_INLINE auto eq(A a, B b) -> decltype(IRMatcher::operator==(a, b))
Definition: IRMatch.h:1197
Halide::Internal::IRMatcher::BinOp::a
A a
Definition: IRMatch.h:624
Halide::Internal::IRMatcher::MatcherState::bound_const
halide_scalar_value_t bound_const[max_wild]
Definition: IRMatch.h:78
Halide::Internal::debug
For optional debugging during codegen, use the debug class as follows:
Definition: Debug.h:49
Halide::Internal::IRMatcher::MatcherState::set_bound_const
HALIDE_ALWAYS_INLINE void set_bound_const(int i, int64_t s, halide_type_t t) noexcept
Definition: IRMatch.h:97
Halide::Internal::div_imp
T div_imp(T a, T b)
Definition: IROperator.h:265
Halide::Internal::IRMatcher::BinOp::canonical
constexpr static bool canonical
Definition: IRMatch.h:635
Halide::Internal::IRMatcher::Intrin::print_args
HALIDE_ALWAYS_INLINE void print_args(int, std::ostream &s) const
Definition: IRMatch.h:1348
Halide::Internal::Sub::b
Expr b
Definition: IR.h:48
Halide::Internal::IRMatcher::CmpOp::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:741
Halide::Internal::IRMatcher::min
HALIDE_ALWAYS_INLINE auto min(A a, B b) noexcept -> BinOp< Min, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1052
Halide::Internal::IRMatcher::NegateOp::pattern_tag
Definition: IRMatch.h:1772
Halide::Internal::IRNodeType::And
@ And
int64_t
signed __INT64_TYPE__ int64_t
Definition: runtime_internal.h:18
Halide::Internal::IRMatcher::BroadcastOp::match
HALIDE_ALWAYS_INLINE bool match(const BroadcastOp< A2, known_lanes_2 > &op, MatcherState &state) const noexcept
Definition: IRMatch.h:1542
Halide::Internal::IRMatcher::Wild::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:459
Halide::Internal::IRNodeType::Max
@ Max
Halide::Internal::IRMatcher::unwrap
HALIDE_ALWAYS_INLINE T unwrap(T t)
Definition: IRMatch.h:586
Halide::Internal::BaseExprNode
A base class for expression nodes.
Definition: Expr.h:141
Halide::Internal::IRMatcher::MatcherState::special_values_mask
static constexpr uint16_t special_values_mask
Definition: IRMatch.h:82
halide_type_uint
@ halide_type_uint
unsigned integers
Definition: HalideRuntime.h:405
Halide::Internal::IRMatcher::Rewriter::Rewriter
HALIDE_ALWAYS_INLINE Rewriter(Instance &&instance, halide_type_t ot, halide_type_t wt)
Definition: IRMatch.h:2283
Halide::Internal::IRMatcher::BinOp::pattern_tag
Definition: IRMatch.h:623
Halide::Internal::IRMatcher::CmpOp::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:740
Halide::Internal::IRMatcher::Fold::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:1903
Halide::abs
Expr abs(Expr a)
Returns the absolute value of a signed integer or floating-point expression.
Halide::Internal::IRMatcher::Intrin::Intrin
HALIDE_ALWAYS_INLINE Intrin(Call::IntrinsicOp intrin, Args... args) noexcept
Definition: IRMatch.h:1379
Halide::Internal::Not::a
Expr a
Definition: IR.h:176
Halide::Internal::IRMatcher::Intrin::print_args
HALIDE_ALWAYS_INLINE void print_args(std::ostream &s) const
Definition: IRMatch.h:1361
Halide::Internal::IRMatcher::Fold
Definition: IRMatch.h:1896
Halide::Internal::make_const
Expr make_const(Type t, int64_t val)
Construct an immediate of the given type from any numeric C++ type.
Halide::Internal::IRMatcher::add
HALIDE_ALWAYS_INLINE auto add(A a, B b) -> decltype(IRMatcher::operator+(a, b))
Definition: IRMatch.h:920
Halide::Internal::IRMatcher::h_min
HALIDE_ALWAYS_INLINE auto h_min(A a, int lanes) noexcept -> VectorReduceOp< decltype(pattern_arg(a)), true, VectorReduce::Min >
Definition: IRMatch.h:1731
Halide::Internal::IRMatcher::RampOp::binds
constexpr static uint32_t binds
Definition: IRMatch.h:1598
Halide::Internal::IRMatcher::Fold::binds
constexpr static uint32_t binds
Definition: IRMatch.h:1900
Halide::Internal::VectorReduce::And
@ And
Definition: IR.h:836
Halide::Internal::IRMatcher::Overflows::binds
constexpr static uint32_t binds
Definition: IRMatch.h:1938
Halide::Internal::IRMatcher::constant_fold_cmp_op< EQ >
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op< EQ >(int64_t a, int64_t b) noexcept
Definition: IRMatch.h:1202
Halide::Internal::IRMatcher::NegateOp::canonical
constexpr static bool canonical
Definition: IRMatch.h:1780
Halide::Internal::Min
The lesser of two values.
Definition: IR.h:85
Halide::Internal::IRMatcher::BroadcastOp
Definition: IRMatch.h:1517
Halide::Internal::IRMatcher::MatcherState
To save stack space, the matcher objects are largely stateless and immutable.
Definition: IRMatch.h:76
Halide::Internal::IRMatcher::Rewriter::state
MatcherState state
Definition: IRMatch.h:2278
Halide::Internal::IRNodeType::VectorReduce
@ VectorReduce
Halide::Internal::IntImm::make
static const IntImm * make(Type t, int64_t value)
Halide::Internal::IRMatcher::operator-
HALIDE_ALWAYS_INLINE auto operator-(A a, B b) noexcept -> BinOp< Sub, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:944
Halide::Internal::Mod
The remainder of a / b.
Definition: IR.h:76
Halide::Internal::IRMatcher::VectorReduceOp::lanes
int lanes
Definition: IRMatch.h:1671
Halide::Internal::IRMatcher::broadcast
HALIDE_ALWAYS_INLINE auto broadcast(A a, int lanes) noexcept -> BroadcastOp< decltype(pattern_arg(a)), true >
Definition: IRMatch.h:1582
Halide::Internal::Not::make
static Expr make(Expr a)
Halide::Internal::VectorReduce::Or
@ Or
Definition: IR.h:837
Halide::Internal::IRMatcher::MatcherState::get_bound_const
HALIDE_ALWAYS_INLINE void get_bound_const(int i, halide_scalar_value_t &val, halide_type_t &type) const noexcept
Definition: IRMatch.h:121
Halide::Internal::IRMatcher::MatcherState::set_bound_const
HALIDE_ALWAYS_INLINE void set_bound_const(int i, double f, halide_type_t t) noexcept
Definition: IRMatch.h:109
Halide::Internal::IRMatcher::IsConst::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition: IRMatch.h:2024
Halide::Internal::IRMatcher::WildConstFloat::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:344
Halide::Internal::IRMatcher::SelectOp::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:1462
Halide::Internal::IRMatcher::WildConst::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:398
Halide::likely_if_innermost
Expr likely_if_innermost(Expr e)
Equivalent to likely, but only triggers a loop partitioning if found in an innermost loop.
Halide::Internal::IRMatcher::Wild::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:450
Halide::Internal::IRMatcher::ge
HALIDE_ALWAYS_INLINE auto ge(A a, B b) -> decltype(IRMatcher::operator>=(a, b))
Definition: IRMatch.h:1172
Halide::Internal::IRMatcher::Overflows::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition: IRMatch.h:1949
Halide::Internal::IRMatcher::WildConst::pattern_tag
Definition: IRMatch.h:389
Halide::Internal::IRMatcher::and_op
HALIDE_ALWAYS_INLINE auto and_op(A a, B b) -> decltype(IRMatcher::operator&&(a, b))
Definition: IRMatch.h:1273
Halide::Internal::IRMatcher::BroadcastOp::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:1530
Halide::Internal::IRMatcher::Intrin::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:1338
Halide::Internal::IRMatcher::CastOp::match
HALIDE_ALWAYS_INLINE bool match(const CastOp< A2 > &op, MatcherState &state) const noexcept
Definition: IRMatch.h:1872
Halide::Internal::IRMatcher::evaluate_predicate
HALIDE_ALWAYS_INLINE bool evaluate_predicate(bool x, MatcherState &) noexcept
Definition: IRMatch.h:2248
Halide::Internal::IRMatcher::CmpOp::pattern_tag
Definition: IRMatch.h:734
Halide::Internal::IRMatcher::BinOp::match
HALIDE_ALWAYS_INLINE bool match(const BinOp< Op2, A2, B2 > &op, MatcherState &state) const noexcept
Definition: IRMatch.h:649
Halide::Internal::IRMatcher::SelectOp::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:1468
Halide::Internal::IRMatcher::WildConstUInt::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:286
Halide::Internal::IRNodeType::NE
@ NE
Halide::Internal::FloatImm::make
static const FloatImm * make(Type t, double value)
Halide::Internal::IRMatcher::operator||
HALIDE_ALWAYS_INLINE auto operator||(A a, B b) noexcept -> BinOp< Or, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1242
Halide::Internal::IRMatcher::WildConstInt::foldable
constexpr static bool foldable
Definition: IRMatch.h:266
Halide::Internal::IRMatcher::ne
HALIDE_ALWAYS_INLINE auto ne(A a, B b) -> decltype(IRMatcher::operator!=(a, b))
Definition: IRMatch.h:1222
internal_error
#define internal_error
Definition: Errors.h:23
Halide::Internal::IRMatcher::IsFloat::binds
constexpr static uint32_t binds
Definition: IRMatch.h:2086
Halide::Internal::IRMatcher::RampOp::canonical
constexpr static bool canonical
Definition: IRMatch.h:1603
Halide::Internal::IRMatcher::CanProve::foldable
constexpr static bool foldable
Definition: IRMatch.h:2057
Halide::Internal::Select::_node_type
static const IRNodeType _node_type
Definition: IR.h:191
Halide::Internal::Call
A function call.
Definition: IR.h:464
Halide::Internal::IRMatcher::Overflows::pattern_tag
Definition: IRMatch.h:1935
Halide::Internal::IRMatcher::operator!
HALIDE_ALWAYS_INLINE auto operator!(A a) noexcept -> NotOp< decltype(pattern_arg(a))>
Definition: IRMatch.h:1438
Halide::Internal::IRMatcher::enable_if_pattern
Definition: IRMatch.h:133
Halide::Internal::IRMatcher::NegateOp::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:1778
Halide::Internal::IRMatcher::Rewriter::result
Expr result
Definition: IRMatch.h:2277
Halide::Internal::IRMatcher::VectorReduceOp::a
A a
Definition: IRMatch.h:1670
Halide::Internal::Call::IntrinsicOp
IntrinsicOp
Definition: IR.h:492
Halide::Internal::IRMatcher::MatcherState::set_bound_const
HALIDE_ALWAYS_INLINE void set_bound_const(int i, uint64_t u, halide_type_t t) noexcept
Definition: IRMatch.h:103
Halide::Internal::Ramp::_node_type
static const IRNodeType _node_type
Definition: IR.h:235
Halide::Internal::IRMatcher::BinOp::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const noexcept
Definition: IRMatch.h:701
Halide::Internal::IRMatcher::Rewriter::wildcard_type
halide_type_t wildcard_type
Definition: IRMatch.h:2279
Halide::Internal::IRMatcher::Rewriter::operator()
HALIDE_ALWAYS_INLINE bool operator()(Before before, int64_t after, Predicate pred)
Definition: IRMatch.h:2413
Halide::Internal::IRMatcher::NotOp::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition: IRMatch.h:1430
Halide::Internal::IRMatcher::MatcherState::bound_const_type
halide_type_t bound_const_type[max_wild]
Definition: IRMatch.h:84
Halide::Internal::IRNodeType::Mul
@ Mul
Halide::Internal::IRMatcher::WildConst
Definition: IRMatch.h:388
Halide::Internal::Call::is_intrinsic
bool is_intrinsic() const
Definition: IR.h:630
Halide::Internal::IRMatcher::NotOp::binds
constexpr static uint32_t binds
Definition: IRMatch.h:1402
Halide::Internal::IRMatcher::h_max
HALIDE_ALWAYS_INLINE auto h_max(A a, int lanes) noexcept -> VectorReduceOp< decltype(pattern_arg(a)), true, VectorReduce::Max >
Definition: IRMatch.h:1741
Halide::Internal::IRMatcher::is_const
HALIDE_ALWAYS_INLINE auto is_const(A a) noexcept -> IsConst< decltype(pattern_arg(a))>
Definition: IRMatch.h:2034
Halide::Internal::IRMatcher::CmpOp::canonical
constexpr static bool canonical
Definition: IRMatch.h:742
Halide::Internal::IRMatcher::select
HALIDE_ALWAYS_INLINE auto select(C c, T t, F f) noexcept -> SelectOp< decltype(pattern_arg(c)), decltype(pattern_arg(t)), decltype(pattern_arg(f))>
Definition: IRMatch.h:1512
Halide::Internal::IRMatcher::WildConstInt::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:234
Halide::Internal::IRNodeType::GT
@ GT
Halide::Internal::IRMatcher::NegateOp
Definition: IRMatch.h:1771
Halide::Internal::IRMatcher::CastOp
Definition: IRMatch.h:1851
Halide::Internal::IRMatcher::constant_fold_bin_op< Add >
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< Add >(halide_type_t &t, int64_t a, int64_t b) noexcept
Definition: IRMatch.h:925
Halide::Internal::IRMatcher::Const::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:536
Halide::Internal::IRMatcher::CanProve::binds
constexpr static uint32_t binds
Definition: IRMatch.h:2050
Halide::Internal::IRMatcher::NegateOp::foldable
constexpr static bool foldable
Definition: IRMatch.h:1804
Halide::likely
Expr likely(Expr e)
Expressions tagged with this intrinsic are considered to be part of the steady state of some loop wit...
Halide::Internal::IRNodeType::UIntImm
@ UIntImm
Halide::Internal::IRMatcher::Intrin::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:1366
Halide::Internal::Select
A ternary operator.
Definition: IR.h:186
Halide::Internal::IRMatcher::CmpOp::foldable
constexpr static bool foldable
Definition: IRMatch.h:765
Halide::Internal::IRMatcher::Wild::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:446
Halide::Internal::IRMatcher::rewriter
HALIDE_ALWAYS_INLINE auto rewriter(Instance instance, halide_type_t output_type, halide_type_t wildcard_type) noexcept -> Rewriter< decltype(pattern_arg(instance))>
Construct a rewriter for the given instance, which may be a pattern with concrete expressions as leav...
Definition: IRMatch.h:2451
Halide::Internal::IRNodeType::Ramp
@ Ramp
Halide::Internal::IRMatcher::is_float
HALIDE_ALWAYS_INLINE auto is_float(A a) noexcept -> IsFloat< decltype(pattern_arg(a))>
Definition: IRMatch.h:2107
Halide::Internal::IRMatcher::BroadcastOp::foldable
constexpr static bool foldable
Definition: IRMatch.h:1559
Halide::Internal::IRMatcher::RampOp::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:1601
Halide::Internal::IRNode::node_type
IRNodeType node_type
Each IR node subclass has a unique identifier.
Definition: Expr.h:111
Halide::Internal::Select::false_value
Expr false_value
Definition: IR.h:187
Halide::Internal::IRMatcher::RampOp::a
A a
Definition: IRMatch.h:1594
Halide::Internal::IRMatcher::Const::foldable
constexpr static bool foldable
Definition: IRMatch.h:540
Halide::Internal::IRMatcher::sub
HALIDE_ALWAYS_INLINE auto sub(A a, B b) -> decltype(IRMatcher::operator-(a, b))
Definition: IRMatch.h:949
Halide::Internal::Ramp::base
Expr base
Definition: IR.h:230
Halide::Internal::IRMatcher::Const::v
int64_t v
Definition: IRMatch.h:499
Halide::Internal::IRMatcher::CastOp::canonical
constexpr static bool canonical
Definition: IRMatch.h:1860
Halide::Internal::Sub::make
static Expr make(Expr a, Expr b)
Halide::Internal::IRMatcher::BinOp::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:639
Halide::Internal::IRMatcher::BinOp::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition: IRMatch.h:658
Halide::Internal::IRMatcher::operator&&
HALIDE_ALWAYS_INLINE auto operator&&(A a, B b) noexcept -> BinOp< And, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1268
Halide::Internal::VectorReduce::_node_type
static const IRNodeType _node_type
Definition: IR.h:845
Halide::Internal::Call::_node_type
static const IRNodeType _node_type
Definition: IR.h:645
Halide::Internal::IRMatcher::NotOp::a
A a
Definition: IRMatch.h:1400
Halide::Internal::Broadcast::_node_type
static const IRNodeType _node_type
Definition: IR.h:247
Halide::Internal::IRMatcher::Overflow::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition: IRMatch.h:1997
Halide::Internal::IRMatcher::VectorReduceOp::match
HALIDE_ALWAYS_INLINE bool match(const VectorReduceOp< A2, known_lanes_2, reduce_op_2 > &op, MatcherState &state) const noexcept
Definition: IRMatch.h:1693
Halide::Expr
A fragment of Halide syntax.
Definition: Expr.h:256
Halide::Internal::IRMatcher::NotOp::foldable
constexpr static bool foldable
Definition: IRMatch.h:1427
Halide::Internal::IRMatcher::Intrin::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:1319
Halide::Internal::IRMatcher::Intrin::foldable
constexpr static bool foldable
Definition: IRMatch.h:1376
Halide::Internal::mul_would_overflow
bool mul_would_overflow(int bits, int64_t a, int64_t b)
Halide::Internal::IRMatcher::Overflows::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:1943
halide_type_int
@ halide_type_int
signed integers
Definition: HalideRuntime.h:404
Halide::Internal::IRMatcher::BroadcastOp::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:1548
Halide::Internal::IRMatcher::Overflows::canonical
constexpr static bool canonical
Definition: IRMatch.h:1944
Halide::Internal::IRMatcher::NotOp::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:1409
Halide::Internal::IRMatcher::WildConstUInt::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition: IRMatch.h:322
Halide::Internal::IRMatcher::WildConstUInt::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:291
Halide::Internal::GT
Is the first expression greater than the second.
Definition: IR.h:139
Halide::Internal::IRMatcher::SelectOp::foldable
constexpr static bool foldable
Definition: IRMatch.h:1489
Halide::Internal::IRMatcher::WildConstUInt::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:312
Halide::Internal::IRMatcher::Overflows
Definition: IRMatch.h:1934
Halide::Internal::IRMatcher::Rewriter
Definition: IRMatch.h:2275
Halide::Internal::IRMatcher::NotOp::pattern_tag
Definition: IRMatch.h:1399
Halide::Expr::get
const HALIDE_ALWAYS_INLINE Internal::BaseExprNode * get() const
Override get() to return a BaseExprNode * instead of an IRNode *.
Definition: Expr.h:314
Halide::Internal::IRMatcher::Intrin::args
std::tuple< Args... > args
Definition: IRMatch.h:1315
Halide::Internal::IRMatcher::gt
HALIDE_ALWAYS_INLINE auto gt(A a, B b) -> decltype(IRMatcher::operator>(a, b))
Definition: IRMatch.h:1122
Halide::Internal::IRMatcher::operator%
HALIDE_ALWAYS_INLINE auto operator%(A a, B b) noexcept -> BinOp< Mod, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1027
uint32_t
unsigned __INT32_TYPE__ uint32_t
Definition: runtime_internal.h:21
Halide::Internal::IRMatcher::h_or
HALIDE_ALWAYS_INLINE auto h_or(A a, int lanes) noexcept -> VectorReduceOp< decltype(pattern_arg(a)), true, VectorReduce::Or >
Definition: IRMatch.h:1761
Halide::Internal::IRNodeType::Shuffle
@ Shuffle
Halide::Internal::UIntImm
Unsigned integer constants.
Definition: Expr.h:225
Halide::Internal::IRMatcher::CanProve::canonical
constexpr static bool canonical
Definition: IRMatch.h:2055
Halide::Internal::IRMatcher::constant_fold_bin_op< Min >
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< Min >(halide_type_t &t, int64_t a, int64_t b) noexcept
Definition: IRMatch.h:1057
Halide::Internal::IRMatcher::Fold::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:1902
Halide::Internal::IRMatcher::BroadcastOp::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition: IRMatch.h:1562
Halide::Internal::IRMatcher::constant_fold_cmp_op< NE >
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op< NE >(int64_t a, int64_t b) noexcept
Definition: IRMatch.h:1227
Halide::Internal::IRMatcher::RampOp::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:1628
IREquality.h
Halide::Internal::IRMatcher::operator==
HALIDE_ALWAYS_INLINE auto operator==(A a, B b) noexcept -> CmpOp< EQ, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1192
Halide::Internal::IRMatcher::Overflow::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:1980
Halide::Internal::IRMatcher::constant_fold_cmp_op< GE >
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op< GE >(int64_t a, int64_t b) noexcept
Definition: IRMatch.h:1177
Halide::Internal::IRMatcher::WildConstFloat::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:339
Halide::Internal::IRMatcher::div
HALIDE_ALWAYS_INLINE auto div(A a, B b) -> decltype(IRMatcher::operator/(a, b))
Definition: IRMatch.h:1007
Halide::Internal::IRMatcher::constant_fold_bin_op< Max >
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< Max >(halide_type_t &t, int64_t a, int64_t b) noexcept
Definition: IRMatch.h:1077
Halide::Internal::IRMatcher::operator+
HALIDE_ALWAYS_INLINE auto operator+(A a, B b) noexcept -> BinOp< Add, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:915
Halide::Internal::IRMatcher::Const::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:513
Halide::Internal::IRMatcher::enable_if_pattern::type
Definition: IRMatch.h:134
Halide::Internal::Sub
The difference of two expressions.
Definition: IR.h:47
Halide::Internal::IRMatcher::CanProve::prover
Prover * prover
Definition: IRMatch.h:2048
halide_scalar_value_t::f64
double f64
Definition: HalideRuntime.h:1569
Halide::Internal::IRMatcher::BroadcastOp::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:1525
Halide::Internal::IRMatcher::operator/
HALIDE_ALWAYS_INLINE auto operator/(A a, B b) noexcept -> BinOp< Div, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1002
Halide::Internal::IRMatcher::Wild::foldable
constexpr static bool foldable
Definition: IRMatch.h:463
Halide::Internal::IRMatcher::constant_fold_bin_op< Sub >
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< Sub >(halide_type_t &t, int64_t a, int64_t b) noexcept
Definition: IRMatch.h:954
Halide::Internal::IRMatcher::constant_fold_bin_op< Or >
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< Or >(halide_type_t &t, int64_t a, int64_t b) noexcept
Definition: IRMatch.h:1252
Halide::Internal::is_zero
bool is_zero(const Expr &e)
Is the expression a const (as defined by is_const), and also equal to zero (in all lanes,...
Halide::Internal::IRMatcher::IsConst::binds
constexpr static uint32_t binds
Definition: IRMatch.h:2012
Halide::Internal::IRMatcher::CmpOp::match
HALIDE_ALWAYS_INLINE bool match(const CmpOp< Op2, A2, B2 > &op, MatcherState &state) const noexcept
Definition: IRMatch.h:759
Halide::Internal::Ramp::stride
Expr stride
Definition: IR.h:230
Halide::Internal::IRMatcher::SelectOp::match
HALIDE_ALWAYS_INLINE bool match(const SelectOp< C2, T2, F2 > &instance, MatcherState &state) const noexcept
Definition: IRMatch.h:1478
Halide::Internal::IRMatcher::CanProve::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:2053
Halide::Internal::IRNodeType::Call
@ Call
Halide::Internal::IRMatcher::WildConst::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:394
Halide::Internal::And
Logical and - are both expressions true.
Definition: IR.h:157
Halide::Internal::IRMatcher::NegateOp::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:1783
Halide::Internal::IRMatcher::SpecificExpr::expr
Expr expr
Definition: IRMatch.h:207
Halide::Internal::Call::likely_if_innermost
@ likely_if_innermost
Definition: IR.h:526
Halide::Internal::IRMatcher::WildConstInt::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:233
Halide::Internal::Cast::_node_type
static const IRNodeType _node_type
Definition: IR.h:34
Halide::Internal::IRMatcher::NotOp::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:1423
Halide::Internal::IRMatcher::WildConstFloat
Definition: IRMatch.h:334
Halide::Internal::IRMatcher::CanProve::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:2054
Halide::Internal::IRMatcher::CastOp::t
Type t
Definition: IRMatch.h:1853
Halide::Internal::IRMatcher::operator<
HALIDE_ALWAYS_INLINE auto operator<(A a, B b) noexcept -> CmpOp< LT, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1092
Halide::Internal::IRMatcher::max
HALIDE_ALWAYS_INLINE auto max(A a, B b) noexcept -> BinOp< Max, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1072
Halide::Internal::IRMatcher::WildConst::binds
constexpr static uint32_t binds
Definition: IRMatch.h:391
Halide::Internal::IRMatcher::Rewriter::operator()
HALIDE_ALWAYS_INLINE bool operator()(Before before, int64_t after) noexcept
Definition: IRMatch.h:2337
Halide::Internal::IRMatcher::WildConstUInt::canonical
constexpr static bool canonical
Definition: IRMatch.h:288
Halide::Internal::Not
Logical not - true if the expression false.
Definition: IR.h:175
Halide::Internal::IRMatcher::Intrin::print_args
HALIDE_ALWAYS_INLINE void print_args(double, std::ostream &s) const
Definition: IRMatch.h:1357
Halide::Internal::LT
Is the first expression less than the second.
Definition: IR.h:121
Halide::Internal::IRMatcher::Wild::binds
constexpr static uint32_t binds
Definition: IRMatch.h:443
Halide::Internal::IRMatcher::constant_fold_bin_op< And >
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< And >(halide_type_t &t, int64_t a, int64_t b) noexcept
Definition: IRMatch.h:1278
Halide::Internal::IRMatcher::equal
HALIDE_ALWAYS_INLINE bool equal(const BaseExprNode &a, const BaseExprNode &b) noexcept
Definition: IRMatch.h:188
halide_scalar_value_t::u64
uint64_t u64
Definition: HalideRuntime.h:1567
Halide::Internal::Mul
The product of two expressions.
Definition: IR.h:56
halide_type_t::code
uint8_t code
The basic type code: signed integer, unsigned integer, or floating point.
Definition: HalideRuntime.h:433
Halide::Internal::VectorReduce::Add
@ Add
Definition: IR.h:832
Halide::Internal::IRMatcher::Rewriter::build_replacement
HALIDE_NEVER_INLINE void build_replacement(After after)
Definition: IRMatch.h:2288
Halide::Internal::IRMatcher::IsConst::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:2016
Halide::Internal::IRMatcher::BroadcastOp::canonical
constexpr static bool canonical
Definition: IRMatch.h:1527
Halide::Internal::IRMatcher::operator>
HALIDE_ALWAYS_INLINE auto operator>(A a, B b) noexcept -> CmpOp< GT, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1117
Halide::Internal::IRMatcher::Overflow::binds
constexpr static uint32_t binds
Definition: IRMatch.h:1972
Halide::Internal::IRMatcher::BroadcastOp::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:1524