Halide
IRMatch.h
Go to the documentation of this file.
1 #ifndef HALIDE_IR_MATCH_H
2 #define HALIDE_IR_MATCH_H
3 
4 /** \file
5  * Defines a method to match a fragment of IR against a pattern containing wildcards
6  */
7 
8 #include <map>
9 #include <random>
10 #include <set>
11 #include <vector>
12 
13 #include "IR.h"
14 #include "IREquality.h"
15 #include "IROperator.h"
16 
17 namespace Halide {
18 namespace Internal {
19 
20 /** Does the first expression have the same structure as the second?
21  * Variables in the first expression with the name * are interpreted
22  * as wildcards, and their matching equivalent in the second
23  * expression is placed in the vector give as the third argument.
24  * Wildcards require the types to match. For the type bits and width,
25  * a 0 indicates "match anything". So an Int(8, 0) will match 8-bit
26  * integer vectors of any width (including scalars), and a UInt(0, 0)
27  * will match any unsigned integer type.
28  *
29  * For example:
30  \code
31  Expr x = Variable::make(Int(32), "*");
32  match(x + x, 3 + (2*k), result)
33  \endcode
34  * should return true, and set result[0] to 3 and
35  * result[1] to 2*k.
36  */
37 bool expr_match(const Expr &pattern, const Expr &expr, std::vector<Expr> &result);
38 
39 /** Does the first expression have the same structure as the second?
40  * Variables are matched consistently. The first time a variable is
41  * matched, it assumes the value of the matching part of the second
42  * expression. Subsequent matches must be equal to the first match.
43  *
44  * For example:
45  \code
46  Var x("x"), y("y");
47  match(x*(x + y), a*(a + b), result)
48  \endcode
49  * should return true, and set result["x"] = a, and result["y"] = b.
50  */
51 bool expr_match(const Expr &pattern, const Expr &expr, std::map<std::string, Expr> &result);
52 
53 /** Rewrite the expression x to have `lanes` lanes. This is useful
54  * for substituting the results of expr_match into a pattern expression. */
55 Expr with_lanes(const Expr &x, int lanes);
56 
57 void expr_match_test();
58 
59 /** An alternative template-metaprogramming approach to expression
60  * matching. Potentially more efficient. We lift the expression
61  * pattern into a type, and then use force-inlined functions to
62  * generate efficient matching and reconstruction code for any
63  * pattern. Pattern elements are either one of the classes in the
64  * namespace IRMatcher, or are non-null Exprs (represented as
65  * BaseExprNode &).
66  *
67  * Pattern elements that are fully specified by their pattern can be
68  * built into an expression using the make method. Some patterns,
69  * such as a broadcast that matches any number of lanes, don't have
70  * enough information to recreate an Expr.
71  */
72 namespace IRMatcher {
73 
74 constexpr int max_wild = 6;
75 
76 static const halide_type_t i64_type = {halide_type_int, 64, 1};
77 
78 /** To save stack space, the matcher objects are largely stateless and
79  * immutable. This state object is built up during matching and then
80  * consumed when constructing a replacement Expr.
81  */
82 struct MatcherState {
85 
86  // values of the lanes field with special meaning.
87  static constexpr uint16_t signed_integer_overflow = 0x8000;
88  static constexpr uint16_t special_values_mask = 0x8000; // currently only one
89 
91 
93  void set_binding(int i, const BaseExprNode &n) noexcept {
94  bindings[i] = &n;
95  }
96 
98  const BaseExprNode *get_binding(int i) const noexcept {
99  return bindings[i];
100  }
101 
103  void set_bound_const(int i, int64_t s, halide_type_t t) noexcept {
104  bound_const[i].u.i64 = s;
105  bound_const_type[i] = t;
106  }
107 
109  void set_bound_const(int i, uint64_t u, halide_type_t t) noexcept {
110  bound_const[i].u.u64 = u;
111  bound_const_type[i] = t;
112  }
113 
115  void set_bound_const(int i, double f, halide_type_t t) noexcept {
116  bound_const[i].u.f64 = f;
117  bound_const_type[i] = t;
118  }
119 
121  void set_bound_const(int i, halide_scalar_value_t val, halide_type_t t) noexcept {
122  bound_const[i] = val;
123  bound_const_type[i] = t;
124  }
125 
127  void get_bound_const(int i, halide_scalar_value_t &val, halide_type_t &type) const noexcept {
128  val = bound_const[i];
129  type = bound_const_type[i];
130  }
131 
133  // NOLINTNEXTLINE(modernize-use-equals-default): Can't use `= default`; clang-tidy complains about noexcept mismatch
134  MatcherState() noexcept {
135  }
136 };
137 
138 template<typename T,
139  typename = typename std::remove_reference<T>::type::pattern_tag>
141  struct type {};
142 };
143 
144 template<typename T>
145 struct bindings {
146  constexpr static uint32_t mask = std::remove_reference<T>::type::binds;
147 };
148 
151  ty.lanes &= ~MatcherState::special_values_mask;
153  return make_signed_integer_overflow(ty);
154  }
155  // unreachable
156  return Expr();
157 }
158 
161  halide_type_t scalar_type = ty;
162  if (scalar_type.lanes & MatcherState::special_values_mask) {
163  return make_const_special_expr(scalar_type);
164  }
165 
166  const int lanes = scalar_type.lanes;
167  scalar_type.lanes = 1;
168 
169  Expr e;
170  switch (scalar_type.code) {
171  case halide_type_int:
172  e = IntImm::make(scalar_type, val.u.i64);
173  break;
174  case halide_type_uint:
175  e = UIntImm::make(scalar_type, val.u.u64);
176  break;
177  case halide_type_float:
178  case halide_type_bfloat:
179  e = FloatImm::make(scalar_type, val.u.f64);
180  break;
181  default:
182  // Unreachable
183  return Expr();
184  }
185  if (lanes > 1) {
186  e = Broadcast::make(e, lanes);
187  }
188  return e;
189 }
190 
191 bool equal_helper(const BaseExprNode &a, const BaseExprNode &b) noexcept;
192 
193 // A fast version of expression equality that assumes a well-typed non-null expression tree.
195 bool equal(const BaseExprNode &a, const BaseExprNode &b) noexcept {
196  // Early out
197  return (&a == &b) ||
198  ((a.type == b.type) &&
199  (a.node_type == b.node_type) &&
200  equal_helper(a, b));
201 }
202 
203 // A pattern that matches a specific expression
204 struct SpecificExpr {
205  struct pattern_tag {};
206 
207  constexpr static uint32_t binds = 0;
208 
209  // What is the weakest and strongest IR node this could possibly be
212  constexpr static bool canonical = true;
213 
215 
216  template<uint32_t bound>
217  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
218  return equal(expr, e);
219  }
220 
222  Expr make(MatcherState &state, halide_type_t type_hint) const {
223  return Expr(&expr);
224  }
225 
226  constexpr static bool foldable = false;
227 };
228 
229 inline std::ostream &operator<<(std::ostream &s, const SpecificExpr &e) {
230  s << Expr(&e.expr);
231  return s;
232 }
233 
234 template<int i>
235 struct WildConstInt {
236  struct pattern_tag {};
237 
238  constexpr static uint32_t binds = 1 << i;
239 
242  constexpr static bool canonical = true;
243 
244  template<uint32_t bound>
245  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
246  static_assert(i >= 0 && i < max_wild, "Wild with out-of-range index");
247  const BaseExprNode *op = &e;
248  if (op->node_type == IRNodeType::Broadcast) {
249  op = ((const Broadcast *)op)->value.get();
250  }
251  if (op->node_type != IRNodeType::IntImm) {
252  return false;
253  }
254  int64_t value = ((const IntImm *)op)->value;
255  if (bound & binds) {
257  halide_type_t type;
258  state.get_bound_const(i, val, type);
259  return (halide_type_t)e.type == type && value == val.u.i64;
260  }
261  state.set_bound_const(i, value, e.type);
262  return true;
263  }
264 
265  template<uint32_t bound>
266  HALIDE_ALWAYS_INLINE bool match(int64_t value, MatcherState &state) const noexcept {
267  static_assert(i >= 0 && i < max_wild, "Wild with out-of-range index");
268  if (bound & binds) {
270  halide_type_t type;
271  state.get_bound_const(i, val, type);
272  return type == i64_type && value == val.u.i64;
273  }
274  state.set_bound_const(i, value, i64_type);
275  return true;
276  }
277 
279  Expr make(MatcherState &state, halide_type_t type_hint) const {
281  halide_type_t type;
282  state.get_bound_const(i, val, type);
283  return make_const_expr(val, type);
284  }
285 
286  constexpr static bool foldable = true;
287 
290  state.get_bound_const(i, val, ty);
291  }
292 };
293 
294 template<int i>
295 std::ostream &operator<<(std::ostream &s, const WildConstInt<i> &c) {
296  s << "ci" << i;
297  return s;
298 }
299 
300 template<int i>
302  struct pattern_tag {};
303 
304  constexpr static uint32_t binds = 1 << i;
305 
308  constexpr static bool canonical = true;
309 
310  template<uint32_t bound>
311  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
312  static_assert(i >= 0 && i < max_wild, "Wild with out-of-range index");
313  const BaseExprNode *op = &e;
314  if (op->node_type == IRNodeType::Broadcast) {
315  op = ((const Broadcast *)op)->value.get();
316  }
317  if (op->node_type != IRNodeType::UIntImm) {
318  return false;
319  }
320  uint64_t value = ((const UIntImm *)op)->value;
321  if (bound & binds) {
323  halide_type_t type;
324  state.get_bound_const(i, val, type);
325  return (halide_type_t)e.type == type && value == val.u.u64;
326  }
327  state.set_bound_const(i, value, e.type);
328  return true;
329  }
330 
332  Expr make(MatcherState &state, halide_type_t type_hint) const {
334  halide_type_t type;
335  state.get_bound_const(i, val, type);
336  return make_const_expr(val, type);
337  }
338 
339  constexpr static bool foldable = true;
340 
342  void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept {
343  state.get_bound_const(i, val, ty);
344  }
345 };
346 
347 template<int i>
348 std::ostream &operator<<(std::ostream &s, const WildConstUInt<i> &c) {
349  s << "cu" << i;
350  return s;
351 }
352 
353 template<int i>
355  struct pattern_tag {};
356 
357  constexpr static uint32_t binds = 1 << i;
358 
361  constexpr static bool canonical = true;
362 
363  template<uint32_t bound>
364  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
365  static_assert(i >= 0 && i < max_wild, "Wild with out-of-range index");
366  const BaseExprNode *op = &e;
367  if (op->node_type == IRNodeType::Broadcast) {
368  op = ((const Broadcast *)op)->value.get();
369  }
370  if (op->node_type != IRNodeType::FloatImm) {
371  return false;
372  }
373  double value = ((const FloatImm *)op)->value;
374  if (bound & binds) {
376  halide_type_t type;
377  state.get_bound_const(i, val, type);
378  return (halide_type_t)e.type == type && value == val.u.f64;
379  }
380  state.set_bound_const(i, value, e.type);
381  return true;
382  }
383 
385  Expr make(MatcherState &state, halide_type_t type_hint) const {
387  halide_type_t type;
388  state.get_bound_const(i, val, type);
389  return make_const_expr(val, type);
390  }
391 
392  constexpr static bool foldable = true;
393 
395  void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept {
396  state.get_bound_const(i, val, ty);
397  }
398 };
399 
400 template<int i>
401 std::ostream &operator<<(std::ostream &s, const WildConstFloat<i> &c) {
402  s << "cf" << i;
403  return s;
404 }
405 
406 // Matches and binds to any constant Expr. Does not support constant-folding.
407 template<int i>
408 struct WildConst {
409  struct pattern_tag {};
410 
411  constexpr static uint32_t binds = 1 << i;
412 
415  constexpr static bool canonical = true;
416 
417  template<uint32_t bound>
418  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
419  static_assert(i >= 0 && i < max_wild, "Wild with out-of-range index");
420  const BaseExprNode *op = &e;
421  if (op->node_type == IRNodeType::Broadcast) {
422  op = ((const Broadcast *)op)->value.get();
423  }
424  switch (op->node_type) {
425  case IRNodeType::IntImm:
426  return WildConstInt<i>().template match<bound>(e, state);
427  case IRNodeType::UIntImm:
428  return WildConstUInt<i>().template match<bound>(e, state);
430  return WildConstFloat<i>().template match<bound>(e, state);
431  default:
432  return false;
433  }
434  }
435 
436  template<uint32_t bound>
437  HALIDE_ALWAYS_INLINE bool match(int64_t e, MatcherState &state) const noexcept {
438  static_assert(i >= 0 && i < max_wild, "Wild with out-of-range index");
439  return WildConstInt<i>().template match<bound>(e, state);
440  }
441 
443  Expr make(MatcherState &state, halide_type_t type_hint) const {
445  halide_type_t type;
446  state.get_bound_const(i, val, type);
447  return make_const_expr(val, type);
448  }
449 
450  constexpr static bool foldable = true;
451 
453  void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept {
454  state.get_bound_const(i, val, ty);
455  }
456 };
457 
458 template<int i>
459 std::ostream &operator<<(std::ostream &s, const WildConst<i> &c) {
460  s << "c" << i;
461  return s;
462 }
463 
464 // Matches and binds to any Expr
465 template<int i>
466 struct Wild {
467  struct pattern_tag {};
468 
469  constexpr static uint32_t binds = 1 << (i + 16);
470 
473  constexpr static bool canonical = true;
474 
475  template<uint32_t bound>
476  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
477  if (bound & binds) {
478  return equal(*state.get_binding(i), e);
479  }
480  state.set_binding(i, e);
481  return true;
482  }
483 
485  Expr make(MatcherState &state, halide_type_t type_hint) const {
486  return state.get_binding(i);
487  }
488 
489  constexpr static bool foldable = false;
490 };
491 
492 template<int i>
493 std::ostream &operator<<(std::ostream &s, const Wild<i> &op) {
494  s << "_" << i;
495  return s;
496 }
497 
498 // Matches a specific constant or broadcast of that constant. The
499 // constant must be representable as an int64_t.
500 struct IntLiteral {
501  struct pattern_tag {};
503 
504  constexpr static uint32_t binds = 0;
505 
508  constexpr static bool canonical = true;
509 
511  explicit IntLiteral(int64_t v)
512  : v(v) {
513  }
514 
515  template<uint32_t bound>
516  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
517  const BaseExprNode *op = &e;
518  if (e.node_type == IRNodeType::Broadcast) {
519  op = ((const Broadcast *)op)->value.get();
520  }
521  switch (op->node_type) {
522  case IRNodeType::IntImm:
523  return ((const IntImm *)op)->value == (int64_t)v;
524  case IRNodeType::UIntImm:
525  return ((const UIntImm *)op)->value == (uint64_t)v;
527  return ((const FloatImm *)op)->value == (double)v;
528  default:
529  return false;
530  }
531  }
532 
533  template<uint32_t bound>
534  HALIDE_ALWAYS_INLINE bool match(int64_t val, MatcherState &state) const noexcept {
535  return v == val;
536  }
537 
538  template<uint32_t bound>
539  HALIDE_ALWAYS_INLINE bool match(const IntLiteral &b, MatcherState &state) const noexcept {
540  return v == b.v;
541  }
542 
544  Expr make(MatcherState &state, halide_type_t type_hint) const {
545  return make_const(type_hint, v);
546  }
547 
548  constexpr static bool foldable = true;
549 
551  void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept {
552  // Assume type is already correct
553  switch (ty.code) {
554  case halide_type_int:
555  val.u.i64 = v;
556  break;
557  case halide_type_uint:
558  val.u.u64 = (uint64_t)v;
559  break;
560  case halide_type_float:
561  case halide_type_bfloat:
562  val.u.f64 = (double)v;
563  break;
564  default:
565  // Unreachable
566  ;
567  }
568  }
569 };
570 
572  return t.v;
573 }
574 
575 // Convert a provided pattern, expr, or constant int into the internal
576 // representation we use in the matcher trees.
577 template<typename T,
578  typename = typename std::decay<T>::type::pattern_tag>
580  return t;
581 }
584  return IntLiteral{x};
585 }
586 
587 template<typename T>
589  static_assert(!std::is_same<typename std::decay<T>::type, Expr>::value || std::is_lvalue_reference<T>::value,
590  "Exprs are captured by reference by IRMatcher objects and so must be lvalues");
591 }
592 
594  return {*e.get()};
595 }
596 
597 // Helpers to deref SpecificExprs to const BaseExprNode & rather than
598 // passing them by value anywhere (incurring lots of refcounting)
599 template<typename T,
600  // T must be a pattern node
601  typename = typename std::decay<T>::type::pattern_tag,
602  // But T may not be SpecificExpr
603  typename = typename std::enable_if<!std::is_same<typename std::decay<T>::type, SpecificExpr>::value>::type>
605  return t;
606 }
607 
609 const BaseExprNode &unwrap(const SpecificExpr &e) {
610  return e.expr;
611 }
612 
613 inline std::ostream &operator<<(std::ostream &s, const IntLiteral &op) {
614  s << op.v;
615  return s;
616 }
617 
618 template<typename Op>
620 
621 template<typename Op>
623 
624 template<typename Op>
625 double constant_fold_bin_op(halide_type_t &, double, double) noexcept;
626 
627 constexpr bool commutative(IRNodeType t) {
628  return (t == IRNodeType::Add ||
629  t == IRNodeType::Mul ||
630  t == IRNodeType::And ||
631  t == IRNodeType::Or ||
632  t == IRNodeType::Min ||
633  t == IRNodeType::Max ||
634  t == IRNodeType::EQ ||
635  t == IRNodeType::NE);
636 }
637 
638 // Matches one of the binary operators
639 template<typename Op, typename A, typename B>
640 struct BinOp {
641  struct pattern_tag {};
642  A a;
643  B b;
644 
646 
647  constexpr static IRNodeType min_node_type = Op::_node_type;
648  constexpr static IRNodeType max_node_type = Op::_node_type;
649 
650  // For commutative bin ops, we expect the weaker IR node type on
651  // the right. That is, for the rule to be canonical it must be
652  // possible that A is at least as strong as B.
653  constexpr static bool canonical =
654  A::canonical && B::canonical && (!commutative(Op::_node_type) || (A::max_node_type >= B::min_node_type));
655 
656  template<uint32_t bound>
657  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
658  if (e.node_type != Op::_node_type) {
659  return false;
660  }
661  const Op &op = (const Op &)e;
662  return (a.template match<bound>(*op.a.get(), state) &&
663  b.template match<bound | bindings<A>::mask>(*op.b.get(), state));
664  }
665 
666  template<uint32_t bound, typename Op2, typename A2, typename B2>
667  HALIDE_ALWAYS_INLINE bool match(const BinOp<Op2, A2, B2> &op, MatcherState &state) const noexcept {
668  return (std::is_same<Op, Op2>::value &&
669  a.template match<bound>(unwrap(op.a), state) &&
670  b.template match<bound | bindings<A>::mask>(unwrap(op.b), state));
671  }
672 
673  constexpr static bool foldable = A::foldable && B::foldable;
674 
676  void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept {
677  halide_scalar_value_t val_a, val_b;
678  if (std::is_same<A, IntLiteral>::value) {
679  b.make_folded_const(val_b, ty, state);
680  if ((std::is_same<Op, And>::value && val_b.u.u64 == 0) ||
681  (std::is_same<Op, Or>::value && val_b.u.u64 == 1)) {
682  // Short circuit
683  val = val_b;
684  return;
685  }
686  const uint16_t l = ty.lanes;
687  a.make_folded_const(val_a, ty, state);
688  ty.lanes |= l; // Make sure the overflow bits are sticky
689  } else {
690  a.make_folded_const(val_a, ty, state);
691  if ((std::is_same<Op, And>::value && val_a.u.u64 == 0) ||
692  (std::is_same<Op, Or>::value && val_a.u.u64 == 1)) {
693  // Short circuit
694  val = val_a;
695  return;
696  }
697  const uint16_t l = ty.lanes;
698  b.make_folded_const(val_b, ty, state);
699  ty.lanes |= l;
700  }
701  switch (ty.code) {
702  case halide_type_int:
703  val.u.i64 = constant_fold_bin_op<Op>(ty, val_a.u.i64, val_b.u.i64);
704  break;
705  case halide_type_uint:
706  val.u.u64 = constant_fold_bin_op<Op>(ty, val_a.u.u64, val_b.u.u64);
707  break;
708  case halide_type_float:
709  case halide_type_bfloat:
710  val.u.f64 = constant_fold_bin_op<Op>(ty, val_a.u.f64, val_b.u.f64);
711  break;
712  default:
713  // unreachable
714  ;
715  }
716  }
717 
719  Expr make(MatcherState &state, halide_type_t type_hint) const noexcept {
720  Expr ea, eb;
721  if (std::is_same<A, IntLiteral>::value) {
722  eb = b.make(state, type_hint);
723  ea = a.make(state, eb.type());
724  } else {
725  ea = a.make(state, type_hint);
726  eb = b.make(state, ea.type());
727  }
728  // We sometimes mix vectors and scalars in the rewrite rules,
729  // so insert a broadcast if necessary.
730  if (ea.type().is_vector() && !eb.type().is_vector()) {
731  eb = Broadcast::make(eb, ea.type().lanes());
732  }
733  if (eb.type().is_vector() && !ea.type().is_vector()) {
734  ea = Broadcast::make(ea, eb.type().lanes());
735  }
736  return Op::make(std::move(ea), std::move(eb));
737  }
738 };
739 
740 template<typename Op>
742 
743 template<typename Op>
745 
746 template<typename Op>
747 uint64_t constant_fold_cmp_op(double, double) noexcept;
748 
749 // Matches one of the comparison operators
750 template<typename Op, typename A, typename B>
751 struct CmpOp {
752  struct pattern_tag {};
753  A a;
754  B b;
755 
757 
758  constexpr static IRNodeType min_node_type = Op::_node_type;
759  constexpr static IRNodeType max_node_type = Op::_node_type;
760  constexpr static bool canonical = (A::canonical &&
761  B::canonical &&
762  (!commutative(Op::_node_type) || A::max_node_type >= B::min_node_type) &&
763  (Op::_node_type != IRNodeType::GE) &&
764  (Op::_node_type != IRNodeType::GT));
765 
766  template<uint32_t bound>
767  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
768  if (e.node_type != Op::_node_type) {
769  return false;
770  }
771  const Op &op = (const Op &)e;
772  return (a.template match<bound>(*op.a.get(), state) &&
773  b.template match<bound | bindings<A>::mask>(*op.b.get(), state));
774  }
775 
776  template<uint32_t bound, typename Op2, typename A2, typename B2>
777  HALIDE_ALWAYS_INLINE bool match(const CmpOp<Op2, A2, B2> &op, MatcherState &state) const noexcept {
778  return (std::is_same<Op, Op2>::value &&
779  a.template match<bound>(unwrap(op.a), state) &&
780  b.template match<bound | bindings<A>::mask>(unwrap(op.b), state));
781  }
782 
783  constexpr static bool foldable = A::foldable && B::foldable;
784 
786  void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept {
787  halide_scalar_value_t val_a, val_b;
788  // If one side is an untyped const, evaluate the other side first to get a type hint.
789  if (std::is_same<A, IntLiteral>::value) {
790  b.make_folded_const(val_b, ty, state);
791  const uint16_t l = ty.lanes;
792  a.make_folded_const(val_a, ty, state);
793  ty.lanes |= l;
794  } else {
795  a.make_folded_const(val_a, ty, state);
796  const uint16_t l = ty.lanes;
797  b.make_folded_const(val_b, ty, state);
798  ty.lanes |= l;
799  }
800  switch (ty.code) {
801  case halide_type_int:
802  val.u.u64 = constant_fold_cmp_op<Op>(val_a.u.i64, val_b.u.i64);
803  break;
804  case halide_type_uint:
805  val.u.u64 = constant_fold_cmp_op<Op>(val_a.u.u64, val_b.u.u64);
806  break;
807  case halide_type_float:
808  case halide_type_bfloat:
809  val.u.u64 = constant_fold_cmp_op<Op>(val_a.u.f64, val_b.u.f64);
810  break;
811  default:
812  // unreachable
813  ;
814  }
815  ty.code = halide_type_uint;
816  ty.bits = 1;
817  }
818 
820  Expr make(MatcherState &state, halide_type_t type_hint) const {
821  // If one side is an untyped const, evaluate the other side first to get a type hint.
822  Expr ea, eb;
823  if (std::is_same<A, IntLiteral>::value) {
824  eb = b.make(state, {});
825  ea = a.make(state, eb.type());
826  } else {
827  ea = a.make(state, {});
828  eb = b.make(state, ea.type());
829  }
830  // We sometimes mix vectors and scalars in the rewrite rules,
831  // so insert a broadcast if necessary.
832  if (ea.type().is_vector() && !eb.type().is_vector()) {
833  eb = Broadcast::make(eb, ea.type().lanes());
834  }
835  if (eb.type().is_vector() && !ea.type().is_vector()) {
836  ea = Broadcast::make(ea, eb.type().lanes());
837  }
838  return Op::make(std::move(ea), std::move(eb));
839  }
840 };
841 
842 template<typename A, typename B>
843 std::ostream &operator<<(std::ostream &s, const BinOp<Add, A, B> &op) {
844  s << "(" << op.a << " + " << op.b << ")";
845  return s;
846 }
847 
848 template<typename A, typename B>
849 std::ostream &operator<<(std::ostream &s, const BinOp<Sub, A, B> &op) {
850  s << "(" << op.a << " - " << op.b << ")";
851  return s;
852 }
853 
854 template<typename A, typename B>
855 std::ostream &operator<<(std::ostream &s, const BinOp<Mul, A, B> &op) {
856  s << "(" << op.a << " * " << op.b << ")";
857  return s;
858 }
859 
860 template<typename A, typename B>
861 std::ostream &operator<<(std::ostream &s, const BinOp<Div, A, B> &op) {
862  s << "(" << op.a << " / " << op.b << ")";
863  return s;
864 }
865 
866 template<typename A, typename B>
867 std::ostream &operator<<(std::ostream &s, const BinOp<And, A, B> &op) {
868  s << "(" << op.a << " && " << op.b << ")";
869  return s;
870 }
871 
872 template<typename A, typename B>
873 std::ostream &operator<<(std::ostream &s, const BinOp<Or, A, B> &op) {
874  s << "(" << op.a << " || " << op.b << ")";
875  return s;
876 }
877 
878 template<typename A, typename B>
879 std::ostream &operator<<(std::ostream &s, const BinOp<Min, A, B> &op) {
880  s << "min(" << op.a << ", " << op.b << ")";
881  return s;
882 }
883 
884 template<typename A, typename B>
885 std::ostream &operator<<(std::ostream &s, const BinOp<Max, A, B> &op) {
886  s << "max(" << op.a << ", " << op.b << ")";
887  return s;
888 }
889 
890 template<typename A, typename B>
891 std::ostream &operator<<(std::ostream &s, const CmpOp<LE, A, B> &op) {
892  s << "(" << op.a << " <= " << op.b << ")";
893  return s;
894 }
895 
896 template<typename A, typename B>
897 std::ostream &operator<<(std::ostream &s, const CmpOp<LT, A, B> &op) {
898  s << "(" << op.a << " < " << op.b << ")";
899  return s;
900 }
901 
902 template<typename A, typename B>
903 std::ostream &operator<<(std::ostream &s, const CmpOp<GE, A, B> &op) {
904  s << "(" << op.a << " >= " << op.b << ")";
905  return s;
906 }
907 
908 template<typename A, typename B>
909 std::ostream &operator<<(std::ostream &s, const CmpOp<GT, A, B> &op) {
910  s << "(" << op.a << " > " << op.b << ")";
911  return s;
912 }
913 
914 template<typename A, typename B>
915 std::ostream &operator<<(std::ostream &s, const CmpOp<EQ, A, B> &op) {
916  s << "(" << op.a << " == " << op.b << ")";
917  return s;
918 }
919 
920 template<typename A, typename B>
921 std::ostream &operator<<(std::ostream &s, const CmpOp<NE, A, B> &op) {
922  s << "(" << op.a << " != " << op.b << ")";
923  return s;
924 }
925 
926 template<typename A, typename B>
927 std::ostream &operator<<(std::ostream &s, const BinOp<Mod, A, B> &op) {
928  s << "(" << op.a << " % " << op.b << ")";
929  return s;
930 }
931 
932 template<typename A, typename B>
933 HALIDE_ALWAYS_INLINE auto operator+(A &&a, B &&b) noexcept -> BinOp<Add, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
934  assert_is_lvalue_if_expr<A>();
935  assert_is_lvalue_if_expr<B>();
936  return {pattern_arg(a), pattern_arg(b)};
937 }
938 
939 template<typename A, typename B>
940 HALIDE_ALWAYS_INLINE auto add(A &&a, B &&b) -> decltype(IRMatcher::operator+(a, b)) {
941  assert_is_lvalue_if_expr<A>();
942  assert_is_lvalue_if_expr<B>();
943  return IRMatcher::operator+(a, b);
944 }
945 
946 template<>
948  t.lanes |= ((t.bits >= 32) && add_would_overflow(t.bits, a, b)) ? MatcherState::signed_integer_overflow : 0;
949  int dead_bits = 64 - t.bits;
950  // Drop the high bits then sign-extend them back
951  return int64_t((uint64_t(a) + uint64_t(b)) << dead_bits) >> dead_bits;
952 }
953 
954 template<>
956  uint64_t ones = (uint64_t)(-1);
957  return (a + b) & (ones >> (64 - t.bits));
958 }
959 
960 template<>
961 HALIDE_ALWAYS_INLINE double constant_fold_bin_op<Add>(halide_type_t &t, double a, double b) noexcept {
962  return a + b;
963 }
964 
965 template<typename A, typename B>
966 HALIDE_ALWAYS_INLINE auto operator-(A &&a, B &&b) noexcept -> BinOp<Sub, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
967  assert_is_lvalue_if_expr<A>();
968  assert_is_lvalue_if_expr<B>();
969  return {pattern_arg(a), pattern_arg(b)};
970 }
971 
972 template<typename A, typename B>
973 HALIDE_ALWAYS_INLINE auto sub(A &&a, B &&b) -> decltype(IRMatcher::operator-(a, b)) {
974  assert_is_lvalue_if_expr<A>();
975  assert_is_lvalue_if_expr<B>();
976  return IRMatcher::operator-(a, b);
977 }
978 
979 template<>
981  t.lanes |= ((t.bits >= 32) && sub_would_overflow(t.bits, a, b)) ? MatcherState::signed_integer_overflow : 0;
982  // Drop the high bits then sign-extend them back
983  int dead_bits = 64 - t.bits;
984  return int64_t((uint64_t(a) - uint64_t(b)) << dead_bits) >> dead_bits;
985 }
986 
987 template<>
989  uint64_t ones = (uint64_t)(-1);
990  return (a - b) & (ones >> (64 - t.bits));
991 }
992 
993 template<>
994 HALIDE_ALWAYS_INLINE double constant_fold_bin_op<Sub>(halide_type_t &t, double a, double b) noexcept {
995  return a - b;
996 }
997 
998 template<typename A, typename B>
999 HALIDE_ALWAYS_INLINE auto operator*(A &&a, B &&b) noexcept -> BinOp<Mul, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1000  assert_is_lvalue_if_expr<A>();
1001  assert_is_lvalue_if_expr<B>();
1002  return {pattern_arg(a), pattern_arg(b)};
1003 }
1004 
1005 template<typename A, typename B>
1006 HALIDE_ALWAYS_INLINE auto mul(A &&a, B &&b) -> decltype(IRMatcher::operator*(a, b)) {
1007  assert_is_lvalue_if_expr<A>();
1008  assert_is_lvalue_if_expr<B>();
1009  return IRMatcher::operator*(a, b);
1010 }
1011 
1012 template<>
1014  t.lanes |= ((t.bits >= 32) && mul_would_overflow(t.bits, a, b)) ? MatcherState::signed_integer_overflow : 0;
1015  int dead_bits = 64 - t.bits;
1016  // Drop the high bits then sign-extend them back
1017  return int64_t((uint64_t(a) * uint64_t(b)) << dead_bits) >> dead_bits;
1018 }
1019 
1020 template<>
1022  uint64_t ones = (uint64_t)(-1);
1023  return (a * b) & (ones >> (64 - t.bits));
1024 }
1025 
1026 template<>
1027 HALIDE_ALWAYS_INLINE double constant_fold_bin_op<Mul>(halide_type_t &t, double a, double b) noexcept {
1028  return a * b;
1029 }
1030 
1031 template<typename A, typename B>
1032 HALIDE_ALWAYS_INLINE auto operator/(A &&a, B &&b) noexcept -> BinOp<Div, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1033  assert_is_lvalue_if_expr<A>();
1034  assert_is_lvalue_if_expr<B>();
1035  return {pattern_arg(a), pattern_arg(b)};
1036 }
1037 
1038 template<typename A, typename B>
1039 HALIDE_ALWAYS_INLINE auto div(A &&a, B &&b) -> decltype(IRMatcher::operator/(a, b)) {
1040  return IRMatcher::operator/(a, b);
1041 }
1042 
1043 template<>
1045  return div_imp(a, b);
1046 }
1047 
1048 template<>
1050  return div_imp(a, b);
1051 }
1052 
1053 template<>
1054 HALIDE_ALWAYS_INLINE double constant_fold_bin_op<Div>(halide_type_t &t, double a, double b) noexcept {
1055  return div_imp(a, b);
1056 }
1057 
1058 template<typename A, typename B>
1059 HALIDE_ALWAYS_INLINE auto operator%(A &&a, B &&b) noexcept -> BinOp<Mod, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1060  assert_is_lvalue_if_expr<A>();
1061  assert_is_lvalue_if_expr<B>();
1062  return {pattern_arg(a), pattern_arg(b)};
1063 }
1064 
1065 template<typename A, typename B>
1066 HALIDE_ALWAYS_INLINE auto mod(A &&a, B &&b) -> decltype(IRMatcher::operator%(a, b)) {
1067  assert_is_lvalue_if_expr<A>();
1068  assert_is_lvalue_if_expr<B>();
1069  return IRMatcher::operator%(a, b);
1070 }
1071 
1072 template<>
1074  return mod_imp(a, b);
1075 }
1076 
1077 template<>
1079  return mod_imp(a, b);
1080 }
1081 
1082 template<>
1083 HALIDE_ALWAYS_INLINE double constant_fold_bin_op<Mod>(halide_type_t &t, double a, double b) noexcept {
1084  return mod_imp(a, b);
1085 }
1086 
1087 template<typename A, typename B>
1088 HALIDE_ALWAYS_INLINE auto min(A &&a, B &&b) noexcept -> BinOp<Min, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1089  assert_is_lvalue_if_expr<A>();
1090  assert_is_lvalue_if_expr<B>();
1091  return {pattern_arg(a), pattern_arg(b)};
1092 }
1093 
1094 template<>
1096  return std::min(a, b);
1097 }
1098 
1099 template<>
1101  return std::min(a, b);
1102 }
1103 
1104 template<>
1105 HALIDE_ALWAYS_INLINE double constant_fold_bin_op<Min>(halide_type_t &t, double a, double b) noexcept {
1106  return std::min(a, b);
1107 }
1108 
1109 template<typename A, typename B>
1110 HALIDE_ALWAYS_INLINE auto max(A &&a, B &&b) noexcept -> BinOp<Max, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1111  assert_is_lvalue_if_expr<A>();
1112  assert_is_lvalue_if_expr<B>();
1113  return {pattern_arg(std::forward<A>(a)), pattern_arg(std::forward<B>(b))};
1114 }
1115 
1116 template<>
1118  return std::max(a, b);
1119 }
1120 
1121 template<>
1123  return std::max(a, b);
1124 }
1125 
1126 template<>
1127 HALIDE_ALWAYS_INLINE double constant_fold_bin_op<Max>(halide_type_t &t, double a, double b) noexcept {
1128  return std::max(a, b);
1129 }
1130 
1131 template<typename A, typename B>
1132 HALIDE_ALWAYS_INLINE auto operator<(A &&a, B &&b) noexcept -> CmpOp<LT, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1133  return {pattern_arg(a), pattern_arg(b)};
1134 }
1135 
1136 template<typename A, typename B>
1137 HALIDE_ALWAYS_INLINE auto lt(A &&a, B &&b) -> decltype(IRMatcher::operator<(a, b)) {
1138  return IRMatcher::operator<(a, b);
1139 }
1140 
1141 template<>
1143  return a < b;
1144 }
1145 
1146 template<>
1148  return a < b;
1149 }
1150 
1151 template<>
1153  return a < b;
1154 }
1155 
1156 template<typename A, typename B>
1157 HALIDE_ALWAYS_INLINE auto operator>(A &&a, B &&b) noexcept -> CmpOp<GT, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1158  return {pattern_arg(a), pattern_arg(b)};
1159 }
1160 
1161 template<typename A, typename B>
1162 HALIDE_ALWAYS_INLINE auto gt(A &&a, B &&b) -> decltype(IRMatcher::operator>(a, b)) {
1163  return IRMatcher::operator>(a, b);
1164 }
1165 
1166 template<>
1168  return a > b;
1169 }
1170 
1171 template<>
1173  return a > b;
1174 }
1175 
1176 template<>
1178  return a > b;
1179 }
1180 
1181 template<typename A, typename B>
1182 HALIDE_ALWAYS_INLINE auto operator<=(A &&a, B &&b) noexcept -> CmpOp<LE, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1183  return {pattern_arg(a), pattern_arg(b)};
1184 }
1185 
1186 template<typename A, typename B>
1187 HALIDE_ALWAYS_INLINE auto le(A &&a, B &&b) -> decltype(IRMatcher::operator<=(a, b)) {
1188  return IRMatcher::operator<=(a, b);
1189 }
1190 
1191 template<>
1193  return a <= b;
1194 }
1195 
1196 template<>
1198  return a <= b;
1199 }
1200 
1201 template<>
1203  return a <= b;
1204 }
1205 
1206 template<typename A, typename B>
1207 HALIDE_ALWAYS_INLINE auto operator>=(A &&a, B &&b) noexcept -> CmpOp<GE, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1208  return {pattern_arg(a), pattern_arg(b)};
1209 }
1210 
1211 template<typename A, typename B>
1212 HALIDE_ALWAYS_INLINE auto ge(A &&a, B &&b) -> decltype(IRMatcher::operator>=(a, b)) {
1213  return IRMatcher::operator>=(a, b);
1214 }
1215 
1216 template<>
1218  return a >= b;
1219 }
1220 
1221 template<>
1223  return a >= b;
1224 }
1225 
1226 template<>
1228  return a >= b;
1229 }
1230 
1231 template<typename A, typename B>
1232 HALIDE_ALWAYS_INLINE auto operator==(A &&a, B &&b) noexcept -> CmpOp<EQ, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1233  return {pattern_arg(a), pattern_arg(b)};
1234 }
1235 
1236 template<typename A, typename B>
1237 HALIDE_ALWAYS_INLINE auto eq(A &&a, B &&b) -> decltype(IRMatcher::operator==(a, b)) {
1238  return IRMatcher::operator==(a, b);
1239 }
1240 
1241 template<>
1243  return a == b;
1244 }
1245 
1246 template<>
1248  return a == b;
1249 }
1250 
1251 template<>
1253  return a == b;
1254 }
1255 
1256 template<typename A, typename B>
1257 HALIDE_ALWAYS_INLINE auto operator!=(A &&a, B &&b) noexcept -> CmpOp<NE, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1258  return {pattern_arg(a), pattern_arg(b)};
1259 }
1260 
1261 template<typename A, typename B>
1262 HALIDE_ALWAYS_INLINE auto ne(A &&a, B &&b) -> decltype(IRMatcher::operator!=(a, b)) {
1263  return IRMatcher::operator!=(a, b);
1264 }
1265 
1266 template<>
1268  return a != b;
1269 }
1270 
1271 template<>
1273  return a != b;
1274 }
1275 
1276 template<>
1278  return a != b;
1279 }
1280 
1281 template<typename A, typename B>
1282 HALIDE_ALWAYS_INLINE auto operator||(A &&a, B &&b) noexcept -> BinOp<Or, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1283  return {pattern_arg(a), pattern_arg(b)};
1284 }
1285 
1286 template<typename A, typename B>
1287 HALIDE_ALWAYS_INLINE auto or_op(A &&a, B &&b) -> decltype(IRMatcher::operator||(a, b)) {
1288  return IRMatcher::operator||(a, b);
1289 }
1290 
1291 template<>
1293  return (a | b) & 1;
1294 }
1295 
1296 template<>
1298  return (a | b) & 1;
1299 }
1300 
1301 template<>
1302 HALIDE_ALWAYS_INLINE double constant_fold_bin_op<Or>(halide_type_t &t, double a, double b) noexcept {
1303  // Unreachable, as it would be a type mismatch.
1304  return 0;
1305 }
1306 
1307 template<typename A, typename B>
1308 HALIDE_ALWAYS_INLINE auto operator&&(A &&a, B &&b) noexcept -> BinOp<And, decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1309  return {pattern_arg(a), pattern_arg(b)};
1310 }
1311 
1312 template<typename A, typename B>
1313 HALIDE_ALWAYS_INLINE auto and_op(A &&a, B &&b) -> decltype(IRMatcher::operator&&(a, b)) {
1314  return IRMatcher::operator&&(a, b);
1315 }
1316 
1317 template<>
1319  return a & b & 1;
1320 }
1321 
1322 template<>
1324  return a & b & 1;
1325 }
1326 
1327 template<>
1328 HALIDE_ALWAYS_INLINE double constant_fold_bin_op<And>(halide_type_t &t, double a, double b) noexcept {
1329  // Unreachable
1330  return 0;
1331 }
1332 
1333 constexpr inline uint32_t bitwise_or_reduce() {
1334  return 0;
1335 }
1336 
1337 template<typename... Args>
1338 constexpr uint32_t bitwise_or_reduce(uint32_t first, Args... rest) {
1339  return first | bitwise_or_reduce(rest...);
1340 }
1341 
1342 constexpr inline bool and_reduce() {
1343  return true;
1344 }
1345 
1346 template<typename... Args>
1347 constexpr bool and_reduce(bool first, Args... rest) {
1348  return first && and_reduce(rest...);
1349 }
1350 
1351 // TODO: this can be replaced with std::min() once we require C++14 or later
1352 constexpr int const_min(int a, int b) {
1353  return a < b ? a : b;
1354 }
1355 
1356 template<typename... Args>
1357 struct Intrin {
1358  struct pattern_tag {};
1360  std::tuple<Args...> args;
1361  // The type of the output of the intrinsic node.
1362  // Only necessary in cases where it can't be inferred
1363  // from the input types (e.g. saturating_cast).
1365 
1367 
1370  constexpr static bool canonical = and_reduce((Args::canonical)...);
1371 
1372  template<int i,
1373  uint32_t bound,
1374  typename = typename std::enable_if<(i < sizeof...(Args))>::type>
1375  HALIDE_ALWAYS_INLINE bool match_args(int, const Call &c, MatcherState &state) const noexcept {
1376  using T = decltype(std::get<i>(args));
1377  return (std::get<i>(args).template match<bound>(*c.args[i].get(), state) &&
1378  match_args<i + 1, bound | bindings<T>::mask>(0, c, state));
1379  }
1380 
1381  template<int i, uint32_t binds>
1382  HALIDE_ALWAYS_INLINE bool match_args(double, const Call &c, MatcherState &state) const noexcept {
1383  return true;
1384  }
1385 
1386  template<uint32_t bound>
1387  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
1388  if (e.node_type != IRNodeType::Call) {
1389  return false;
1390  }
1391  const Call &c = (const Call &)e;
1392  return (c.is_intrinsic(intrin) &&
1393  ((optional_type_hint == Type()) || optional_type_hint == e.type) &&
1394  match_args<0, bound>(0, c, state));
1395  }
1396 
1397  template<int i,
1398  typename = typename std::enable_if<(i < sizeof...(Args))>::type>
1399  HALIDE_ALWAYS_INLINE void print_args(int, std::ostream &s) const {
1400  s << std::get<i>(args);
1401  if (i + 1 < sizeof...(Args)) {
1402  s << ", ";
1403  }
1404  print_args<i + 1>(0, s);
1405  }
1406 
1407  template<int i>
1408  HALIDE_ALWAYS_INLINE void print_args(double, std::ostream &s) const {
1409  }
1410 
1412  void print_args(std::ostream &s) const {
1413  print_args<0>(0, s);
1414  }
1415 
1417  Expr make(MatcherState &state, halide_type_t type_hint) const {
1418  Expr arg0 = std::get<0>(args).make(state, type_hint);
1419  if (intrin == Call::likely) {
1420  return likely(arg0);
1421  } else if (intrin == Call::likely_if_innermost) {
1422  return likely_if_innermost(arg0);
1423  } else if (intrin == Call::abs) {
1424  return abs(arg0);
1425  } else if (intrin == Call::saturating_cast) {
1426  return saturating_cast(optional_type_hint, arg0);
1427  }
1428 
1429  Expr arg1 = std::get<const_min(1, sizeof...(Args) - 1)>(args).make(state, type_hint);
1430  if (intrin == Call::absd) {
1431  return absd(arg0, arg1);
1432  } else if (intrin == Call::widen_right_add) {
1433  return widen_right_add(arg0, arg1);
1434  } else if (intrin == Call::widen_right_mul) {
1435  return widen_right_mul(arg0, arg1);
1436  } else if (intrin == Call::widen_right_sub) {
1437  return widen_right_sub(arg0, arg1);
1438  } else if (intrin == Call::widening_add) {
1439  return widening_add(arg0, arg1);
1440  } else if (intrin == Call::widening_sub) {
1441  return widening_sub(arg0, arg1);
1442  } else if (intrin == Call::widening_mul) {
1443  return widening_mul(arg0, arg1);
1444  } else if (intrin == Call::saturating_add) {
1445  return saturating_add(arg0, arg1);
1446  } else if (intrin == Call::saturating_sub) {
1447  return saturating_sub(arg0, arg1);
1448  } else if (intrin == Call::halving_add) {
1449  return halving_add(arg0, arg1);
1450  } else if (intrin == Call::halving_sub) {
1451  return halving_sub(arg0, arg1);
1452  } else if (intrin == Call::rounding_halving_add) {
1453  return rounding_halving_add(arg0, arg1);
1454  } else if (intrin == Call::shift_left) {
1455  return arg0 << arg1;
1456  } else if (intrin == Call::shift_right) {
1457  return arg0 >> arg1;
1458  } else if (intrin == Call::rounding_shift_left) {
1459  return rounding_shift_left(arg0, arg1);
1460  } else if (intrin == Call::rounding_shift_right) {
1461  return rounding_shift_right(arg0, arg1);
1462  }
1463 
1464  Expr arg2 = std::get<const_min(2, sizeof...(Args) - 1)>(args).make(state, type_hint);
1465  if (intrin == Call::mul_shift_right) {
1466  return mul_shift_right(arg0, arg1, arg2);
1467  } else if (intrin == Call::rounding_mul_shift_right) {
1468  return rounding_mul_shift_right(arg0, arg1, arg2);
1469  }
1470 
1471  internal_error << "Unhandled intrinsic in IRMatcher: " << intrin;
1472  return Expr();
1473  }
1474 
1475  constexpr static bool foldable = true;
1476 
1478  halide_scalar_value_t arg1;
1479  // Assuming the args have the same type as the intrinsic is incorrect in
1480  // general. But for the intrinsics we can fold (just shifts), the LHS
1481  // has the same type as the intrinsic, and we can always treat the RHS
1482  // as a signed int, because we're using 64 bits for it.
1483  std::get<0>(args).make_folded_const(val, ty, state);
1484  halide_type_t signed_ty = ty;
1485  signed_ty.code = halide_type_int;
1486  // We can just directly get the second arg here, because we only want to
1487  // instantiate this method for shifts, which have two args.
1488  std::get<1>(args).make_folded_const(arg1, signed_ty, state);
1489 
1490  if (intrin == Call::shift_left) {
1491  if (arg1.u.i64 < 0) {
1492  if (ty.code == halide_type_int) {
1493  // Arithmetic shift
1494  val.u.i64 >>= -arg1.u.i64;
1495  } else {
1496  // Logical shift
1497  val.u.u64 >>= -arg1.u.i64;
1498  }
1499  } else {
1500  val.u.u64 <<= arg1.u.i64;
1501  }
1502  } else if (intrin == Call::shift_right) {
1503  if (arg1.u.i64 > 0) {
1504  if (ty.code == halide_type_int) {
1505  // Arithmetic shift
1506  val.u.i64 >>= arg1.u.i64;
1507  } else {
1508  // Logical shift
1509  val.u.u64 >>= arg1.u.i64;
1510  }
1511  } else {
1512  val.u.u64 <<= -arg1.u.i64;
1513  }
1514  } else {
1515  internal_error << "Folding not implemented for intrinsic: " << intrin;
1516  }
1517  }
1518 
1521  : intrin(intrin), args(args...) {
1522  }
1523 };
1524 
1525 template<typename... Args>
1526 std::ostream &operator<<(std::ostream &s, const Intrin<Args...> &op) {
1527  s << op.intrin << "(";
1528  op.print_args(s);
1529  s << ")";
1530  return s;
1531 }
1532 
1533 template<typename... Args>
1534 HALIDE_ALWAYS_INLINE auto intrin(Call::IntrinsicOp intrinsic_op, Args... args) noexcept -> Intrin<decltype(pattern_arg(args))...> {
1535  return {intrinsic_op, pattern_arg(args)...};
1536 }
1537 
1538 template<typename A, typename B>
1539 auto widen_right_add(A &&a, B &&b) noexcept -> Intrin<decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1541 }
1542 template<typename A, typename B>
1543 auto widen_right_mul(A &&a, B &&b) noexcept -> Intrin<decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1545 }
1546 template<typename A, typename B>
1547 auto widen_right_sub(A &&a, B &&b) noexcept -> Intrin<decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1549 }
1550 
1551 template<typename A, typename B>
1552 auto widening_add(A &&a, B &&b) noexcept -> Intrin<decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1553  return {Call::widening_add, pattern_arg(a), pattern_arg(b)};
1554 }
1555 template<typename A, typename B>
1556 auto widening_sub(A &&a, B &&b) noexcept -> Intrin<decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1557  return {Call::widening_sub, pattern_arg(a), pattern_arg(b)};
1558 }
1559 template<typename A, typename B>
1560 auto widening_mul(A &&a, B &&b) noexcept -> Intrin<decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1561  return {Call::widening_mul, pattern_arg(a), pattern_arg(b)};
1562 }
1563 template<typename A, typename B>
1564 auto saturating_add(A &&a, B &&b) noexcept -> Intrin<decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1565  return {Call::saturating_add, pattern_arg(a), pattern_arg(b)};
1566 }
1567 template<typename A, typename B>
1568 auto saturating_sub(A &&a, B &&b) noexcept -> Intrin<decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1569  return {Call::saturating_sub, pattern_arg(a), pattern_arg(b)};
1570 }
1571 template<typename A>
1572 auto saturating_cast(const Type &t, A &&a) noexcept -> Intrin<decltype(pattern_arg(a))> {
1573  Intrin<decltype(pattern_arg(a))> p = {Call::saturating_cast, pattern_arg(a)};
1574  p.optional_type_hint = t;
1575  return p;
1576 }
1577 template<typename A, typename B>
1578 auto halving_add(A &&a, B &&b) noexcept -> Intrin<decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1579  return {Call::halving_add, pattern_arg(a), pattern_arg(b)};
1580 }
1581 template<typename A, typename B>
1582 auto halving_sub(A &&a, B &&b) noexcept -> Intrin<decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1583  return {Call::halving_sub, pattern_arg(a), pattern_arg(b)};
1584 }
1585 template<typename A, typename B>
1586 auto rounding_halving_add(A &&a, B &&b) noexcept -> Intrin<decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1588 }
1589 template<typename A, typename B>
1590 auto shift_left(A &&a, B &&b) noexcept -> Intrin<decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1591  return {Call::shift_left, pattern_arg(a), pattern_arg(b)};
1592 }
1593 template<typename A, typename B>
1594 auto shift_right(A &&a, B &&b) noexcept -> Intrin<decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1595  return {Call::shift_right, pattern_arg(a), pattern_arg(b)};
1596 }
1597 template<typename A, typename B>
1598 auto rounding_shift_left(A &&a, B &&b) noexcept -> Intrin<decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1600 }
1601 template<typename A, typename B>
1602 auto rounding_shift_right(A &&a, B &&b) noexcept -> Intrin<decltype(pattern_arg(a)), decltype(pattern_arg(b))> {
1604 }
1605 template<typename A, typename B, typename C>
1606 auto mul_shift_right(A &&a, B &&b, C &&c) noexcept -> Intrin<decltype(pattern_arg(a)), decltype(pattern_arg(b)), decltype(pattern_arg(c))> {
1608 }
1609 template<typename A, typename B, typename C>
1610 auto rounding_mul_shift_right(A &&a, B &&b, C &&c) noexcept -> Intrin<decltype(pattern_arg(a)), decltype(pattern_arg(b)), decltype(pattern_arg(c))> {
1612 }
1613 
1614 template<typename A>
1615 struct NotOp {
1616  struct pattern_tag {};
1617  A a;
1618 
1619  constexpr static uint32_t binds = bindings<A>::mask;
1620 
1623  constexpr static bool canonical = A::canonical;
1624 
1625  template<uint32_t bound>
1626  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
1627  if (e.node_type != IRNodeType::Not) {
1628  return false;
1629  }
1630  const Not &op = (const Not &)e;
1631  return (a.template match<bound>(*op.a.get(), state));
1632  }
1633 
1634  template<uint32_t bound, typename A2>
1635  HALIDE_ALWAYS_INLINE bool match(const NotOp<A2> &op, MatcherState &state) const noexcept {
1636  return a.template match<bound>(unwrap(op.a), state);
1637  }
1638 
1640  Expr make(MatcherState &state, halide_type_t type_hint) const {
1641  return Not::make(a.make(state, type_hint));
1642  }
1643 
1644  constexpr static bool foldable = A::foldable;
1645 
1646  template<typename A1 = A>
1648  a.make_folded_const(val, ty, state);
1649  val.u.u64 = ~val.u.u64;
1650  val.u.u64 &= 1;
1651  }
1652 };
1653 
1654 template<typename A>
1655 HALIDE_ALWAYS_INLINE auto operator!(A &&a) noexcept -> NotOp<decltype(pattern_arg(a))> {
1656  assert_is_lvalue_if_expr<A>();
1657  return {pattern_arg(a)};
1658 }
1659 
1660 template<typename A>
1661 HALIDE_ALWAYS_INLINE auto not_op(A &&a) -> decltype(IRMatcher::operator!(a)) {
1662  assert_is_lvalue_if_expr<A>();
1663  return IRMatcher::operator!(a);
1664 }
1665 
1666 template<typename A>
1667 inline std::ostream &operator<<(std::ostream &s, const NotOp<A> &op) {
1668  s << "!(" << op.a << ")";
1669  return s;
1670 }
1671 
1672 template<typename C, typename T, typename F>
1673 struct SelectOp {
1674  struct pattern_tag {};
1675  C c;
1676  T t;
1677  F f;
1678 
1680 
1683 
1684  constexpr static bool canonical = C::canonical && T::canonical && F::canonical;
1685 
1686  template<uint32_t bound>
1687  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
1688  if (e.node_type != Select::_node_type) {
1689  return false;
1690  }
1691  const Select &op = (const Select &)e;
1692  return (c.template match<bound>(*op.condition.get(), state) &&
1693  t.template match<bound | bindings<C>::mask>(*op.true_value.get(), state) &&
1694  f.template match<bound | bindings<C>::mask | bindings<T>::mask>(*op.false_value.get(), state));
1695  }
1696  template<uint32_t bound, typename C2, typename T2, typename F2>
1697  HALIDE_ALWAYS_INLINE bool match(const SelectOp<C2, T2, F2> &instance, MatcherState &state) const noexcept {
1698  return (c.template match<bound>(unwrap(instance.c), state) &&
1699  t.template match<bound | bindings<C>::mask>(unwrap(instance.t), state) &&
1700  f.template match<bound | bindings<C>::mask | bindings<T>::mask>(unwrap(instance.f), state));
1701  }
1702 
1704  Expr make(MatcherState &state, halide_type_t type_hint) const {
1705  return Select::make(c.make(state, {}), t.make(state, type_hint), f.make(state, type_hint));
1706  }
1707 
1708  constexpr static bool foldable = C::foldable && T::foldable && F::foldable;
1709 
1710  template<typename C1 = C>
1712  halide_scalar_value_t c_val, t_val, f_val;
1713  halide_type_t c_ty;
1714  c.make_folded_const(c_val, c_ty, state);
1715  if ((c_val.u.u64 & 1) == 1) {
1716  t.make_folded_const(val, ty, state);
1717  } else {
1718  f.make_folded_const(val, ty, state);
1719  }
1720  ty.lanes |= c_ty.lanes & MatcherState::special_values_mask;
1721  }
1722 };
1723 
1724 template<typename C, typename T, typename F>
1725 std::ostream &operator<<(std::ostream &s, const SelectOp<C, T, F> &op) {
1726  s << "select(" << op.c << ", " << op.t << ", " << op.f << ")";
1727  return s;
1728 }
1729 
1730 template<typename C, typename T, typename F>
1731 HALIDE_ALWAYS_INLINE auto select(C &&c, T &&t, F &&f) noexcept -> SelectOp<decltype(pattern_arg(c)), decltype(pattern_arg(t)), decltype(pattern_arg(f))> {
1732  assert_is_lvalue_if_expr<C>();
1733  assert_is_lvalue_if_expr<T>();
1734  assert_is_lvalue_if_expr<F>();
1735  return {pattern_arg(c), pattern_arg(t), pattern_arg(f)};
1736 }
1737 
1738 template<typename A, typename B>
1739 struct BroadcastOp {
1740  struct pattern_tag {};
1741  A a;
1743 
1745 
1748 
1749  constexpr static bool canonical = A::canonical && B::canonical;
1750 
1751  template<uint32_t bound>
1752  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
1753  if (e.node_type == Broadcast::_node_type) {
1754  const Broadcast &op = (const Broadcast &)e;
1755  if (a.template match<bound>(*op.value.get(), state) &&
1756  lanes.template match<bound>(op.lanes, state)) {
1757  return true;
1758  }
1759  }
1760  return false;
1761  }
1762 
1763  template<uint32_t bound, typename A2, typename B2>
1764  HALIDE_ALWAYS_INLINE bool match(const BroadcastOp<A2, B2> &op, MatcherState &state) const noexcept {
1765  return (a.template match<bound>(unwrap(op.a), state) &&
1766  lanes.template match<bound | bindings<A>::mask>(unwrap(op.lanes), state));
1767  }
1768 
1770  Expr make(MatcherState &state, halide_type_t type_hint) const {
1771  halide_scalar_value_t lanes_val;
1772  halide_type_t ty;
1773  lanes.make_folded_const(lanes_val, ty, state);
1774  int32_t l = (int32_t)lanes_val.u.i64;
1775  type_hint.lanes /= l;
1776  Expr val = a.make(state, type_hint);
1777  if (l == 1) {
1778  return val;
1779  } else {
1780  return Broadcast::make(std::move(val), l);
1781  }
1782  }
1783 
1784  constexpr static bool foldable = false;
1785 
1786  template<typename A1 = A>
1788  halide_scalar_value_t lanes_val;
1789  halide_type_t lanes_ty;
1790  lanes.make_folded_const(lanes_val, lanes_ty, state);
1791  uint16_t l = (uint16_t)lanes_val.u.i64;
1792  a.make_folded_const(val, ty, state);
1793  ty.lanes = l | (ty.lanes & MatcherState::special_values_mask);
1794  }
1795 };
1796 
1797 template<typename A, typename B>
1798 inline std::ostream &operator<<(std::ostream &s, const BroadcastOp<A, B> &op) {
1799  s << "broadcast(" << op.a << ", " << op.lanes << ")";
1800  return s;
1801 }
1802 
1803 template<typename A, typename B>
1804 HALIDE_ALWAYS_INLINE auto broadcast(A &&a, B lanes) noexcept -> BroadcastOp<decltype(pattern_arg(a)), decltype(pattern_arg(lanes))> {
1805  assert_is_lvalue_if_expr<A>();
1806  return {pattern_arg(a), pattern_arg(lanes)};
1807 }
1808 
1809 template<typename A, typename B, typename C>
1810 struct RampOp {
1811  struct pattern_tag {};
1812  A a;
1813  B b;
1815 
1817 
1820 
1821  constexpr static bool canonical = A::canonical && B::canonical && C::canonical;
1822 
1823  template<uint32_t bound>
1824  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
1825  if (e.node_type != Ramp::_node_type) {
1826  return false;
1827  }
1828  const Ramp &op = (const Ramp &)e;
1829  if (a.template match<bound>(*op.base.get(), state) &&
1830  b.template match<bound | bindings<A>::mask>(*op.stride.get(), state) &&
1831  lanes.template match<bound | bindings<A>::mask | bindings<B>::mask>(op.lanes, state)) {
1832  return true;
1833  } else {
1834  return false;
1835  }
1836  }
1837 
1838  template<uint32_t bound, typename A2, typename B2, typename C2>
1839  HALIDE_ALWAYS_INLINE bool match(const RampOp<A2, B2, C2> &op, MatcherState &state) const noexcept {
1840  return (a.template match<bound>(unwrap(op.a), state) &&
1841  b.template match<bound | bindings<A>::mask>(unwrap(op.b), state) &&
1842  lanes.template match<bound | bindings<A>::mask | bindings<B>::mask>(unwrap(op.lanes), state));
1843  }
1844 
1846  Expr make(MatcherState &state, halide_type_t type_hint) const {
1847  halide_scalar_value_t lanes_val;
1848  halide_type_t ty;
1849  lanes.make_folded_const(lanes_val, ty, state);
1850  int32_t l = (int32_t)lanes_val.u.i64;
1851  type_hint.lanes /= l;
1852  Expr ea, eb;
1853  eb = b.make(state, type_hint);
1854  ea = a.make(state, eb.type());
1855  return Ramp::make(ea, eb, l);
1856  }
1857 
1858  constexpr static bool foldable = false;
1859 };
1860 
1861 template<typename A, typename B, typename C>
1862 std::ostream &operator<<(std::ostream &s, const RampOp<A, B, C> &op) {
1863  s << "ramp(" << op.a << ", " << op.b << ", " << op.lanes << ")";
1864  return s;
1865 }
1866 
1867 template<typename A, typename B, typename C>
1868 HALIDE_ALWAYS_INLINE auto ramp(A &&a, B &&b, C &&c) noexcept -> RampOp<decltype(pattern_arg(a)), decltype(pattern_arg(b)), decltype(pattern_arg(c))> {
1869  assert_is_lvalue_if_expr<A>();
1870  assert_is_lvalue_if_expr<B>();
1871  assert_is_lvalue_if_expr<C>();
1872  return {pattern_arg(a), pattern_arg(b), pattern_arg(c)};
1873 }
1874 
1875 template<typename A, typename B, VectorReduce::Operator reduce_op>
1877  struct pattern_tag {};
1878  A a;
1880 
1882 
1885  constexpr static bool canonical = A::canonical;
1886 
1887  template<uint32_t bound>
1888  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
1889  if (e.node_type == VectorReduce::_node_type) {
1890  const VectorReduce &op = (const VectorReduce &)e;
1891  if (op.op == reduce_op &&
1892  a.template match<bound>(*op.value.get(), state) &&
1893  lanes.template match<bound | bindings<A>::mask>(op.type.lanes(), state)) {
1894  return true;
1895  }
1896  }
1897  return false;
1898  }
1899 
1900  template<uint32_t bound, typename A2, typename B2, VectorReduce::Operator reduce_op_2>
1902  return (reduce_op == reduce_op_2 &&
1903  a.template match<bound>(unwrap(op.a), state) &&
1904  lanes.template match<bound | bindings<A>::mask>(unwrap(op.lanes), state));
1905  }
1906 
1908  Expr make(MatcherState &state, halide_type_t type_hint) const {
1909  halide_scalar_value_t lanes_val;
1910  halide_type_t ty;
1911  lanes.make_folded_const(lanes_val, ty, state);
1912  int l = (int)lanes_val.u.i64;
1913  return VectorReduce::make(reduce_op, a.make(state, type_hint), l);
1914  }
1915 
1916  constexpr static bool foldable = false;
1917 };
1918 
1919 template<typename A, typename B, VectorReduce::Operator reduce_op>
1920 inline std::ostream &operator<<(std::ostream &s, const VectorReduceOp<A, B, reduce_op> &op) {
1921  s << "vector_reduce(" << reduce_op << ", " << op.a << ", " << op.lanes << ")";
1922  return s;
1923 }
1924 
1925 template<typename A, typename B>
1926 HALIDE_ALWAYS_INLINE auto h_add(A &&a, B lanes) noexcept -> VectorReduceOp<decltype(pattern_arg(a)), decltype(pattern_arg(lanes)), VectorReduce::Add> {
1927  assert_is_lvalue_if_expr<A>();
1928  return {pattern_arg(a), pattern_arg(lanes)};
1929 }
1930 
1931 template<typename A, typename B>
1932 HALIDE_ALWAYS_INLINE auto h_min(A &&a, B lanes) noexcept -> VectorReduceOp<decltype(pattern_arg(a)), decltype(pattern_arg(lanes)), VectorReduce::Min> {
1933  assert_is_lvalue_if_expr<A>();
1934  return {pattern_arg(a), pattern_arg(lanes)};
1935 }
1936 
1937 template<typename A, typename B>
1938 HALIDE_ALWAYS_INLINE auto h_max(A &&a, B lanes) noexcept -> VectorReduceOp<decltype(pattern_arg(a)), decltype(pattern_arg(lanes)), VectorReduce::Max> {
1939  assert_is_lvalue_if_expr<A>();
1940  return {pattern_arg(a), pattern_arg(lanes)};
1941 }
1942 
1943 template<typename A, typename B>
1944 HALIDE_ALWAYS_INLINE auto h_and(A &&a, B lanes) noexcept -> VectorReduceOp<decltype(pattern_arg(a)), decltype(pattern_arg(lanes)), VectorReduce::And> {
1945  assert_is_lvalue_if_expr<A>();
1946  return {pattern_arg(a), pattern_arg(lanes)};
1947 }
1948 
1949 template<typename A, typename B>
1950 HALIDE_ALWAYS_INLINE auto h_or(A &&a, B lanes) noexcept -> VectorReduceOp<decltype(pattern_arg(a)), decltype(pattern_arg(lanes)), VectorReduce::Or> {
1951  assert_is_lvalue_if_expr<A>();
1952  return {pattern_arg(a), pattern_arg(lanes)};
1953 }
1954 
1955 template<typename A>
1956 struct NegateOp {
1957  struct pattern_tag {};
1958  A a;
1959 
1960  constexpr static uint32_t binds = bindings<A>::mask;
1961 
1964 
1965  constexpr static bool canonical = A::canonical;
1966 
1967  template<uint32_t bound>
1968  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
1969  if (e.node_type != Sub::_node_type) {
1970  return false;
1971  }
1972  const Sub &op = (const Sub &)e;
1973  return (a.template match<bound>(*op.b.get(), state) &&
1974  is_const_zero(op.a));
1975  }
1976 
1977  template<uint32_t bound, typename A2>
1978  HALIDE_ALWAYS_INLINE bool match(NegateOp<A2> &&p, MatcherState &state) const noexcept {
1979  return a.template match<bound>(unwrap(p.a), state);
1980  }
1981 
1983  Expr make(MatcherState &state, halide_type_t type_hint) const {
1984  Expr ea = a.make(state, type_hint);
1985  Expr z = make_zero(ea.type());
1986  return Sub::make(std::move(z), std::move(ea));
1987  }
1988 
1989  constexpr static bool foldable = A::foldable;
1990 
1991  template<typename A1 = A>
1993  a.make_folded_const(val, ty, state);
1994  int dead_bits = 64 - ty.bits;
1995  switch (ty.code) {
1996  case halide_type_int:
1997  if (ty.bits >= 32 && val.u.u64 && (val.u.u64 << (65 - ty.bits)) == 0) {
1998  // Trying to negate the most negative signed int for a no-overflow type.
2000  } else {
2001  // Negate, drop the high bits, and then sign-extend them back
2002  val.u.i64 = int64_t(uint64_t(-val.u.i64) << dead_bits) >> dead_bits;
2003  }
2004  break;
2005  case halide_type_uint:
2006  val.u.u64 = ((-val.u.u64) << dead_bits) >> dead_bits;
2007  break;
2008  case halide_type_float:
2009  case halide_type_bfloat:
2010  val.u.f64 = -val.u.f64;
2011  break;
2012  default:
2013  // unreachable
2014  ;
2015  }
2016  }
2017 };
2018 
2019 template<typename A>
2020 std::ostream &operator<<(std::ostream &s, const NegateOp<A> &op) {
2021  s << "-" << op.a;
2022  return s;
2023 }
2024 
2025 template<typename A>
2026 HALIDE_ALWAYS_INLINE auto operator-(A &&a) noexcept -> NegateOp<decltype(pattern_arg(a))> {
2027  assert_is_lvalue_if_expr<A>();
2028  return {pattern_arg(a)};
2029 }
2030 
2031 template<typename A>
2032 HALIDE_ALWAYS_INLINE auto negate(A &&a) -> decltype(IRMatcher::operator-(a)) {
2033  assert_is_lvalue_if_expr<A>();
2034  return IRMatcher::operator-(a);
2035 }
2036 
2037 template<typename A>
2038 struct CastOp {
2039  struct pattern_tag {};
2041  A a;
2042 
2043  constexpr static uint32_t binds = bindings<A>::mask;
2044 
2047  constexpr static bool canonical = A::canonical;
2048 
2049  template<uint32_t bound>
2050  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
2051  if (e.node_type != Cast::_node_type) {
2052  return false;
2053  }
2054  const Cast &op = (const Cast &)e;
2055  return (e.type == t &&
2056  a.template match<bound>(*op.value.get(), state));
2057  }
2058  template<uint32_t bound, typename A2>
2059  HALIDE_ALWAYS_INLINE bool match(const CastOp<A2> &op, MatcherState &state) const noexcept {
2060  return t == op.t && a.template match<bound>(unwrap(op.a), state);
2061  }
2062 
2064  Expr make(MatcherState &state, halide_type_t type_hint) const {
2065  return cast(t, a.make(state, {}));
2066  }
2067 
2068  constexpr static bool foldable = false;
2069 };
2070 
2071 template<typename A>
2072 std::ostream &operator<<(std::ostream &s, const CastOp<A> &op) {
2073  s << "cast(" << op.t << ", " << op.a << ")";
2074  return s;
2075 }
2076 
2077 template<typename A>
2078 HALIDE_ALWAYS_INLINE auto cast(halide_type_t t, A &&a) noexcept -> CastOp<decltype(pattern_arg(a))> {
2079  assert_is_lvalue_if_expr<A>();
2080  return {t, pattern_arg(a)};
2081 }
2082 
2083 template<typename Vec, typename Base, typename Stride, typename Lanes>
2084 struct SliceOp {
2085  struct pattern_tag {};
2086  Vec vec;
2087  Base base;
2088  Stride stride;
2089  Lanes lanes;
2090 
2091  static constexpr uint32_t binds = Vec::binds | Base::binds | Stride::binds | Lanes::binds;
2092 
2095  constexpr static bool canonical = Vec::canonical && Base::canonical && Stride::canonical && Lanes::canonical;
2096 
2097  template<uint32_t bound>
2098  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
2099  if (e.node_type != IRNodeType::Shuffle) {
2100  return false;
2101  }
2102  const Shuffle &v = (const Shuffle &)e;
2103  return v.vectors.size() == 1 &&
2104  vec.template match<bound>(*v.vectors[0].get(), state) &&
2105  base.template match<bound | bindings<Vec>::mask>(v.slice_begin(), state) &&
2106  stride.template match<bound | bindings<Vec>::mask | bindings<Base>::mask>(v.slice_stride(), state) &&
2108  }
2109 
2111  Expr make(MatcherState &state, halide_type_t type_hint) const {
2112  halide_scalar_value_t base_val, stride_val, lanes_val;
2113  halide_type_t ty;
2114  base.make_folded_const(base_val, ty, state);
2115  int b = (int)base_val.u.i64;
2116  stride.make_folded_const(stride_val, ty, state);
2117  int s = (int)stride_val.u.i64;
2118  lanes.make_folded_const(lanes_val, ty, state);
2119  int l = (int)lanes_val.u.i64;
2120  return Shuffle::make_slice(vec.make(state, type_hint), b, s, l);
2121  }
2122 
2123  constexpr static bool foldable = false;
2124 
2126  SliceOp(Vec v, Base b, Stride s, Lanes l)
2127  : vec(v), base(b), stride(s), lanes(l) {
2128  static_assert(Base::foldable, "Base of slice should consist only of operations that constant-fold");
2129  static_assert(Stride::foldable, "Stride of slice should consist only of operations that constant-fold");
2130  static_assert(Lanes::foldable, "Lanes of slice should consist only of operations that constant-fold");
2131  }
2132 };
2133 
2134 template<typename Vec, typename Base, typename Stride, typename Lanes>
2135 std::ostream &operator<<(std::ostream &s, const SliceOp<Vec, Base, Stride, Lanes> &op) {
2136  s << "slice(" << op.vec << ", " << op.base << ", " << op.stride << ", " << op.lanes << ")";
2137  return s;
2138 }
2139 
2140 template<typename Vec, typename Base, typename Stride, typename Lanes>
2141 HALIDE_ALWAYS_INLINE auto slice(Vec vec, Base base, Stride stride, Lanes lanes) noexcept
2142  -> SliceOp<decltype(pattern_arg(vec)), decltype(pattern_arg(base)), decltype(pattern_arg(stride)), decltype(pattern_arg(lanes))> {
2143  return {pattern_arg(vec), pattern_arg(base), pattern_arg(stride), pattern_arg(lanes)};
2144 }
2145 
2146 template<typename A>
2147 struct Fold {
2148  struct pattern_tag {};
2149  A a;
2150 
2151  constexpr static uint32_t binds = bindings<A>::mask;
2152 
2155  constexpr static bool canonical = true;
2156 
2158  Expr make(MatcherState &state, halide_type_t type_hint) const noexcept {
2160  halide_type_t ty = type_hint;
2161  a.make_folded_const(c, ty, state);
2162 
2163  // The result of the fold may have an underspecified type
2164  // (e.g. because it's from an int literal). Make the type code
2165  // and bits match the required type, if there is one (we can
2166  // tell from the bits field).
2167  if (type_hint.bits) {
2168  if (((int)ty.code == (int)halide_type_int) &&
2169  ((int)type_hint.code == (int)halide_type_float)) {
2170  int64_t x = c.u.i64;
2171  c.u.f64 = (double)x;
2172  }
2173  ty.code = type_hint.code;
2174  ty.bits = type_hint.bits;
2175  }
2176 
2177  Expr e = make_const_expr(c, ty);
2178  return e;
2179  }
2180 
2181  constexpr static bool foldable = A::foldable;
2182 
2183  template<typename A1 = A>
2185  a.make_folded_const(val, ty, state);
2186  }
2187 };
2188 
2189 template<typename A>
2190 HALIDE_ALWAYS_INLINE auto fold(A &&a) noexcept -> Fold<decltype(pattern_arg(a))> {
2191  assert_is_lvalue_if_expr<A>();
2192  return {pattern_arg(a)};
2193 }
2194 
2195 template<typename A>
2196 std::ostream &operator<<(std::ostream &s, const Fold<A> &op) {
2197  s << "fold(" << op.a << ")";
2198  return s;
2199 }
2200 
2201 template<typename A>
2202 struct Overflows {
2203  struct pattern_tag {};
2204  A a;
2205 
2206  constexpr static uint32_t binds = bindings<A>::mask;
2207 
2208  // This rule is a predicate, so it always evaluates to a boolean,
2209  // which has IRNodeType UIntImm
2212  constexpr static bool canonical = true;
2213 
2214  constexpr static bool foldable = A::foldable;
2215 
2216  template<typename A1 = A>
2218  a.make_folded_const(val, ty, state);
2219  ty.code = halide_type_uint;
2220  ty.bits = 64;
2221  val.u.u64 = (ty.lanes & MatcherState::special_values_mask) != 0;
2222  ty.lanes = 1;
2223  }
2224 };
2225 
2226 template<typename A>
2227 HALIDE_ALWAYS_INLINE auto overflows(A &&a) noexcept -> Overflows<decltype(pattern_arg(a))> {
2228  assert_is_lvalue_if_expr<A>();
2229  return {pattern_arg(a)};
2230 }
2231 
2232 template<typename A>
2233 std::ostream &operator<<(std::ostream &s, const Overflows<A> &op) {
2234  s << "overflows(" << op.a << ")";
2235  return s;
2236 }
2237 
2238 struct Overflow {
2239  struct pattern_tag {};
2240 
2241  constexpr static uint32_t binds = 0;
2242 
2243  // Overflow is an intrinsic, represented as a Call node
2246  constexpr static bool canonical = true;
2247 
2248  template<uint32_t bound>
2249  HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept {
2250  if (e.node_type != Call::_node_type) {
2251  return false;
2252  }
2253  const Call &op = (const Call &)e;
2255  }
2256 
2258  Expr make(MatcherState &state, halide_type_t type_hint) const {
2260  return make_const_special_expr(type_hint);
2261  }
2262 
2263  constexpr static bool foldable = true;
2264 
2266  void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept {
2267  val.u.u64 = 0;
2269  }
2270 };
2271 
2272 inline std::ostream &operator<<(std::ostream &s, const Overflow &op) {
2273  s << "overflow()";
2274  return s;
2275 }
2276 
2277 template<typename A>
2278 struct IsConst {
2279  struct pattern_tag {};
2280 
2281  constexpr static uint32_t binds = bindings<A>::mask;
2282 
2283  // This rule is a boolean-valued predicate. Bools have type UIntImm.
2286  constexpr static bool canonical = true;
2287 
2288  A a;
2289  bool check_v;
2291 
2292  constexpr static bool foldable = true;
2293 
2294  template<typename A1 = A>
2296  Expr e = a.make(state, {});
2297  ty.code = halide_type_uint;
2298  ty.bits = 64;
2299  ty.lanes = 1;
2300  if (check_v) {
2301  val.u.u64 = ::Halide::Internal::is_const(e, v) ? 1 : 0;
2302  } else {
2303  val.u.u64 = ::Halide::Internal::is_const(e) ? 1 : 0;
2304  }
2305  }
2306 };
2307 
2308 template<typename A>
2309 HALIDE_ALWAYS_INLINE auto is_const(A &&a) noexcept -> IsConst<decltype(pattern_arg(a))> {
2310  assert_is_lvalue_if_expr<A>();
2311  return {pattern_arg(a), false, 0};
2312 }
2313 
2314 template<typename A>
2315 HALIDE_ALWAYS_INLINE auto is_const(A &&a, int64_t value) noexcept -> IsConst<decltype(pattern_arg(a))> {
2316  assert_is_lvalue_if_expr<A>();
2317  return {pattern_arg(a), true, value};
2318 }
2319 
2320 template<typename A>
2321 std::ostream &operator<<(std::ostream &s, const IsConst<A> &op) {
2322  if (op.check_v) {
2323  s << "is_const(" << op.a << ")";
2324  } else {
2325  s << "is_const(" << op.a << ", " << op.v << ")";
2326  }
2327  return s;
2328 }
2329 
2330 template<typename A, typename Prover>
2331 struct CanProve {
2332  struct pattern_tag {};
2333  A a;
2334  Prover *prover; // An existing simplifying mutator
2335 
2336  constexpr static uint32_t binds = bindings<A>::mask;
2337 
2338  // This rule is a boolean-valued predicate. Bools have type UIntImm.
2341  constexpr static bool canonical = true;
2342 
2343  constexpr static bool foldable = true;
2344 
2345  // Includes a raw call to an inlined make method, so don't inline.
2347  Expr condition = a.make(state, {});
2348  condition = prover->mutate(condition, nullptr);
2349  val.u.u64 = is_const_one(condition);
2350  ty.code = halide_type_uint;
2351  ty.bits = 1;
2352  ty.lanes = condition.type().lanes();
2353  }
2354 };
2355 
2356 template<typename A, typename Prover>
2357 HALIDE_ALWAYS_INLINE auto can_prove(A &&a, Prover *p) noexcept -> CanProve<decltype(pattern_arg(a)), Prover> {
2358  assert_is_lvalue_if_expr<A>();
2359  return {pattern_arg(a), p};
2360 }
2361 
2362 template<typename A, typename Prover>
2363 std::ostream &operator<<(std::ostream &s, const CanProve<A, Prover> &op) {
2364  s << "can_prove(" << op.a << ")";
2365  return s;
2366 }
2367 
2368 template<typename A>
2369 struct IsFloat {
2370  struct pattern_tag {};
2371  A a;
2372 
2373  constexpr static uint32_t binds = bindings<A>::mask;
2374 
2375  // This rule is a boolean-valued predicate. Bools have type UIntImm.
2378  constexpr static bool canonical = true;
2379 
2380  constexpr static bool foldable = true;
2381 
2384  // a is almost certainly a very simple pattern (e.g. a wild), so just inline the make method.
2385  Type t = a.make(state, {}).type();
2386  val.u.u64 = t.is_float();
2387  ty.code = halide_type_uint;
2388  ty.bits = 1;
2389  ty.lanes = t.lanes();
2390  }
2391 };
2392 
2393 template<typename A>
2394 HALIDE_ALWAYS_INLINE auto is_float(A &&a) noexcept -> IsFloat<decltype(pattern_arg(a))> {
2395  assert_is_lvalue_if_expr<A>();
2396  return {pattern_arg(a)};
2397 }
2398 
2399 template<typename A>
2400 std::ostream &operator<<(std::ostream &s, const IsFloat<A> &op) {
2401  s << "is_float(" << op.a << ")";
2402  return s;
2403 }
2404 
2405 template<typename A>
2406 struct IsInt {
2407  struct pattern_tag {};
2408  A a;
2409  int bits, lanes;
2410 
2411  constexpr static uint32_t binds = bindings<A>::mask;
2412 
2413  // This rule is a boolean-valued predicate. Bools have type UIntImm.
2416  constexpr static bool canonical = true;
2417 
2418  constexpr static bool foldable = true;
2419 
2422  // a is almost certainly a very simple pattern (e.g. a wild), so just inline the make method.
2423  Type t = a.make(state, {}).type();
2424  val.u.u64 = t.is_int() && (bits == 0 || t.bits() == bits) && (lanes == 0 || t.lanes() == lanes);
2425  ty.code = halide_type_uint;
2426  ty.bits = 1;
2427  ty.lanes = t.lanes();
2428  }
2429 };
2430 
2431 template<typename A>
2432 HALIDE_ALWAYS_INLINE auto is_int(A &&a, int bits = 0, int lanes = 0) noexcept -> IsInt<decltype(pattern_arg(a))> {
2433  assert_is_lvalue_if_expr<A>();
2434  return {pattern_arg(a), bits, lanes};
2435 }
2436 
2437 template<typename A>
2438 std::ostream &operator<<(std::ostream &s, const IsInt<A> &op) {
2439  s << "is_int(" << op.a;
2440  if (op.bits > 0) {
2441  s << ", " << op.bits;
2442  }
2443  if (op.lanes > 0) {
2444  s << ", " << op.lanes;
2445  }
2446  s << ")";
2447  return s;
2448 }
2449 
2450 template<typename A>
2451 struct IsUInt {
2452  struct pattern_tag {};
2453  A a;
2454  int bits, lanes;
2455 
2456  constexpr static uint32_t binds = bindings<A>::mask;
2457 
2458  // This rule is a boolean-valued predicate. Bools have type UIntImm.
2461  constexpr static bool canonical = true;
2462 
2463  constexpr static bool foldable = true;
2464 
2467  // a is almost certainly a very simple pattern (e.g. a wild), so just inline the make method.
2468  Type t = a.make(state, {}).type();
2469  val.u.u64 = t.is_uint() && (bits == 0 || t.bits() == bits) && (lanes == 0 || t.lanes() == lanes);
2470  ty.code = halide_type_uint;
2471  ty.bits = 1;
2472  ty.lanes = t.lanes();
2473  }
2474 };
2475 
2476 template<typename A>
2477 HALIDE_ALWAYS_INLINE auto is_uint(A &&a, int bits = 0, int lanes = 0) noexcept -> IsUInt<decltype(pattern_arg(a))> {
2478  assert_is_lvalue_if_expr<A>();
2479  return {pattern_arg(a), bits, lanes};
2480 }
2481 
2482 template<typename A>
2483 std::ostream &operator<<(std::ostream &s, const IsUInt<A> &op) {
2484  s << "is_uint(" << op.a;
2485  if (op.bits > 0) {
2486  s << ", " << op.bits;
2487  }
2488  if (op.lanes > 0) {
2489  s << ", " << op.lanes;
2490  }
2491  s << ")";
2492  return s;
2493 }
2494 
2495 template<typename A>
2496 struct IsScalar {
2497  struct pattern_tag {};
2498  A a;
2499 
2500  constexpr static uint32_t binds = bindings<A>::mask;
2501 
2502  // This rule is a boolean-valued predicate. Bools have type UIntImm.
2505  constexpr static bool canonical = true;
2506 
2507  constexpr static bool foldable = true;
2508 
2511  // a is almost certainly a very simple pattern (e.g. a wild), so just inline the make method.
2512  Type t = a.make(state, {}).type();
2513  val.u.u64 = t.is_scalar();
2514  ty.code = halide_type_uint;
2515  ty.bits = 1;
2516  ty.lanes = t.lanes();
2517  }
2518 };
2519 
2520 template<typename A>
2521 HALIDE_ALWAYS_INLINE auto is_scalar(A &&a) noexcept -> IsScalar<decltype(pattern_arg(a))> {
2522  assert_is_lvalue_if_expr<A>();
2523  return {pattern_arg(a)};
2524 }
2525 
2526 template<typename A>
2527 std::ostream &operator<<(std::ostream &s, const IsScalar<A> &op) {
2528  s << "is_scalar(" << op.a << ")";
2529  return s;
2530 }
2531 
2532 template<typename A>
2533 struct IsMaxValue {
2534  struct pattern_tag {};
2535  A a;
2536 
2537  constexpr static uint32_t binds = bindings<A>::mask;
2538 
2539  // This rule is a boolean-valued predicate. Bools have type UIntImm.
2542  constexpr static bool canonical = true;
2543 
2544  constexpr static bool foldable = true;
2545 
2548  // a is almost certainly a very simple pattern (e.g. a wild), so just inline the make method.
2549  a.make_folded_const(val, ty, state);
2550  const uint64_t max_bits = (uint64_t)(-1) >> (64 - ty.bits + (ty.code == halide_type_int));
2551  if (ty.code == halide_type_uint || ty.code == halide_type_int) {
2552  val.u.u64 = (val.u.u64 == max_bits);
2553  } else {
2554  val.u.u64 = 0;
2555  }
2556  ty.code = halide_type_uint;
2557  ty.bits = 1;
2558  }
2559 };
2560 
2561 template<typename A>
2562 HALIDE_ALWAYS_INLINE auto is_max_value(A &&a) noexcept -> IsMaxValue<decltype(pattern_arg(a))> {
2563  assert_is_lvalue_if_expr<A>();
2564  return {pattern_arg(a)};
2565 }
2566 
2567 template<typename A>
2568 std::ostream &operator<<(std::ostream &s, const IsMaxValue<A> &op) {
2569  s << "is_max_value(" << op.a << ")";
2570  return s;
2571 }
2572 
2573 template<typename A>
2574 struct IsMinValue {
2575  struct pattern_tag {};
2576  A a;
2577 
2578  constexpr static uint32_t binds = bindings<A>::mask;
2579 
2580  // This rule is a boolean-valued predicate. Bools have type UIntImm.
2583  constexpr static bool canonical = true;
2584 
2585  constexpr static bool foldable = true;
2586 
2589  // a is almost certainly a very simple pattern (e.g. a wild), so just inline the make method.
2590  a.make_folded_const(val, ty, state);
2591  if (ty.code == halide_type_int) {
2592  const uint64_t min_bits = (uint64_t)(-1) << (ty.bits - 1);
2593  val.u.u64 = (val.u.u64 == min_bits);
2594  } else if (ty.code == halide_type_uint) {
2595  val.u.u64 = (val.u.u64 == 0);
2596  } else {
2597  val.u.u64 = 0;
2598  }
2599  ty.code = halide_type_uint;
2600  ty.bits = 1;
2601  }
2602 };
2603 
2604 template<typename A>
2605 HALIDE_ALWAYS_INLINE auto is_min_value(A &&a) noexcept -> IsMinValue<decltype(pattern_arg(a))> {
2606  assert_is_lvalue_if_expr<A>();
2607  return {pattern_arg(a)};
2608 }
2609 
2610 template<typename A>
2611 std::ostream &operator<<(std::ostream &s, const IsMinValue<A> &op) {
2612  s << "is_min_value(" << op.a << ")";
2613  return s;
2614 }
2615 
2616 template<typename A>
2617 struct LanesOf {
2618  struct pattern_tag {};
2619  A a;
2620 
2621  constexpr static uint32_t binds = bindings<A>::mask;
2622 
2623  // This rule is a boolean-valued predicate. Bools have type UIntImm.
2626  constexpr static bool canonical = true;
2627 
2628  constexpr static bool foldable = true;
2629 
2632  // a is almost certainly a very simple pattern (e.g. a wild), so just inline the make method.
2633  Type t = a.make(state, {}).type();
2634  val.u.u64 = t.lanes();
2635  ty.code = halide_type_uint;
2636  ty.bits = 32;
2637  ty.lanes = 1;
2638  }
2639 };
2640 
2641 template<typename A>
2642 HALIDE_ALWAYS_INLINE auto lanes_of(A &&a) noexcept -> LanesOf<decltype(pattern_arg(a))> {
2643  assert_is_lvalue_if_expr<A>();
2644  return {pattern_arg(a)};
2645 }
2646 
2647 template<typename A>
2648 std::ostream &operator<<(std::ostream &s, const LanesOf<A> &op) {
2649  s << "lanes_of(" << op.a << ")";
2650  return s;
2651 }
2652 
2653 // Verify properties of each rewrite rule. Currently just fuzz tests them.
2654 template<typename Before,
2655  typename After,
2656  typename Predicate,
2657  typename = typename std::enable_if<std::decay<Before>::type::foldable &&
2658  std::decay<After>::type::foldable>::type>
2659 HALIDE_NEVER_INLINE void fuzz_test_rule(Before &&before, After &&after, Predicate &&pred,
2660  halide_type_t wildcard_type, halide_type_t output_type) noexcept {
2661 
2662  // We only validate the rules in the scalar case
2663  wildcard_type.lanes = output_type.lanes = 1;
2664 
2665  // Track which types this rule has been tested for before
2666  static std::set<uint32_t> tested;
2667 
2668  if (!tested.insert(reinterpret_bits<uint32_t>(wildcard_type)).second) {
2669  return;
2670  }
2671 
2672  // Print it in a form where it can be piped into a python/z3 validator
2673  debug(0) << "validate('" << before << "', '" << after << "', '" << pred << "', " << Type(wildcard_type) << ", " << Type(output_type) << ")\n";
2674 
2675  // Substitute some random constants into the before and after
2676  // expressions and see if the rule holds true. This should catch
2677  // silly errors, but not necessarily corner cases.
2678  static std::mt19937_64 rng(0);
2679  MatcherState state;
2680 
2681  Expr exprs[max_wild];
2682 
2683  for (int trials = 0; trials < 100; trials++) {
2684  // We want to test small constants more frequently than
2685  // large ones, otherwise we'll just get coverage of
2686  // overflow rules.
2687  int shift = (int)(rng() & (wildcard_type.bits - 1));
2688 
2689  for (int i = 0; i < max_wild; i++) {
2690  // Bind all the exprs and constants
2691  switch (wildcard_type.code) {
2692  case halide_type_uint: {
2693  // Normalize to the type's range by adding zero
2694  uint64_t val = constant_fold_bin_op<Add>(wildcard_type, (uint64_t)rng() >> shift, 0);
2695  state.set_bound_const(i, val, wildcard_type);
2696  val = constant_fold_bin_op<Add>(wildcard_type, (uint64_t)rng() >> shift, 0);
2697  exprs[i] = make_const(wildcard_type, val);
2698  state.set_binding(i, *exprs[i].get());
2699  } break;
2700  case halide_type_int: {
2701  int64_t val = constant_fold_bin_op<Add>(wildcard_type, (int64_t)rng() >> shift, 0);
2702  state.set_bound_const(i, val, wildcard_type);
2703  val = constant_fold_bin_op<Add>(wildcard_type, (int64_t)rng() >> shift, 0);
2704  exprs[i] = make_const(wildcard_type, val);
2705  } break;
2706  case halide_type_float:
2707  case halide_type_bfloat: {
2708  // Use a very narrow range of precise floats, so
2709  // that none of the rules a human is likely to
2710  // write have instabilities.
2711  double val = ((int64_t)(rng() & 15) - 8) / 2.0;
2712  state.set_bound_const(i, val, wildcard_type);
2713  val = ((int64_t)(rng() & 15) - 8) / 2.0;
2714  exprs[i] = make_const(wildcard_type, val);
2715  } break;
2716  default:
2717  return; // Don't care about handles
2718  }
2719  state.set_binding(i, *exprs[i].get());
2720  }
2721 
2722  halide_scalar_value_t val_pred, val_before, val_after;
2723  halide_type_t type = output_type;
2724  if (!evaluate_predicate(pred, state)) {
2725  continue;
2726  }
2727  before.make_folded_const(val_before, type, state);
2728  uint16_t lanes = type.lanes;
2729  after.make_folded_const(val_after, type, state);
2730  lanes |= type.lanes;
2731 
2732  if (lanes & MatcherState::special_values_mask) {
2733  continue;
2734  }
2735 
2736  bool ok = true;
2737  switch (output_type.code) {
2738  case halide_type_uint:
2739  // Compare normalized representations
2740  ok &= (constant_fold_bin_op<Add>(output_type, val_before.u.u64, 0) ==
2741  constant_fold_bin_op<Add>(output_type, val_after.u.u64, 0));
2742  break;
2743  case halide_type_int:
2744  ok &= (constant_fold_bin_op<Add>(output_type, val_before.u.i64, 0) ==
2745  constant_fold_bin_op<Add>(output_type, val_after.u.i64, 0));
2746  break;
2747  case halide_type_float:
2748  case halide_type_bfloat: {
2749  double error = std::abs(val_before.u.f64 - val_after.u.f64);
2750  // We accept an equal bit pattern (e.g. inf vs inf),
2751  // a small floating point difference, or turning a nan into not-a-nan.
2752  ok &= (error < 0.01 ||
2753  val_before.u.u64 == val_after.u.u64 ||
2754  std::isnan(val_before.u.f64));
2755  break;
2756  }
2757  default:
2758  return;
2759  }
2760 
2761  if (!ok) {
2762  debug(0) << "Fails with values:\n";
2763  for (int i = 0; i < max_wild; i++) {
2765  state.get_bound_const(i, val, wildcard_type);
2766  debug(0) << " c" << i << ": " << make_const_expr(val, wildcard_type) << "\n";
2767  }
2768  for (int i = 0; i < max_wild; i++) {
2769  debug(0) << " _" << i << ": " << Expr(state.get_binding(i)) << "\n";
2770  }
2771  debug(0) << " Before: " << make_const_expr(val_before, output_type) << "\n";
2772  debug(0) << " After: " << make_const_expr(val_after, output_type) << "\n";
2773  debug(0) << val_before.u.u64 << " " << val_after.u.u64 << "\n";
2775  }
2776  }
2777 }
2778 
2779 template<typename Before,
2780  typename After,
2781  typename Predicate,
2782  typename = typename std::enable_if<!(std::decay<Before>::type::foldable &&
2783  std::decay<After>::type::foldable)>::type>
2784 HALIDE_ALWAYS_INLINE void fuzz_test_rule(Before &&before, After &&after, Predicate &&pred,
2785  halide_type_t, halide_type_t, int dummy = 0) noexcept {
2786  // We can't verify rewrite rules that can't be constant-folded.
2787 }
2788 
2790 bool evaluate_predicate(bool x, MatcherState &) noexcept {
2791  return x;
2792 }
2793 
2794 template<typename Pattern,
2795  typename = typename enable_if_pattern<Pattern>::type>
2798  halide_type_t ty = halide_type_of<bool>();
2799  p.make_folded_const(c, ty, state);
2800  // Overflow counts as a failed predicate
2801  return (c.u.u64 != 0) && ((ty.lanes & MatcherState::special_values_mask) == 0);
2802 }
2803 
2804 // #defines for testing
2805 
2806 // Print all successful or failed matches
2807 #define HALIDE_DEBUG_MATCHED_RULES 0
2808 #define HALIDE_DEBUG_UNMATCHED_RULES 0
2809 
2810 // Set to true if you want to fuzz test every rewrite passed to
2811 // operator() to ensure the input and the output have the same value
2812 // for lots of random values of the wildcards. Run
2813 // correctness_simplify with this on.
2814 #define HALIDE_FUZZ_TEST_RULES 0
2815 
2816 template<typename Instance>
2817 struct Rewriter {
2818  Instance instance;
2822  bool validate;
2823 
2826  : instance(std::move(instance)), output_type(ot), wildcard_type(wt) {
2827  }
2828 
2829  template<typename After>
2831  result = after.make(state, output_type);
2832  }
2833 
2834  template<typename Before,
2835  typename After,
2836  typename = typename enable_if_pattern<Before>::type,
2837  typename = typename enable_if_pattern<After>::type>
2838  HALIDE_ALWAYS_INLINE bool operator()(Before before, After after) {
2839  static_assert((Before::binds & After::binds) == After::binds, "Rule result uses unbound values");
2840  static_assert(Before::canonical, "LHS of rewrite rule should be in canonical form");
2841  static_assert(After::canonical, "RHS of rewrite rule should be in canonical form");
2842 #if HALIDE_FUZZ_TEST_RULES
2843  fuzz_test_rule(before, after, true, wildcard_type, output_type);
2844 #endif
2845  if (before.template match<0>(unwrap(instance), state)) {
2846  build_replacement(after);
2847 #if HALIDE_DEBUG_MATCHED_RULES
2848  debug(0) << instance << " -> " << result << " via " << before << " -> " << after << "\n";
2849 #endif
2850  return true;
2851  } else {
2852 #if HALIDE_DEBUG_UNMATCHED_RULES
2853  debug(0) << instance << " does not match " << before << "\n";
2854 #endif
2855  return false;
2856  }
2857  }
2858 
2859  template<typename Before,
2860  typename = typename enable_if_pattern<Before>::type>
2861  HALIDE_ALWAYS_INLINE bool operator()(Before before, const Expr &after) noexcept {
2862  static_assert(Before::canonical, "LHS of rewrite rule should be in canonical form");
2863  if (before.template match<0>(unwrap(instance), state)) {
2864  result = after;
2865 #if HALIDE_DEBUG_MATCHED_RULES
2866  debug(0) << instance << " -> " << result << " via " << before << " -> " << after << "\n";
2867 #endif
2868  return true;
2869  } else {
2870 #if HALIDE_DEBUG_UNMATCHED_RULES
2871  debug(0) << instance << " does not match " << before << "\n";
2872 #endif
2873  return false;
2874  }
2875  }
2876 
2877  template<typename Before,
2878  typename = typename enable_if_pattern<Before>::type>
2879  HALIDE_ALWAYS_INLINE bool operator()(Before before, int64_t after) noexcept {
2880  static_assert(Before::canonical, "LHS of rewrite rule should be in canonical form");
2881 #if HALIDE_FUZZ_TEST_RULES
2882  fuzz_test_rule(before, IntLiteral(after), true, wildcard_type, output_type);
2883 #endif
2884  if (before.template match<0>(unwrap(instance), state)) {
2885  result = make_const(output_type, after);
2886 #if HALIDE_DEBUG_MATCHED_RULES
2887  debug(0) << instance << " -> " << result << " via " << before << " -> " << after << "\n";
2888 #endif
2889  return true;
2890  } else {
2891 #if HALIDE_DEBUG_UNMATCHED_RULES
2892  debug(0) << instance << " does not match " << before << "\n";
2893 #endif
2894  return false;
2895  }
2896  }
2897 
2898  template<typename Before,
2899  typename After,
2900  typename Predicate,
2901  typename = typename enable_if_pattern<Before>::type,
2902  typename = typename enable_if_pattern<After>::type,
2903  typename = typename enable_if_pattern<Predicate>::type>
2904  HALIDE_ALWAYS_INLINE bool operator()(Before before, After after, Predicate pred) {
2905  static_assert(Predicate::foldable, "Predicates must consist only of operations that can constant-fold");
2906  static_assert((Before::binds & After::binds) == After::binds, "Rule result uses unbound values");
2907  static_assert((Before::binds & Predicate::binds) == Predicate::binds, "Rule predicate uses unbound values");
2908  static_assert(Before::canonical, "LHS of rewrite rule should be in canonical form");
2909  static_assert(After::canonical, "RHS of rewrite rule should be in canonical form");
2910 
2911 #if HALIDE_FUZZ_TEST_RULES
2912  fuzz_test_rule(before, after, pred, wildcard_type, output_type);
2913 #endif
2914  if (before.template match<0>(unwrap(instance), state) &&
2915  evaluate_predicate(pred, state)) {
2916  build_replacement(after);
2917 #if HALIDE_DEBUG_MATCHED_RULES
2918  debug(0) << instance << " -> " << result << " via " << before << " -> " << after << " when " << pred << "\n";
2919 #endif
2920  return true;
2921  } else {
2922 #if HALIDE_DEBUG_UNMATCHED_RULES
2923  debug(0) << instance << " does not match " << before << "\n";
2924 #endif
2925  return false;
2926  }
2927  }
2928 
2929  template<typename Before,
2930  typename Predicate,
2931  typename = typename enable_if_pattern<Before>::type,
2932  typename = typename enable_if_pattern<Predicate>::type>
2933  HALIDE_ALWAYS_INLINE bool operator()(Before before, const Expr &after, Predicate pred) {
2934  static_assert(Predicate::foldable, "Predicates must consist only of operations that can constant-fold");
2935  static_assert(Before::canonical, "LHS of rewrite rule should be in canonical form");
2936 
2937  if (before.template match<0>(unwrap(instance), state) &&
2938  evaluate_predicate(pred, state)) {
2939  result = after;
2940 #if HALIDE_DEBUG_MATCHED_RULES
2941  debug(0) << instance << " -> " << result << " via " << before << " -> " << after << " when " << pred << "\n";
2942 #endif
2943  return true;
2944  } else {
2945 #if HALIDE_DEBUG_UNMATCHED_RULES
2946  debug(0) << instance << " does not match " << before << "\n";
2947 #endif
2948  return false;
2949  }
2950  }
2951 
2952  template<typename Before,
2953  typename Predicate,
2954  typename = typename enable_if_pattern<Before>::type,
2955  typename = typename enable_if_pattern<Predicate>::type>
2956  HALIDE_ALWAYS_INLINE bool operator()(Before before, int64_t after, Predicate pred) {
2957  static_assert(Predicate::foldable, "Predicates must consist only of operations that can constant-fold");
2958  static_assert(Before::canonical, "LHS of rewrite rule should be in canonical form");
2959 #if HALIDE_FUZZ_TEST_RULES
2960  fuzz_test_rule(before, IntLiteral(after), pred, wildcard_type, output_type);
2961 #endif
2962  if (before.template match<0>(unwrap(instance), state) &&
2963  evaluate_predicate(pred, state)) {
2964  result = make_const(output_type, after);
2965 #if HALIDE_DEBUG_MATCHED_RULES
2966  debug(0) << instance << " -> " << result << " via " << before << " -> " << after << " when " << pred << "\n";
2967 #endif
2968  return true;
2969  } else {
2970 #if HALIDE_DEBUG_UNMATCHED_RULES
2971  debug(0) << instance << " does not match " << before << "\n";
2972 #endif
2973  return false;
2974  }
2975  }
2976 };
2977 
2978 /** Construct a rewriter for the given instance, which may be a pattern
2979  * with concrete expressions as leaves, or just an expression. The
2980  * second optional argument (wildcard_type) is a hint as to what the
2981  * type of the wildcards is likely to be. If omitted it uses the same
2982  * type as the expression itself. They are not required to be this
2983  * type, but the rule will only be tested for wildcards of that type
2984  * when testing is enabled.
2985  *
2986  * The rewriter can be used to check to see if the instance is one of
2987  * some number of patterns and if so rewrite it into another form,
2988  * using its operator() method. See Simplify.cpp for a bunch of
2989  * example usage.
2990  *
2991  * Important: Any Exprs in patterns are captured by reference, not by
2992  * value, so ensure they outlive the rewriter.
2993  */
2994 // @{
2995 template<typename Instance,
2996  typename = typename enable_if_pattern<Instance>::type>
2997 HALIDE_ALWAYS_INLINE auto rewriter(Instance instance, halide_type_t output_type, halide_type_t wildcard_type) noexcept -> Rewriter<decltype(pattern_arg(instance))> {
2998  return {pattern_arg(instance), output_type, wildcard_type};
2999 }
3000 
3001 template<typename Instance,
3002  typename = typename enable_if_pattern<Instance>::type>
3003 HALIDE_ALWAYS_INLINE auto rewriter(Instance instance, halide_type_t output_type) noexcept -> Rewriter<decltype(pattern_arg(instance))> {
3004  return {pattern_arg(instance), output_type, output_type};
3005 }
3006 
3008 auto rewriter(const Expr &e, halide_type_t wildcard_type) noexcept -> Rewriter<decltype(pattern_arg(e))> {
3009  return {pattern_arg(e), e.type(), wildcard_type};
3010 }
3011 
3013 auto rewriter(const Expr &e) noexcept -> Rewriter<decltype(pattern_arg(e))> {
3014  return {pattern_arg(e), e.type(), e.type()};
3015 }
3016 // @}
3017 
3018 } // namespace IRMatcher
3019 
3020 } // namespace Internal
3021 } // namespace Halide
3022 
3023 #endif
Halide::Internal::IRMatcher::Overflow::pattern_tag
Definition: IRMatch.h:2239
Halide::Internal::IRMatcher::Intrin::binds
static constexpr uint32_t binds
Definition: IRMatch.h:1366
int32_t
signed __INT32_TYPE__ int32_t
Definition: runtime_internal.h:24
Halide::Internal::IRMatcher::not_op
HALIDE_ALWAYS_INLINE auto not_op(A &&a) -> decltype(IRMatcher::operator!(a))
Definition: IRMatch.h:1661
Halide::Internal::IRMatcher::RampOp::b
B b
Definition: IRMatch.h:1813
Halide::Internal::IRMatcher::cast
HALIDE_ALWAYS_INLINE auto cast(halide_type_t t, A &&a) noexcept -> CastOp< decltype(pattern_arg(a))>
Definition: IRMatch.h:2078
Halide::Internal::IRMatcher::WildConst::foldable
constexpr static bool foldable
Definition: IRMatch.h:450
Halide::Internal::Cast::value
Expr value
Definition: IR.h:30
Halide::Internal::IRMatcher::BroadcastOp::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition: IRMatch.h:1787
Halide::Internal::IRMatcher::Intrin::pattern_tag
Definition: IRMatch.h:1358
Halide::Internal::IRMatcher::const_min
constexpr int const_min(int a, int b)
Definition: IRMatch.h:1352
Halide::Internal::IRMatcher::WildConstUInt::binds
constexpr static uint32_t binds
Definition: IRMatch.h:304
Halide::Internal::Broadcast::value
Expr value
Definition: IR.h:252
Halide::Internal::Call::signed_integer_overflow
@ signed_integer_overflow
Definition: IR.h:586
Halide::Internal::IRMatcher::SelectOp::pattern_tag
Definition: IRMatch.h:1674
Halide::Internal::IRMatcher::intrin
HALIDE_ALWAYS_INLINE auto intrin(Call::IntrinsicOp intrinsic_op, Args... args) noexcept -> Intrin< decltype(pattern_arg(args))... >
Definition: IRMatch.h:1534
Halide::Internal::Add
The sum of two expressions.
Definition: IR.h:48
Halide::Internal::IRNodeType::Cast
@ Cast
Halide::Internal::Shuffle::slice_stride
int slice_stride() const
Check if this shuffle is a contiguous strict subset of the vector arguments, and if so,...
Definition: IR.h:874
Halide::Internal::IRMatcher::assert_is_lvalue_if_expr
HALIDE_ALWAYS_INLINE void assert_is_lvalue_if_expr()
Definition: IRMatch.h:588
Halide::Internal::IRMatcher::constant_fold_bin_op< Div >
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< Div >(halide_type_t &t, int64_t a, int64_t b) noexcept
Definition: IRMatch.h:1044
Halide::Internal::IRMatcher::constant_fold_cmp_op< LE >
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op< LE >(int64_t a, int64_t b) noexcept
Definition: IRMatch.h:1192
Halide::Internal::Call::widening_sub
@ widening_sub
Definition: IR.h:609
Halide::Internal::IRMatcher::IsInt::foldable
constexpr static bool foldable
Definition: IRMatch.h:2418
Halide::Internal::IRMatcher::operator%
HALIDE_ALWAYS_INLINE auto operator%(A &&a, B &&b) noexcept -> BinOp< Mod, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1059
Halide::Internal::IRMatcher::SelectOp::canonical
constexpr static bool canonical
Definition: IRMatch.h:1684
Halide::Internal::IRMatcher::mod
HALIDE_ALWAYS_INLINE auto mod(A &&a, B &&b) -> decltype(IRMatcher::operator%(a, b))
Definition: IRMatch.h:1066
Halide::Internal::IRMatcher::NegateOp::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:1983
Halide::Internal::IRMatcher::Rewriter::validate
bool validate
Definition: IRMatch.h:2822
Halide::Internal::IRMatcher::IsMinValue::canonical
constexpr static bool canonical
Definition: IRMatch.h:2583
Halide::Internal::IRMatcher::WildConstUInt::foldable
constexpr static bool foldable
Definition: IRMatch.h:339
Halide::Internal::IRMatcher::bitwise_or_reduce
constexpr uint32_t bitwise_or_reduce()
Definition: IRMatch.h:1333
Halide::Internal::IRMatcher::MatcherState::signed_integer_overflow
static constexpr uint16_t signed_integer_overflow
Definition: IRMatch.h:87
Halide::Internal::IRMatcher::SpecificExpr::canonical
constexpr static bool canonical
Definition: IRMatch.h:212
Halide::Internal::IRMatcher::Overflow::canonical
constexpr static bool canonical
Definition: IRMatch.h:2246
Halide::Internal::IRMatcher::LanesOf::foldable
constexpr static bool foldable
Definition: IRMatch.h:2628
Halide::Internal::IRMatcher::operator+
HALIDE_ALWAYS_INLINE auto operator+(A &&a, B &&b) noexcept -> BinOp< Add, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:933
Halide::Internal::IRMatcher::widen_right_mul
auto widen_right_mul(A &&a, B &&b) noexcept -> Intrin< decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1543
Halide::Internal::IRMatcher::VectorReduceOp::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:1888
Halide::Internal::IRMatcher::ramp
HALIDE_ALWAYS_INLINE auto ramp(A &&a, B &&b, C &&c) noexcept -> RampOp< decltype(pattern_arg(a)), decltype(pattern_arg(b)), decltype(pattern_arg(c))>
Definition: IRMatch.h:1868
Halide::Internal::Call::halving_sub
@ halving_sub
Definition: IR.h:544
Halide::Internal::IRMatcher::IsScalar::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:2503
Halide::Internal::IRMatcher::gt
HALIDE_ALWAYS_INLINE auto gt(A &&a, B &&b) -> decltype(IRMatcher::operator>(a, b))
Definition: IRMatch.h:1162
Halide::Internal::IRMatcher::IsInt::canonical
constexpr static bool canonical
Definition: IRMatch.h:2416
Halide::Internal::IRMatcher::IsFloat::a
A a
Definition: IRMatch.h:2371
Halide::Internal::IRMatcher::WildConstInt::canonical
constexpr static bool canonical
Definition: IRMatch.h:242
Halide::Internal::IRMatcher::operator&&
HALIDE_ALWAYS_INLINE auto operator&&(A &&a, B &&b) noexcept -> BinOp< And, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1308
Halide::Internal::IRMatcher::IsMinValue::a
A a
Definition: IRMatch.h:2576
Halide::Internal::VectorReduce::op
Operator op
Definition: IR.h:944
Halide::Internal::IRMatcher::Intrin::match_args
HALIDE_ALWAYS_INLINE bool match_args(double, const Call &c, MatcherState &state) const noexcept
Definition: IRMatch.h:1382
Halide::Internal::IRMatcher::Overflow::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:2244
Halide::Internal::IRMatcher::VectorReduceOp::match
HALIDE_ALWAYS_INLINE bool match(const VectorReduceOp< A2, B2, reduce_op_2 > &op, MatcherState &state) const noexcept
Definition: IRMatch.h:1901
Halide::Internal::IRMatcher::bindings
Definition: IRMatch.h:145
Halide::Internal::StrongestExprNodeType
constexpr IRNodeType StrongestExprNodeType
Definition: Expr.h:80
Halide::Internal::IRMatcher::WildConstInt
Definition: IRMatch.h:235
Halide::Internal::Shuffle::vectors
std::vector< Expr > vectors
Definition: IR.h:820
Halide::Internal::IRMatcher::VectorReduceOp::canonical
constexpr static bool canonical
Definition: IRMatch.h:1885
Halide::Internal::IRMatcher::WildConstInt::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const
Definition: IRMatch.h:289
Halide::Internal::IRMatcher::NotOp::canonical
constexpr static bool canonical
Definition: IRMatch.h:1623
Halide::Internal::IRMatcher::Rewriter::instance
Instance instance
Definition: IRMatch.h:2818
Halide::Internal::IRMatcher::SelectOp::f
F f
Definition: IRMatch.h:1677
Halide::Internal::IRMatcher::WildConstFloat::canonical
constexpr static bool canonical
Definition: IRMatch.h:361
Halide::Internal::IRMatcher::constant_fold_cmp_op< LT >
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op< LT >(int64_t a, int64_t b) noexcept
Definition: IRMatch.h:1142
Halide::Internal::IRMatcher::IsConst
Definition: IRMatch.h:2278
Halide::Internal::mod_imp
T mod_imp(T a, T b)
Implementations of division and mod that are specific to Halide.
Definition: IROperator.h:239
Halide::Internal::IRMatcher::SliceOp::SliceOp
HALIDE_ALWAYS_INLINE SliceOp(Vec v, Base b, Stride s, Lanes l)
Definition: IRMatch.h:2126
Halide::Internal::IRMatcher::IntLiteral::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:544
Halide::Internal::Call::saturating_add
@ saturating_add
Definition: IR.h:579
Halide::Internal::IRMatcher::CastOp::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:2050
Halide::Internal::IRMatcher::VectorReduceOp::a
A a
Definition: IRMatch.h:1878
Halide::Internal::IRMatcher::Rewriter::Rewriter
HALIDE_ALWAYS_INLINE Rewriter(Instance instance, halide_type_t ot, halide_type_t wt)
Definition: IRMatch.h:2825
Halide::Internal::IRMatcher::SpecificExpr
Definition: IRMatch.h:204
Halide::Internal::IRMatcher::IsMaxValue
Definition: IRMatch.h:2533
halide_type_bfloat
@ halide_type_bfloat
floating point numbers in the bfloat format
Definition: HalideRuntime.h:458
Halide::Internal::IRMatcher::IsMaxValue::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:2540
Halide::Internal::IRNodeType::Not
@ Not
Halide::Internal::IRMatcher::widening_mul
auto widening_mul(A &&a, B &&b) noexcept -> Intrin< decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1560
Halide::Internal::IRMatcher::Overflow::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:2258
Halide::Internal::IRNodeType::Select
@ Select
Halide::Internal::IRMatcher::SpecificExpr::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:222
Halide::Internal::IRMatcher::CmpOp::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition: IRMatch.h:786
Halide::Internal::IRMatcher::VectorReduceOp
Definition: IRMatch.h:1876
Halide::Internal::IRMatcher::IsFloat::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const
Definition: IRMatch.h:2383
Halide::Internal::IRMatcher::CanProve::make_folded_const
HALIDE_NEVER_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const
Definition: IRMatch.h:2346
Halide::Internal::IRMatcher::LanesOf::pattern_tag
Definition: IRMatch.h:2618
Halide::Internal::IRMatcher::RampOp::a
A a
Definition: IRMatch.h:1812
Halide::Internal::VectorReduce
Horizontally reduce a vector to a scalar or narrower vector using the given commutative and associati...
Definition: IR.h:929
uint16_t
unsigned __INT16_TYPE__ uint16_t
Definition: runtime_internal.h:27
Halide::Internal::IRMatcher::CanProve
Definition: IRMatch.h:2331
Halide::Internal::IRMatcher::Fold::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition: IRMatch.h:2184
Halide::Internal::Select::condition
Expr condition
Definition: IR.h:197
Halide::Internal::IRMatcher::IsFloat::foldable
constexpr static bool foldable
Definition: IRMatch.h:2380
Halide::Internal::BaseExprNode::type
Type type
Definition: Expr.h:147
Halide::Internal::sub_would_overflow
bool sub_would_overflow(int bits, int64_t a, int64_t b)
Halide::Internal::IRMatcher::LanesOf::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:2624
Halide::Internal::IRMatcher::IsConst::a
A a
Definition: IRMatch.h:2288
Halide::TailStrategy::Predicate
@ Predicate
Guard the loads and stores in the loop with an if statement that prevents evaluation beyond the origi...
Halide::Internal::IRMatcher::Rewriter::operator()
HALIDE_ALWAYS_INLINE bool operator()(Before before, const Expr &after, Predicate pred)
Definition: IRMatch.h:2933
Halide::Internal::IRMatcher::CastOp::foldable
constexpr static bool foldable
Definition: IRMatch.h:2068
Halide::Internal::IRMatcher::IsMinValue::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const
Definition: IRMatch.h:2588
Halide::Internal::IRMatcher::Fold::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const noexcept
Definition: IRMatch.h:2158
Halide::Internal::IRMatcher::CmpOp::a
A a
Definition: IRMatch.h:753
Halide::Internal::IRMatcher::ne
HALIDE_ALWAYS_INLINE auto ne(A &&a, B &&b) -> decltype(IRMatcher::operator!=(a, b))
Definition: IRMatch.h:1262
Halide::Internal::IRMatcher::make_const_expr
HALIDE_ALWAYS_INLINE Expr make_const_expr(halide_scalar_value_t val, halide_type_t ty)
Definition: IRMatch.h:160
Halide::Internal::VectorReduce::value
Expr value
Definition: IR.h:943
Halide::Internal::IRMatcher::MatcherState::set_bound_const
HALIDE_ALWAYS_INLINE void set_bound_const(int i, halide_scalar_value_t val, halide_type_t t) noexcept
Definition: IRMatch.h:121
Halide::Internal::IRMatcher::IsUInt::foldable
constexpr static bool foldable
Definition: IRMatch.h:2463
Halide::Internal::IRMatcher::constant_fold_bin_op< Mul >
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< Mul >(halide_type_t &t, int64_t a, int64_t b) noexcept
Definition: IRMatch.h:1013
Halide::Internal::IRMatcher::CastOp::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:2064
Halide::Internal::IRMatcher::IsInt::a
A a
Definition: IRMatch.h:2408
Halide::Internal::IRMatcher::NegateOp::a
A a
Definition: IRMatch.h:1958
Halide::Internal::IRMatcher::RampOp::foldable
constexpr static bool foldable
Definition: IRMatch.h:1858
Halide::Internal::GE
Is the first expression greater than or equal to the second.
Definition: IR.h:158
Halide::Internal::IRMatcher::MatcherState::MatcherState
HALIDE_ALWAYS_INLINE MatcherState() noexcept
Definition: IRMatch.h:134
Halide::Internal::IRMatcher::IsInt::bits
int bits
Definition: IRMatch.h:2409
Halide::Internal::IRMatcher::IsConst::foldable
constexpr static bool foldable
Definition: IRMatch.h:2292
halide_type_float
@ halide_type_float
IEEE floating point numbers.
Definition: HalideRuntime.h:456
Halide::Internal::IRMatcher::max_wild
constexpr int max_wild
Definition: IRMatch.h:74
Halide::Internal::IRMatcher::BroadcastOp::pattern_tag
Definition: IRMatch.h:1740
Halide::Internal::IRMatcher::shift_right
auto shift_right(A &&a, B &&b) noexcept -> Intrin< decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1594
Halide::Internal::IRNodeType
IRNodeType
All our IR node types get unique IDs for the purposes of RTTI.
Definition: Expr.h:25
Halide::Internal::IRMatcher::fuzz_test_rule
HALIDE_NEVER_INLINE void fuzz_test_rule(Before &&before, After &&after, Predicate &&pred, halide_type_t wildcard_type, halide_type_t output_type) noexcept
Definition: IRMatch.h:2659
Halide::Internal::IRMatcher::IsConst::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:2284
Halide::Internal::IRNodeType::EQ
@ EQ
Halide::Internal::IRMatcher::IsFloat::canonical
constexpr static bool canonical
Definition: IRMatch.h:2378
Halide::Internal::IRMatcher::IntLiteral::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:506
Halide::Internal::IRMatcher::Intrin::optional_type_hint
Type optional_type_hint
Definition: IRMatch.h:1364
Halide::Internal::IRMatcher::Overflow::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:2245
Halide::Internal::IRMatcher::Rewriter::operator()
HALIDE_ALWAYS_INLINE bool operator()(Before before, const Expr &after) noexcept
Definition: IRMatch.h:2861
Halide::Internal::IRMatcher::Intrin::canonical
constexpr static bool canonical
Definition: IRMatch.h:1370
Halide::Internal::IRMatcher::Fold::foldable
constexpr static bool foldable
Definition: IRMatch.h:2181
IROperator.h
Halide::Internal::IRMatcher::IsInt::binds
constexpr static uint32_t binds
Definition: IRMatch.h:2411
Halide::Internal::IRMatcher::SelectOp::t
T t
Definition: IRMatch.h:1676
Halide::Internal::IRMatcher::SpecificExpr::expr
const BaseExprNode & expr
Definition: IRMatch.h:214
Halide::Internal::IRMatcher::rounding_mul_shift_right
auto rounding_mul_shift_right(A &&a, B &&b, C &&c) noexcept -> Intrin< decltype(pattern_arg(a)), decltype(pattern_arg(b)), decltype(pattern_arg(c))>
Definition: IRMatch.h:1610
Halide::Internal::IRMatcher::IsFloat::pattern_tag
Definition: IRMatch.h:2370
Halide::Internal::IRMatcher::IntLiteral::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:507
Halide::Internal::IRMatcher::CastOp::pattern_tag
Definition: IRMatch.h:2039
Halide::Internal::IRMatcher::constant_fold_bin_op
int64_t constant_fold_bin_op(halide_type_t &, int64_t, int64_t) noexcept
Halide::Internal::FloatImm
Floating point constants.
Definition: Expr.h:235
Halide::Internal::IRMatcher::h_max
HALIDE_ALWAYS_INLINE auto h_max(A &&a, B lanes) noexcept -> VectorReduceOp< decltype(pattern_arg(a)), decltype(pattern_arg(lanes)), VectorReduce::Max >
Definition: IRMatch.h:1938
Halide::Internal::IRMatcher::SpecificExpr::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:211
Halide::Type::lanes
HALIDE_ALWAYS_INLINE int lanes() const
Return the number of vector elements in this type.
Definition: Type.h:344
Halide::Internal::IRMatcher::CanProve::pattern_tag
Definition: IRMatch.h:2332
Halide::Internal::IRMatcher::saturating_cast
auto saturating_cast(const Type &t, A &&a) noexcept -> Intrin< decltype(pattern_arg(a))>
Definition: IRMatch.h:1572
Halide::Internal::IRMatcher::WildConstUInt::pattern_tag
Definition: IRMatch.h:302
Halide::Internal::IRMatcher::min
HALIDE_ALWAYS_INLINE auto min(A &&a, B &&b) noexcept -> BinOp< Min, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1088
Halide::Internal::IRNodeType::Or
@ Or
Halide::Internal::IRMatcher::WildConstFloat::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition: IRMatch.h:395
Halide::Internal::IRMatcher::Rewriter::output_type
halide_type_t output_type
Definition: IRMatch.h:2821
Halide::Internal::UIntImm::make
static const UIntImm * make(Type t, uint64_t value)
Halide::Internal::IRMatcher::IntLiteral::binds
constexpr static uint32_t binds
Definition: IRMatch.h:504
Halide::Internal::IRMatcher::CmpOp::binds
constexpr static uint32_t binds
Definition: IRMatch.h:756
Halide::Internal::IRMatcher::SpecificExpr::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:210
Halide::Internal::Broadcast
A vector with 'lanes' elements, in which every element is 'value'.
Definition: IR.h:251
Halide::Internal::Div
The ratio of two expressions.
Definition: IR.h:75
Halide::Internal::IRMatcher::is_uint
HALIDE_ALWAYS_INLINE auto is_uint(A &&a, int bits=0, int lanes=0) noexcept -> IsUInt< decltype(pattern_arg(a))>
Definition: IRMatch.h:2477
Halide::Type::is_float
HALIDE_ALWAYS_INLINE bool is_float() const
Is this type a floating point type (float or double).
Definition: Type.h:412
Halide::Internal::IRMatcher::commutative
constexpr bool commutative(IRNodeType t)
Definition: IRMatch.h:627
Halide::Type::is_uint
HALIDE_ALWAYS_INLINE bool is_uint() const
Is this type an unsigned integer type?
Definition: Type.h:430
Halide::Internal::IRMatcher::BinOp::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:647
Halide::Internal::is_const_zero
bool is_const_zero(const Expr &e)
Is the expression a const (as defined by is_const), and also equal to zero (in all lanes,...
Halide::Internal::IRMatcher::CmpOp::b
B b
Definition: IRMatch.h:754
Halide::Internal::IRMatcher::SliceOp::vec
Vec vec
Definition: IRMatch.h:2086
Halide::Internal::IRMatcher::LanesOf::a
A a
Definition: IRMatch.h:2619
Halide::Internal::IRMatcher::halving_add
auto halving_add(A &&a, B &&b) noexcept -> Intrin< decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1578
Halide::Internal::IRMatcher::BroadcastOp::match
HALIDE_ALWAYS_INLINE bool match(const BroadcastOp< A2, B2 > &op, MatcherState &state) const noexcept
Definition: IRMatch.h:1764
Halide::Internal::IRMatcher::SelectOp::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition: IRMatch.h:1711
Halide::Internal::IRMatcher::IsInt::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:2415
Halide::Internal::IRMatcher::SelectOp
Definition: IRMatch.h:1673
Halide::Internal::IRMatcher::bindings::mask
constexpr static uint32_t mask
Definition: IRMatch.h:146
Halide::Internal::IRMatcher::select
HALIDE_ALWAYS_INLINE auto select(C &&c, T &&t, F &&f) noexcept -> SelectOp< decltype(pattern_arg(c)), decltype(pattern_arg(t)), decltype(pattern_arg(f))>
Definition: IRMatch.h:1731
Halide::Internal::IntImm
Integer constants.
Definition: Expr.h:217
Halide::Internal::Call::widen_right_sub
@ widen_right_sub
Definition: IR.h:603
Halide::Internal::IRMatcher::BinOp::binds
constexpr static uint32_t binds
Definition: IRMatch.h:645
Halide::Internal::IRMatcher::SelectOp::binds
constexpr static uint32_t binds
Definition: IRMatch.h:1679
Halide::Internal::IRMatcher::lt
HALIDE_ALWAYS_INLINE auto lt(A &&a, B &&b) -> decltype(IRMatcher::operator<(a, b))
Definition: IRMatch.h:1137
halide_type_t::bits
uint8_t bits
The number of bits of precision of a single scalar value of this type.
Definition: HalideRuntime.h:488
Halide::Internal::IRMatcher::operator||
HALIDE_ALWAYS_INLINE auto operator||(A &&a, B &&b) noexcept -> BinOp< Or, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1282
Halide::Internal::IRMatcher::is_const
HALIDE_ALWAYS_INLINE auto is_const(A &&a) noexcept -> IsConst< decltype(pattern_arg(a))>
Definition: IRMatch.h:2309
Halide::Internal::with_lanes
Expr with_lanes(const Expr &x, int lanes)
Rewrite the expression x to have lanes lanes.
Halide::Internal::IRMatcher::IntLiteral::match
HALIDE_ALWAYS_INLINE bool match(const IntLiteral &b, MatcherState &state) const noexcept
Definition: IRMatch.h:539
Halide::absd
Expr absd(Expr a, Expr b)
Return the absolute difference between two values.
Halide::Internal::IRMatcher::WildConst::match
HALIDE_ALWAYS_INLINE bool match(int64_t e, MatcherState &state) const noexcept
Definition: IRMatch.h:437
Halide::Internal::IRMatcher::LanesOf
Definition: IRMatch.h:2617
Halide::Internal::IRMatcher::IsMinValue::binds
constexpr static uint32_t binds
Definition: IRMatch.h:2578
Halide::Internal::IRMatcher::Intrin::match_args
HALIDE_ALWAYS_INLINE bool match_args(int, const Call &c, MatcherState &state) const noexcept
Definition: IRMatch.h:1375
Halide::Internal::IRMatcher::NotOp::match
HALIDE_ALWAYS_INLINE bool match(const NotOp< A2 > &op, MatcherState &state) const noexcept
Definition: IRMatch.h:1635
Halide::Internal::IRMatcher::Wild::pattern_tag
Definition: IRMatch.h:467
Halide::Internal::IRMatcher::WildConstInt::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:279
Halide::Internal::IRMatcher::Wild
Definition: IRMatch.h:466
Halide::Internal::IRMatcher::VectorReduceOp::foldable
constexpr static bool foldable
Definition: IRMatch.h:1916
Halide::Internal::IRMatcher::WildConstInt::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:245
Halide::Internal::IRMatcher::MatcherState::set_binding
HALIDE_ALWAYS_INLINE void set_binding(int i, const BaseExprNode &n) noexcept
Definition: IRMatch.h:93
Halide::Internal::IRMatcher::SliceOp::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:2111
Halide::Internal::IRMatcher::CastOp::binds
constexpr static uint32_t binds
Definition: IRMatch.h:2043
Halide::Internal::IRMatcher::IsUInt::canonical
constexpr static bool canonical
Definition: IRMatch.h:2461
Halide::Internal::IRMatcher::operator<<
std::ostream & operator<<(std::ostream &s, const SpecificExpr &e)
Definition: IRMatch.h:229
Halide::Internal::Call::rounding_mul_shift_right
@ rounding_mul_shift_right
Definition: IR.h:576
Halide::Internal::IRNodeType::Min
@ Min
Halide::Internal::IRMatcher::VectorReduceOp::lanes
B lanes
Definition: IRMatch.h:1879
IR.h
Halide::Internal::IRMatcher::NegateOp::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:1962
Halide::Internal::IRMatcher::IsScalar::foldable
constexpr static bool foldable
Definition: IRMatch.h:2507
Halide::Internal::IRMatcher::Intrin::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:1369
Halide::Internal::Call::saturating_cast
@ saturating_cast
Definition: IR.h:581
Halide::Internal::IRMatcher::Rewriter::operator()
HALIDE_ALWAYS_INLINE bool operator()(Before before, After after, Predicate pred)
Definition: IRMatch.h:2904
Halide::Internal::Cast
The actual IR nodes begin here.
Definition: IR.h:29
Halide::Internal::LE
Is the first expression less than or equal to the second.
Definition: IR.h:140
Halide::Internal::make_signed_integer_overflow
Expr make_signed_integer_overflow(Type type)
Construct a unique signed_integer_overflow Expr.
Halide::Internal::Sub::_node_type
static const IRNodeType _node_type
Definition: IR.h:62
halide_type_t
A runtime tag for a type in the halide type system.
Definition: HalideRuntime.h:476
Halide::Internal::IRNodeType::GE
@ GE
Halide::Internal::IRMatcher::SpecificExpr::pattern_tag
Definition: IRMatch.h:205
Halide::Internal::IRMatcher::equal_helper
bool equal_helper(const BaseExprNode &a, const BaseExprNode &b) noexcept
Halide::Internal::IRMatcher::Intrin::intrin
Call::IntrinsicOp intrin
Definition: IRMatch.h:1359
Halide::Internal::IRMatcher::rounding_shift_left
auto rounding_shift_left(A &&a, B &&b) noexcept -> Intrin< decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1598
Halide::Internal::IRMatcher::CmpOp::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:767
Halide::Internal::NE
Is the first expression not equal to the second.
Definition: IR.h:122
Halide::Internal::IRMatcher::Overflow::foldable
constexpr static bool foldable
Definition: IRMatch.h:2263
Halide::Internal::IRMatcher::BroadcastOp::canonical
constexpr static bool canonical
Definition: IRMatch.h:1749
Halide::Internal::Call::rounding_halving_add
@ rounding_halving_add
Definition: IR.h:575
Halide::Internal::IRMatcher::widening_sub
auto widening_sub(A &&a, B &&b) noexcept -> Intrin< decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1556
Halide::Internal::IRMatcher::WildConst::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition: IRMatch.h:453
Halide::Internal::IRMatcher::NegateOp::match
HALIDE_ALWAYS_INLINE bool match(NegateOp< A2 > &&p, MatcherState &state) const noexcept
Definition: IRMatch.h:1978
Halide::Internal::Call::absd
@ absd
Definition: IR.h:512
Halide::Internal::IRMatcher::IntLiteral::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition: IRMatch.h:551
Halide::Internal::IRMatcher::IsInt
Definition: IRMatch.h:2406
Halide::Internal::IRMatcher::IsFloat
Definition: IRMatch.h:2369
Halide::Internal::IRMatcher::CastOp::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:2045
Halide::Internal::IRMatcher::SelectOp::c
C c
Definition: IRMatch.h:1675
Halide::Internal::IRMatcher::operator<
HALIDE_ALWAYS_INLINE auto operator<(A &&a, B &&b) noexcept -> CmpOp< LT, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1132
Halide::Internal::IRMatcher::operator<=
HALIDE_ALWAYS_INLINE auto operator<=(A &&a, B &&b) noexcept -> CmpOp< LE, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1182
Halide::Internal::IRMatcher::BinOp
Definition: IRMatch.h:640
Halide::Internal::IRMatcher::Wild::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:471
Halide::Internal::IRMatcher::RampOp::pattern_tag
Definition: IRMatch.h:1811
Halide::Internal::IRMatcher::WildConstFloat::binds
constexpr static uint32_t binds
Definition: IRMatch.h:357
Halide::NameMangling::C
@ C
No name mangling.
Halide::Internal::IRMatcher::is_int
HALIDE_ALWAYS_INLINE auto is_int(A &&a, int bits=0, int lanes=0) noexcept -> IsInt< decltype(pattern_arg(a))>
Definition: IRMatch.h:2432
Halide::Internal::IRMatcher::IsMaxValue::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const
Definition: IRMatch.h:2547
Halide::Internal::IRMatcher::operator>
HALIDE_ALWAYS_INLINE auto operator>(A &&a, B &&b) noexcept -> CmpOp< GT, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1157
Halide::Internal::IRMatcher::IntLiteral::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:516
Halide::Internal::IRMatcher::rounding_halving_add
auto rounding_halving_add(A &&a, B &&b) noexcept -> Intrin< decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1586
Halide::Internal::IRMatcher::or_op
HALIDE_ALWAYS_INLINE auto or_op(A &&a, B &&b) -> decltype(IRMatcher::operator||(a, b))
Definition: IRMatch.h:1287
Halide::Internal::IRMatcher::IsInt::pattern_tag
Definition: IRMatch.h:2407
Halide::Internal::IRMatcher::SliceOp::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:2098
Halide::Internal::IRMatcher::WildConstInt::match
HALIDE_ALWAYS_INLINE bool match(int64_t value, MatcherState &state) const noexcept
Definition: IRMatch.h:266
Halide::Internal::IRMatcher::Fold::canonical
constexpr static bool canonical
Definition: IRMatch.h:2155
Halide::Internal::IRMatcher::WildConstUInt
Definition: IRMatch.h:301
Halide::Internal::IRMatcher::CmpOp::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:820
Halide::Internal::IRMatcher::Overflows::a
A a
Definition: IRMatch.h:2204
Halide::Internal::IRMatcher::Fold::pattern_tag
Definition: IRMatch.h:2148
Halide::Internal::IRMatcher::widen_right_add
auto widen_right_add(A &&a, B &&b) noexcept -> Intrin< decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1539
uint64_t
unsigned __INT64_TYPE__ uint64_t
Definition: runtime_internal.h:23
Halide::Internal::IRMatcher::IsMinValue::pattern_tag
Definition: IRMatch.h:2575
Halide::Internal::Shuffle::slice_begin
int slice_begin() const
Check if this shuffle is a contiguous strict subset of the vector arguments, and if so,...
Definition: IR.h:871
Halide::Internal::IRMatcher::constant_fold_cmp_op
uint64_t constant_fold_cmp_op(int64_t, int64_t) noexcept
Halide::Internal::IRMatcher::IsUInt::a
A a
Definition: IRMatch.h:2453
Halide::Internal::IRMatcher::overflows
HALIDE_ALWAYS_INLINE auto overflows(A &&a) noexcept -> Overflows< decltype(pattern_arg(a))>
Definition: IRMatch.h:2227
Halide::Internal::IRMatcher::IsUInt::pattern_tag
Definition: IRMatch.h:2452
Halide::Type::is_vector
HALIDE_ALWAYS_INLINE bool is_vector() const
Is this type a vector type? (lanes() != 1).
Definition: Type.h:399
Halide::Internal::IRMatcher::MatcherState::get_binding
const HALIDE_ALWAYS_INLINE BaseExprNode * get_binding(int i) const noexcept
Definition: IRMatch.h:98
Halide::Internal::IRMatcher::Rewriter::operator()
HALIDE_ALWAYS_INLINE bool operator()(Before before, After after)
Definition: IRMatch.h:2838
Halide::Type
Types in the halide type system.
Definition: Type.h:276
Halide::Internal::IRMatcher::SpecificExpr::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:217
Halide::Internal::IRMatcher::SliceOp::lanes
Lanes lanes
Definition: IRMatch.h:2089
Halide::Internal::IRMatcher::IsUInt::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:2459
HALIDE_NEVER_INLINE
#define HALIDE_NEVER_INLINE
Definition: HalideRuntime.h:41
Halide::Internal::IRMatcher::constant_fold_bin_op< Mod >
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< Mod >(halide_type_t &t, int64_t a, int64_t b) noexcept
Definition: IRMatch.h:1073
Halide::Internal::IRMatcher::operator*
HALIDE_ALWAYS_INLINE auto operator*(A &&a, B &&b) noexcept -> BinOp< Mul, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:999
Halide::Internal::IRMatcher::CmpOp
Definition: IRMatch.h:751
Halide::Internal::Broadcast::make
static Expr make(Expr value, int lanes)
Halide::Internal::IRMatcher::saturating_sub
auto saturating_sub(A &&a, B &&b) noexcept -> Intrin< decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1568
Halide::Internal::Call::abs
@ abs
Definition: IR.h:511
Halide::Internal::expr_match
bool expr_match(const Expr &pattern, const Expr &expr, std::vector< Expr > &result)
Does the first expression have the same structure as the second? Variables in the first expression wi...
Halide::Internal::add_would_overflow
bool add_would_overflow(int bits, int64_t a, int64_t b)
Routines to test if math would overflow for signed integers with the given number of bits.
Halide::Internal::IRMatcher::constant_fold_cmp_op< GT >
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op< GT >(int64_t a, int64_t b) noexcept
Definition: IRMatch.h:1167
Halide::Internal::Call::likely
@ likely
Definition: IR.h:554
Halide::Internal::IRMatcher::Overflows::foldable
constexpr static bool foldable
Definition: IRMatch.h:2214
Halide::Internal::IRMatcher::SliceOp::foldable
constexpr static bool foldable
Definition: IRMatch.h:2123
Halide::Internal::Shuffle::make_slice
static Expr make_slice(Expr vector, int begin, int stride, int size)
Convenience constructor for making a shuffle representing a contiguous subset of a vector.
Halide::Internal::IRMatcher::is_const
HALIDE_ALWAYS_INLINE auto is_const(A &&a, int64_t value) noexcept -> IsConst< decltype(pattern_arg(a))>
Definition: IRMatch.h:2315
Halide
This file defines the class FunctionDAG, which is our representation of a Halide pipeline,...
Definition: AbstractGenerator.h:19
Halide::Internal::IRMatcher::RampOp::canonical
constexpr static bool canonical
Definition: IRMatch.h:1821
Halide::Internal::Or
Logical or - is at least one of the expression true.
Definition: IR.h:176
Halide::Internal::EQ
Is the first expression equal to the second.
Definition: IR.h:113
Halide::Internal::IRMatcher::IsFloat::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:2377
Halide::Internal::IRMatcher::NotOp::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:1621
Halide::Internal::IRMatcher::CastOp::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:2046
Halide::Internal::IRMatcher::BroadcastOp::binds
constexpr static uint32_t binds
Definition: IRMatch.h:1744
Halide::Internal::IRMatcher::WildConst::canonical
constexpr static bool canonical
Definition: IRMatch.h:415
Halide::Internal::IRMatcher::CanProve::a
A a
Definition: IRMatch.h:2333
Halide::Internal::IRMatcher::mul_shift_right
auto mul_shift_right(A &&a, B &&b, C &&c) noexcept -> Intrin< decltype(pattern_arg(a)), decltype(pattern_arg(b)), decltype(pattern_arg(c))>
Definition: IRMatch.h:1606
Halide::Internal::IRMatcher::negate
HALIDE_ALWAYS_INLINE auto negate(A &&a) -> decltype(IRMatcher::operator-(a))
Definition: IRMatch.h:2032
Halide::Internal::IRMatcher::IsConst::canonical
constexpr static bool canonical
Definition: IRMatch.h:2286
Halide::Internal::IRMatcher::WildConstInt::pattern_tag
Definition: IRMatch.h:236
Halide::Internal::IRMatcher::LanesOf::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:2625
Halide::Internal::IRMatcher::WildConstFloat::foldable
constexpr static bool foldable
Definition: IRMatch.h:392
halide_scalar_value_t
halide_scalar_value_t is a simple union able to represent all the well-known scalar values in a filte...
Definition: HalideRuntime.h:1662
Halide::Internal::IRMatcher::WildConstFloat::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:385
Halide::Internal::IRMatcher::RampOp::lanes
C lanes
Definition: IRMatch.h:1814
Halide::Internal::IRMatcher::IsFloat::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:2376
Halide::Internal::IRMatcher::pattern_arg
HALIDE_ALWAYS_INLINE T pattern_arg(T t)
Definition: IRMatch.h:579
Halide::Internal::Call::widening_mul
@ widening_mul
Definition: IR.h:606
Halide::Internal::IRMatcher::SelectOp::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:1682
Halide::LinkageType::Internal
@ Internal
Not visible externally, similar to 'static' linkage in C.
Halide::Internal::Max
The greater of two values.
Definition: IR.h:104
Halide::Internal::IRMatcher::WildConst::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:413
Halide::Internal::Sub::a
Expr a
Definition: IR.h:58
Halide::Internal::IRMatcher::Overflows::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:2210
Halide::Internal::IRMatcher::sub
HALIDE_ALWAYS_INLINE auto sub(A &&a, B &&b) -> decltype(IRMatcher::operator-(a, b))
Definition: IRMatch.h:973
Halide::Internal::IRMatcher::SliceOp::binds
static constexpr uint32_t binds
Definition: IRMatch.h:2091
Halide::Internal::IRMatcher::WildConstFloat::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:360
Halide::Internal::make_zero
Expr make_zero(Type t)
Construct the representation of zero in the given type.
Halide::Internal::IRMatcher::make_const_special_expr
HALIDE_NEVER_INLINE Expr make_const_special_expr(halide_type_t ty)
Definition: IRMatch.h:149
Halide::Internal::IRMatcher::NegateOp::binds
constexpr static uint32_t binds
Definition: IRMatch.h:1960
Halide::Internal::VectorReduce::Min
@ Min
Definition: IR.h:937
Halide::Internal::IRMatcher::IsMinValue::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:2581
Halide::Internal::IRMatcher::IsUInt::binds
constexpr static uint32_t binds
Definition: IRMatch.h:2456
Halide::Internal::IRMatcher::BinOp::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:648
Halide::Internal::IRMatcher::CastOp::a
A a
Definition: IRMatch.h:2041
Halide::Internal::IRMatcher::shift_left
auto shift_left(A &&a, B &&b) noexcept -> Intrin< decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1590
Halide::Internal::IRMatcher::IsMaxValue::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:2541
Halide::Internal::IRMatcher::SliceOp::base
Base base
Definition: IRMatch.h:2087
Halide::Internal::IRMatcher::Fold::a
A a
Definition: IRMatch.h:2149
Halide::Internal::IRMatcher::NotOp
Definition: IRMatch.h:1615
Halide::Internal::IRNodeType::Broadcast
@ Broadcast
Halide::Internal::IRMatcher::slice
HALIDE_ALWAYS_INLINE auto slice(Vec vec, Base base, Stride stride, Lanes lanes) noexcept -> SliceOp< decltype(pattern_arg(vec)), decltype(pattern_arg(base)), decltype(pattern_arg(stride)), decltype(pattern_arg(lanes))>
Definition: IRMatch.h:2141
Halide::Internal::IRMatcher::is_scalar
HALIDE_ALWAYS_INLINE auto is_scalar(A &&a) noexcept -> IsScalar< decltype(pattern_arg(a))>
Definition: IRMatch.h:2521
halide_scalar_value_t::i64
int64_t i64
Definition: HalideRuntime.h:1668
Halide::Internal::IRMatcher::eq
HALIDE_ALWAYS_INLINE auto eq(A &&a, B &&b) -> decltype(IRMatcher::operator==(a, b))
Definition: IRMatch.h:1237
Halide::Internal::IRMatcher::WildConstFloat::pattern_tag
Definition: IRMatch.h:355
Halide::Internal::IRNodeType::Add
@ Add
Halide::Internal::IRMatcher::RampOp::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:1818
Halide::Internal::VectorReduce::make
static Expr make(Operator op, Expr vec, int lanes)
Halide::Internal::IRMatcher::NotOp::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:1622
halide_scalar_value_t::u
union halide_scalar_value_t::@4 u
Halide::Internal::IRMatcher::IntLiteral::canonical
constexpr static bool canonical
Definition: IRMatch.h:508
Halide::Internal::IRMatcher::RampOp
Definition: IRMatch.h:1810
Halide::Internal::IRMatcher::Intrin::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition: IRMatch.h:1477
Halide::Internal::IRMatcher::WildConst::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:443
Halide::Internal::IRMatcher::WildConstInt::binds
constexpr static uint32_t binds
Definition: IRMatch.h:238
Halide::Internal::Select::make
static Expr make(Expr condition, Expr true_value, Expr false_value)
Halide::Internal::IRMatcher::le
HALIDE_ALWAYS_INLINE auto le(A &&a, B &&b) -> decltype(IRMatcher::operator<=(a, b))
Definition: IRMatch.h:1187
Halide::Internal::IRMatcher::BinOp::foldable
constexpr static bool foldable
Definition: IRMatch.h:673
Halide::Internal::IRMatcher::SpecificExpr::binds
constexpr static uint32_t binds
Definition: IRMatch.h:207
Halide::Internal::IRMatcher::IsScalar::pattern_tag
Definition: IRMatch.h:2497
Halide::Internal::IRMatcher::WildConstUInt::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:307
Halide::Internal::IRMatcher::BinOp::b
B b
Definition: IRMatch.h:643
Halide::Internal::IRMatcher::SpecificExpr::foldable
constexpr static bool foldable
Definition: IRMatch.h:226
Halide::Internal::IRMatcher::Wild::canonical
constexpr static bool canonical
Definition: IRMatch.h:473
Halide::Internal::IRMatcher::IsScalar::canonical
constexpr static bool canonical
Definition: IRMatch.h:2505
Halide::Internal::IRNodeType::Sub
@ Sub
Halide::Internal::IRMatcher::BroadcastOp::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:1770
Halide::Internal::IRMatcher::Intrin
Definition: IRMatch.h:1357
Halide::Internal::Ramp::lanes
int lanes
Definition: IR.h:241
Halide::Internal::IRMatcher::SelectOp::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:1704
Halide::Internal::VectorReduce::Max
@ Max
Definition: IR.h:938
Halide::Internal::IRMatcher::IntLiteral
Definition: IRMatch.h:500
Halide::Internal::IRNodeType::FloatImm
@ FloatImm
Halide::Internal::Ramp
A linear ramp vector node.
Definition: IR.h:239
Halide::Internal::Ramp::make
static Expr make(Expr base, Expr stride, int lanes)
HALIDE_ALWAYS_INLINE
#define HALIDE_ALWAYS_INLINE
Definition: HalideRuntime.h:40
Halide::Internal::IRMatcher::is_min_value
HALIDE_ALWAYS_INLINE auto is_min_value(A &&a) noexcept -> IsMinValue< decltype(pattern_arg(a))>
Definition: IRMatch.h:2605
Halide::Internal::IRMatcher::IsScalar
Definition: IRMatch.h:2496
Halide::Internal::IRMatcher::and_reduce
constexpr bool and_reduce()
Definition: IRMatch.h:1342
Halide::Internal::Select::true_value
Expr true_value
Definition: IR.h:197
Halide::Internal::Call::saturating_sub
@ saturating_sub
Definition: IR.h:580
Halide::Internal::IRMatcher::is_float
HALIDE_ALWAYS_INLINE auto is_float(A &&a) noexcept -> IsFloat< decltype(pattern_arg(a))>
Definition: IRMatch.h:2394
Halide::Internal::Broadcast::lanes
int lanes
Definition: IR.h:253
Halide::Expr::type
HALIDE_ALWAYS_INLINE Type type() const
Get the type of this expression node.
Definition: Expr.h:321
Halide::Internal::expr_match_test
void expr_match_test()
halide_type_t::lanes
uint16_t lanes
How many elements in a vector.
Definition: HalideRuntime.h:492
Halide::Internal::IRMatcher::IsScalar::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const
Definition: IRMatch.h:2510
Halide::Internal::IRMatcher::VectorReduceOp::pattern_tag
Definition: IRMatch.h:1877
Halide::Type::bits
HALIDE_ALWAYS_INLINE int bits() const
Return the bit size of a single element of this type.
Definition: Type.h:338
Halide::Internal::IRMatcher::Overflow
Definition: IRMatch.h:2238
Halide::Internal::IRMatcher::IsConst::pattern_tag
Definition: IRMatch.h:2279
Halide::Internal::IRNodeType::IntImm
@ IntImm
Halide::Internal::Call::widen_right_add
@ widen_right_add
Definition: IR.h:599
Halide::Internal::IRMatcher::SliceOp::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:2094
Halide::Internal::IRMatcher::NegateOp::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition: IRMatch.h:1992
Halide::Internal::IRMatcher::BinOp::a
A a
Definition: IRMatch.h:642
Halide::Internal::IRMatcher::MatcherState::bound_const
halide_scalar_value_t bound_const[max_wild]
Definition: IRMatch.h:84
Halide::Internal::debug
For optional debugging during codegen, use the debug class as follows:
Definition: Debug.h:49
Halide::Internal::IRMatcher::MatcherState::set_bound_const
HALIDE_ALWAYS_INLINE void set_bound_const(int i, int64_t s, halide_type_t t) noexcept
Definition: IRMatch.h:103
Halide::Internal::div_imp
T div_imp(T a, T b)
Definition: IROperator.h:260
Halide::Internal::IRMatcher::IsMaxValue::a
A a
Definition: IRMatch.h:2535
Halide::Internal::Call::widening_add
@ widening_add
Definition: IR.h:605
Halide::Internal::IRMatcher::BinOp::canonical
constexpr static bool canonical
Definition: IRMatch.h:653
Halide::Internal::IRMatcher::Intrin::print_args
HALIDE_ALWAYS_INLINE void print_args(int, std::ostream &s) const
Definition: IRMatch.h:1399
Halide::Internal::Sub::b
Expr b
Definition: IR.h:58
Halide::Internal::Call::shift_left
@ shift_left
Definition: IR.h:584
Halide::Internal::IRMatcher::CmpOp::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:759
Halide::Internal::IRMatcher::NegateOp::pattern_tag
Definition: IRMatch.h:1957
Halide::Internal::IRNodeType::And
@ And
Halide::Internal::IRMatcher::SliceOp::stride
Stride stride
Definition: IRMatch.h:2088
int64_t
signed __INT64_TYPE__ int64_t
Definition: runtime_internal.h:22
Halide::Internal::IRMatcher::IsUInt::bits
int bits
Definition: IRMatch.h:2454
Halide::Internal::IRMatcher::Wild::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:485
Halide::Internal::IRNodeType::Max
@ Max
Halide::Internal::BaseExprNode
A base class for expression nodes.
Definition: Expr.h:142
Halide::Internal::IRMatcher::MatcherState::special_values_mask
static constexpr uint16_t special_values_mask
Definition: IRMatch.h:88
halide_type_uint
@ halide_type_uint
unsigned integers
Definition: HalideRuntime.h:455
Halide::Internal::IRMatcher::BinOp::pattern_tag
Definition: IRMatch.h:641
Halide::Internal::IRMatcher::CmpOp::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:758
Halide::Internal::IRMatcher::Fold::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:2154
Halide::abs
Expr abs(Expr a)
Returns the absolute value of a signed integer or floating-point expression.
Halide::Internal::IRMatcher::Intrin::Intrin
HALIDE_ALWAYS_INLINE Intrin(Call::IntrinsicOp intrin, Args... args) noexcept
Definition: IRMatch.h:1520
Halide::Internal::Call::mul_shift_right
@ mul_shift_right
Definition: IR.h:560
Halide::Internal::Not::a
Expr a
Definition: IR.h:186
Halide::Internal::IRMatcher::IntLiteral::v
int64_t v
Definition: IRMatch.h:502
Halide::Internal::IRMatcher::Intrin::print_args
HALIDE_ALWAYS_INLINE void print_args(std::ostream &s) const
Definition: IRMatch.h:1412
Halide::Internal::IRMatcher::Fold
Definition: IRMatch.h:2147
Halide::Internal::IRMatcher::RampOp::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:1824
Halide::Internal::make_const
Expr make_const(Type t, int64_t val)
Construct an immediate of the given type from any numeric C++ type.
Halide::Internal::IRMatcher::operator!
HALIDE_ALWAYS_INLINE auto operator!(A &&a) noexcept -> NotOp< decltype(pattern_arg(a))>
Definition: IRMatch.h:1655
Halide::Internal::IRMatcher::Fold::binds
constexpr static uint32_t binds
Definition: IRMatch.h:2151
Halide::Internal::VectorReduce::And
@ And
Definition: IR.h:939
Halide::Internal::IRMatcher::IsScalar::a
A a
Definition: IRMatch.h:2498
Halide::Internal::IRMatcher::Overflows::binds
constexpr static uint32_t binds
Definition: IRMatch.h:2206
Halide::Internal::IRMatcher::constant_fold_cmp_op< EQ >
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op< EQ >(int64_t a, int64_t b) noexcept
Definition: IRMatch.h:1242
Halide::Internal::IRMatcher::NegateOp::canonical
constexpr static bool canonical
Definition: IRMatch.h:1965
Halide::Internal::Min
The lesser of two values.
Definition: IR.h:95
Halide::Internal::IRMatcher::BroadcastOp
Definition: IRMatch.h:1739
Halide::Internal::IRMatcher::BroadcastOp::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:1747
Halide::Internal::IRMatcher::MatcherState
To save stack space, the matcher objects are largely stateless and immutable.
Definition: IRMatch.h:82
Halide::Internal::IRMatcher::Rewriter::state
MatcherState state
Definition: IRMatch.h:2820
Halide::Internal::IRNodeType::VectorReduce
@ VectorReduce
Halide::Internal::IntImm::make
static const IntImm * make(Type t, int64_t value)
Halide::Internal::IRMatcher::widening_add
auto widening_add(A &&a, B &&b) noexcept -> Intrin< decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1552
Halide::Internal::IRMatcher::RampOp::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:1819
Halide::Internal::IRMatcher::IsMinValue::foldable
constexpr static bool foldable
Definition: IRMatch.h:2585
Halide::Internal::Mod
The remainder of a / b.
Definition: IR.h:86
Halide::Internal::Not::make
static Expr make(Expr a)
Halide::Internal::IRMatcher::IntLiteral::pattern_tag
Definition: IRMatch.h:501
Halide::Internal::VectorReduce::Or
@ Or
Definition: IR.h:940
Halide::Internal::IRMatcher::MatcherState::get_bound_const
HALIDE_ALWAYS_INLINE void get_bound_const(int i, halide_scalar_value_t &val, halide_type_t &type) const noexcept
Definition: IRMatch.h:127
Halide::Internal::IRMatcher::MatcherState::set_bound_const
HALIDE_ALWAYS_INLINE void set_bound_const(int i, double f, halide_type_t t) noexcept
Definition: IRMatch.h:115
Halide::Internal::IRMatcher::IsConst::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition: IRMatch.h:2295
Halide::Internal::IRMatcher::RampOp::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:1846
Halide::Internal::IRMatcher::WildConstFloat::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:364
Halide::Internal::IRMatcher::SelectOp::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:1681
Halide::Internal::IRMatcher::WildConst::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:418
Halide::Internal::IRMatcher::unwrap
HALIDE_ALWAYS_INLINE int64_t unwrap(IntLiteral t)
Definition: IRMatch.h:571
Halide::likely_if_innermost
Expr likely_if_innermost(Expr e)
Equivalent to likely, but only triggers a loop partitioning if found in an innermost loop.
Halide::Internal::IRMatcher::Wild::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:476
Halide::Internal::IRMatcher::SliceOp
Definition: IRMatch.h:2084
Halide::Internal::IRMatcher::Overflows::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition: IRMatch.h:2217
Halide::Internal::IRMatcher::WildConst::pattern_tag
Definition: IRMatch.h:409
Halide::Internal::IRMatcher::BroadcastOp::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:1752
Halide::Internal::IRMatcher::is_max_value
HALIDE_ALWAYS_INLINE auto is_max_value(A &&a) noexcept -> IsMaxValue< decltype(pattern_arg(a))>
Definition: IRMatch.h:2562
Halide::Internal::IRMatcher::Intrin::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:1387
Halide::Internal::IRMatcher::CastOp::match
HALIDE_ALWAYS_INLINE bool match(const CastOp< A2 > &op, MatcherState &state) const noexcept
Definition: IRMatch.h:2059
Halide::Type::is_scalar
HALIDE_ALWAYS_INLINE bool is_scalar() const
Is this type a scalar type? (lanes() == 1).
Definition: Type.h:406
Halide::Internal::IRMatcher::evaluate_predicate
HALIDE_ALWAYS_INLINE bool evaluate_predicate(bool x, MatcherState &) noexcept
Definition: IRMatch.h:2790
Halide::Internal::IRMatcher::CmpOp::pattern_tag
Definition: IRMatch.h:752
Halide::Internal::IRMatcher::BinOp::match
HALIDE_ALWAYS_INLINE bool match(const BinOp< Op2, A2, B2 > &op, MatcherState &state) const noexcept
Definition: IRMatch.h:667
Halide::Internal::IRMatcher::SelectOp::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:1687
Halide::Internal::Call::shift_right
@ shift_right
Definition: IR.h:585
Halide::Internal::IRMatcher::WildConstUInt::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:306
Halide::Internal::IRMatcher::operator!=
HALIDE_ALWAYS_INLINE auto operator!=(A &&a, B &&b) noexcept -> CmpOp< NE, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1257
Halide::Internal::IRNodeType::NE
@ NE
Halide::Internal::FloatImm::make
static const FloatImm * make(Type t, double value)
Halide::Internal::IRMatcher::WildConstInt::foldable
constexpr static bool foldable
Definition: IRMatch.h:286
internal_error
#define internal_error
Definition: Errors.h:23
Halide::Internal::IRMatcher::IsFloat::binds
constexpr static uint32_t binds
Definition: IRMatch.h:2373
Halide::Internal::IRMatcher::CanProve::foldable
constexpr static bool foldable
Definition: IRMatch.h:2343
Halide::Internal::Select::_node_type
static const IRNodeType _node_type
Definition: IR.h:201
Halide::Internal::IRMatcher::IsUInt::lanes
int lanes
Definition: IRMatch.h:2454
Halide::Internal::Call
A function call.
Definition: IR.h:482
Halide::Internal::IRMatcher::IsMinValue
Definition: IRMatch.h:2574
Halide::Internal::IRMatcher::IsMinValue::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:2582
Halide::Internal::IRMatcher::Overflows::pattern_tag
Definition: IRMatch.h:2203
Halide::Internal::IRMatcher::BroadcastOp::foldable
constexpr static bool foldable
Definition: IRMatch.h:1784
Halide::Internal::IRMatcher::enable_if_pattern
Definition: IRMatch.h:140
Halide::Internal::IRMatcher::NegateOp::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:1963
Halide::Internal::IRMatcher::Rewriter::result
Expr result
Definition: IRMatch.h:2819
Halide::Internal::Call::IntrinsicOp
IntrinsicOp
Definition: IR.h:510
Halide::Internal::IRMatcher::MatcherState::set_bound_const
HALIDE_ALWAYS_INLINE void set_bound_const(int i, uint64_t u, halide_type_t t) noexcept
Definition: IRMatch.h:109
Halide::Internal::Ramp::_node_type
static const IRNodeType _node_type
Definition: IR.h:245
Halide::Internal::IRMatcher::BinOp::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const noexcept
Definition: IRMatch.h:719
Halide::Internal::IRMatcher::Rewriter::wildcard_type
halide_type_t wildcard_type
Definition: IRMatch.h:2821
Halide::Internal::IRMatcher::Rewriter::operator()
HALIDE_ALWAYS_INLINE bool operator()(Before before, int64_t after, Predicate pred)
Definition: IRMatch.h:2956
Halide::Internal::IRMatcher::NotOp::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition: IRMatch.h:1647
Halide::Internal::IRMatcher::MatcherState::bound_const_type
halide_type_t bound_const_type[max_wild]
Definition: IRMatch.h:90
Halide::Internal::IRMatcher::div
HALIDE_ALWAYS_INLINE auto div(A &&a, B &&b) -> decltype(IRMatcher::operator/(a, b))
Definition: IRMatch.h:1039
Halide::Internal::IRNodeType::Mul
@ Mul
Halide::Internal::Call::rounding_shift_right
@ rounding_shift_right
Definition: IR.h:578
Halide::Internal::IRMatcher::WildConst
Definition: IRMatch.h:408
Halide::Internal::IRMatcher::fold
HALIDE_ALWAYS_INLINE auto fold(A &&a) noexcept -> Fold< decltype(pattern_arg(a))>
Definition: IRMatch.h:2190
Halide::Internal::Call::is_intrinsic
bool is_intrinsic() const
Definition: IR.h:690
Halide::Internal::IRMatcher::IntLiteral::IntLiteral
HALIDE_ALWAYS_INLINE IntLiteral(int64_t v)
Definition: IRMatch.h:511
Halide::Internal::IRMatcher::IntLiteral::foldable
constexpr static bool foldable
Definition: IRMatch.h:548
Halide::Internal::IRMatcher::IsMaxValue::canonical
constexpr static bool canonical
Definition: IRMatch.h:2542
Halide::Internal::IRMatcher::NotOp::binds
constexpr static uint32_t binds
Definition: IRMatch.h:1619
Halide::Internal::IRMatcher::SliceOp::canonical
constexpr static bool canonical
Definition: IRMatch.h:2095
Halide::Internal::IRMatcher::CmpOp::canonical
constexpr static bool canonical
Definition: IRMatch.h:760
Halide::Internal::IRMatcher::WildConstInt::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:241
Halide::Type::is_int
HALIDE_ALWAYS_INLINE bool is_int() const
Is this type a signed integer type?
Definition: Type.h:424
Halide::Internal::IRNodeType::GT
@ GT
Halide::Internal::IRMatcher::max
HALIDE_ALWAYS_INLINE auto max(A &&a, B &&b) noexcept -> BinOp< Max, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1110
Halide::Internal::IRMatcher::broadcast
HALIDE_ALWAYS_INLINE auto broadcast(A &&a, B lanes) noexcept -> BroadcastOp< decltype(pattern_arg(a)), decltype(pattern_arg(lanes))>
Definition: IRMatch.h:1804
Halide::Internal::IRMatcher::IsInt::lanes
int lanes
Definition: IRMatch.h:2409
Halide::Internal::Call::rounding_shift_left
@ rounding_shift_left
Definition: IR.h:577
Halide::Internal::IRMatcher::rounding_shift_right
auto rounding_shift_right(A &&a, B &&b) noexcept -> Intrin< decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1602
Halide::Internal::IRMatcher::NegateOp
Definition: IRMatch.h:1956
Halide::Internal::IRMatcher::CastOp
Definition: IRMatch.h:2038
Halide::Internal::IRMatcher::constant_fold_bin_op< Add >
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< Add >(halide_type_t &t, int64_t a, int64_t b) noexcept
Definition: IRMatch.h:947
Halide::Internal::IRMatcher::CanProve::binds
constexpr static uint32_t binds
Definition: IRMatch.h:2336
Halide::Internal::IRMatcher::IsMaxValue::foldable
constexpr static bool foldable
Definition: IRMatch.h:2544
Halide::Internal::IRMatcher::NegateOp::foldable
constexpr static bool foldable
Definition: IRMatch.h:1989
Halide::likely
Expr likely(Expr e)
Expressions tagged with this intrinsic are considered to be part of the steady state of some loop wit...
Halide::Internal::IRMatcher::mul
HALIDE_ALWAYS_INLINE auto mul(A &&a, B &&b) -> decltype(IRMatcher::operator*(a, b))
Definition: IRMatch.h:1006
Halide::Internal::IRNodeType::UIntImm
@ UIntImm
Halide::Internal::IRMatcher::Intrin::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:1417
Halide::Internal::Select
A ternary operator.
Definition: IR.h:196
Halide::Internal::IRMatcher::LanesOf::canonical
constexpr static bool canonical
Definition: IRMatch.h:2626
Halide::Internal::IRMatcher::CmpOp::foldable
constexpr static bool foldable
Definition: IRMatch.h:783
Halide::Internal::IRMatcher::Wild::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:472
Halide::Internal::IRMatcher::rewriter
HALIDE_ALWAYS_INLINE auto rewriter(Instance instance, halide_type_t output_type, halide_type_t wildcard_type) noexcept -> Rewriter< decltype(pattern_arg(instance))>
Construct a rewriter for the given instance, which may be a pattern with concrete expressions as leav...
Definition: IRMatch.h:2997
Halide::Internal::IRNodeType::Ramp
@ Ramp
Halide::Internal::IRMatcher::h_add
HALIDE_ALWAYS_INLINE auto h_add(A &&a, B lanes) noexcept -> VectorReduceOp< decltype(pattern_arg(a)), decltype(pattern_arg(lanes)), VectorReduce::Add >
Definition: IRMatch.h:1926
Halide::Internal::IRMatcher::IsInt::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const
Definition: IRMatch.h:2421
Halide::Internal::IRNode::node_type
IRNodeType node_type
Each IR node subclass has a unique identifier.
Definition: Expr.h:112
Halide::Internal::Select::false_value
Expr false_value
Definition: IR.h:197
Halide::Internal::IRMatcher::LanesOf::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const
Definition: IRMatch.h:2631
Halide::Internal::IRMatcher::IsUInt::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const
Definition: IRMatch.h:2466
Halide::Internal::IRMatcher::VectorReduceOp::binds
constexpr static uint32_t binds
Definition: IRMatch.h:1881
Halide::Internal::IRMatcher::operator>=
HALIDE_ALWAYS_INLINE auto operator>=(A &&a, B &&b) noexcept -> CmpOp< GE, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1207
Halide::Internal::Ramp::base
Expr base
Definition: IR.h:240
Halide::Internal::IRMatcher::CastOp::canonical
constexpr static bool canonical
Definition: IRMatch.h:2047
Halide::Internal::Sub::make
static Expr make(Expr a, Expr b)
Halide::Internal::IRMatcher::can_prove
HALIDE_ALWAYS_INLINE auto can_prove(A &&a, Prover *p) noexcept -> CanProve< decltype(pattern_arg(a)), Prover >
Definition: IRMatch.h:2357
Halide::Internal::IRMatcher::BinOp::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:657
Halide::Internal::IRMatcher::BinOp::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition: IRMatch.h:676
Halide::Internal::IRMatcher::ge
HALIDE_ALWAYS_INLINE auto ge(A &&a, B &&b) -> decltype(IRMatcher::operator>=(a, b))
Definition: IRMatch.h:1212
Halide::Internal::VectorReduce::_node_type
static const IRNodeType _node_type
Definition: IR.h:948
Halide::Internal::Call::_node_type
static const IRNodeType _node_type
Definition: IR.h:735
Halide::Internal::IRMatcher::NotOp::a
A a
Definition: IRMatch.h:1617
Halide::Internal::Broadcast::_node_type
static const IRNodeType _node_type
Definition: IR.h:257
Halide::Internal::IRMatcher::Overflow::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition: IRMatch.h:2266
Halide::Expr
A fragment of Halide syntax.
Definition: Expr.h:257
Halide::Internal::IRMatcher::h_and
HALIDE_ALWAYS_INLINE auto h_and(A &&a, B lanes) noexcept -> VectorReduceOp< decltype(pattern_arg(a)), decltype(pattern_arg(lanes)), VectorReduce::And >
Definition: IRMatch.h:1944
Halide::Internal::IRMatcher::NotOp::foldable
constexpr static bool foldable
Definition: IRMatch.h:1644
Halide::Internal::IRMatcher::Intrin::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:1368
Halide::Internal::IRMatcher::Intrin::foldable
constexpr static bool foldable
Definition: IRMatch.h:1475
Halide::Internal::mul_would_overflow
bool mul_would_overflow(int bits, int64_t a, int64_t b)
Halide::Internal::IRMatcher::Overflows::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:2211
halide_type_int
@ halide_type_int
signed integers
Definition: HalideRuntime.h:454
Halide::Internal::IRMatcher::IntLiteral::match
HALIDE_ALWAYS_INLINE bool match(int64_t val, MatcherState &state) const noexcept
Definition: IRMatch.h:534
Halide::Internal::IRMatcher::h_min
HALIDE_ALWAYS_INLINE auto h_min(A &&a, B lanes) noexcept -> VectorReduceOp< decltype(pattern_arg(a)), decltype(pattern_arg(lanes)), VectorReduce::Min >
Definition: IRMatch.h:1932
Halide::Internal::IRMatcher::Overflows::canonical
constexpr static bool canonical
Definition: IRMatch.h:2212
Halide::Internal::IRMatcher::widen_right_sub
auto widen_right_sub(A &&a, B &&b) noexcept -> Intrin< decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1547
Halide::Internal::IRMatcher::NotOp::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:1626
Halide::Internal::IRMatcher::WildConstUInt::make_folded_const
HALIDE_ALWAYS_INLINE void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept
Definition: IRMatch.h:342
Halide::Internal::IRMatcher::WildConstUInt::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:311
Halide::Internal::GT
Is the first expression greater than the second.
Definition: IR.h:149
Halide::Internal::IRMatcher::SelectOp::foldable
constexpr static bool foldable
Definition: IRMatch.h:1708
Halide::Internal::IRMatcher::RampOp::binds
constexpr static uint32_t binds
Definition: IRMatch.h:1816
Halide::Internal::IRMatcher::operator/
HALIDE_ALWAYS_INLINE auto operator/(A &&a, B &&b) noexcept -> BinOp< Div, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1032
Halide::Internal::Shuffle
Construct a new vector by taking elements from another sequence of vectors.
Definition: IR.h:819
Halide::Internal::IRMatcher::WildConstUInt::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:332
Halide::Internal::IRMatcher::lanes_of
HALIDE_ALWAYS_INLINE auto lanes_of(A &&a) noexcept -> LanesOf< decltype(pattern_arg(a))>
Definition: IRMatch.h:2642
Halide::Internal::IRMatcher::IsInt::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:2414
Halide::Internal::IRMatcher::Overflows
Definition: IRMatch.h:2202
Halide::Internal::IRMatcher::SliceOp::pattern_tag
Definition: IRMatch.h:2085
Halide::Internal::IRMatcher::h_or
HALIDE_ALWAYS_INLINE auto h_or(A &&a, B lanes) noexcept -> VectorReduceOp< decltype(pattern_arg(a)), decltype(pattern_arg(lanes)), VectorReduce::Or >
Definition: IRMatch.h:1950
Halide::Internal::IRMatcher::IsScalar::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:2504
Halide::Internal::IRMatcher::Rewriter
Definition: IRMatch.h:2817
Halide::Internal::IRMatcher::SliceOp::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:2093
Halide::Internal::IRMatcher::NotOp::pattern_tag
Definition: IRMatch.h:1616
Halide::Expr::get
const HALIDE_ALWAYS_INLINE Internal::BaseExprNode * get() const
Override get() to return a BaseExprNode * instead of an IRNode *.
Definition: Expr.h:315
Halide::Internal::IRMatcher::saturating_add
auto saturating_add(A &&a, B &&b) noexcept -> Intrin< decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1564
Halide::Internal::IRMatcher::Intrin::args
std::tuple< Args... > args
Definition: IRMatch.h:1360
uint32_t
unsigned __INT32_TYPE__ uint32_t
Definition: runtime_internal.h:25
Halide::Internal::IRNodeType::Shuffle
@ Shuffle
Halide::Internal::IRMatcher::LanesOf::binds
constexpr static uint32_t binds
Definition: IRMatch.h:2621
Halide::Internal::UIntImm
Unsigned integer constants.
Definition: Expr.h:226
Halide::Internal::IRMatcher::IsMaxValue::binds
constexpr static uint32_t binds
Definition: IRMatch.h:2537
Halide::Internal::IRMatcher::IsScalar::binds
constexpr static uint32_t binds
Definition: IRMatch.h:2500
Halide::Internal::IRMatcher::CanProve::canonical
constexpr static bool canonical
Definition: IRMatch.h:2341
Halide::Internal::IRMatcher::constant_fold_bin_op< Min >
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< Min >(halide_type_t &t, int64_t a, int64_t b) noexcept
Definition: IRMatch.h:1095
Halide::Internal::IRMatcher::BroadcastOp::lanes
B lanes
Definition: IRMatch.h:1742
Halide::Internal::IRMatcher::Fold::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:2153
Halide::Internal::IRMatcher::operator-
HALIDE_ALWAYS_INLINE auto operator-(A &&a, B &&b) noexcept -> BinOp< Sub, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:966
Halide::Internal::IRMatcher::IsConst::check_v
bool check_v
Definition: IRMatch.h:2289
Halide::Internal::IRMatcher::constant_fold_cmp_op< NE >
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op< NE >(int64_t a, int64_t b) noexcept
Definition: IRMatch.h:1267
IREquality.h
Halide::Internal::IRMatcher::Overflow::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:2249
Halide::Internal::IRMatcher::constant_fold_cmp_op< GE >
HALIDE_ALWAYS_INLINE uint64_t constant_fold_cmp_op< GE >(int64_t a, int64_t b) noexcept
Definition: IRMatch.h:1217
Halide::Internal::IRMatcher::IsUInt::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:2460
Halide::Internal::IRMatcher::IsUInt
Definition: IRMatch.h:2451
Halide::Internal::IRMatcher::WildConstFloat::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:359
Halide::Internal::is_const_one
bool is_const_one(const Expr &e)
Is the expression a const (as defined by is_const), and also equal to one (in all lanes,...
Halide::Internal::IRMatcher::constant_fold_bin_op< Max >
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< Max >(halide_type_t &t, int64_t a, int64_t b) noexcept
Definition: IRMatch.h:1117
Halide::Internal::IRMatcher::BroadcastOp::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:1746
Halide::Internal::IRMatcher::enable_if_pattern::type
Definition: IRMatch.h:141
Halide::Internal::Sub
The difference of two expressions.
Definition: IR.h:57
Halide::Internal::IRMatcher::CanProve::prover
Prover * prover
Definition: IRMatch.h:2334
halide_scalar_value_t::f64
double f64
Definition: HalideRuntime.h:1674
Halide::Internal::IRMatcher::and_op
HALIDE_ALWAYS_INLINE auto and_op(A &&a, B &&b) -> decltype(IRMatcher::operator&&(a, b))
Definition: IRMatch.h:1313
Halide::Internal::IRMatcher::Wild::foldable
constexpr static bool foldable
Definition: IRMatch.h:489
Halide::Internal::IRMatcher::constant_fold_bin_op< Sub >
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< Sub >(halide_type_t &t, int64_t a, int64_t b) noexcept
Definition: IRMatch.h:980
Halide::Internal::IRMatcher::constant_fold_bin_op< Or >
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< Or >(halide_type_t &t, int64_t a, int64_t b) noexcept
Definition: IRMatch.h:1292
Halide::Internal::Call::widen_right_mul
@ widen_right_mul
Definition: IR.h:601
Halide::Internal::IRMatcher::IsConst::binds
constexpr static uint32_t binds
Definition: IRMatch.h:2281
Halide::Internal::IRMatcher::CmpOp::match
HALIDE_ALWAYS_INLINE bool match(const CmpOp< Op2, A2, B2 > &op, MatcherState &state) const noexcept
Definition: IRMatch.h:777
Halide::Internal::Ramp::stride
Expr stride
Definition: IR.h:240
Halide::Internal::IRMatcher::SelectOp::match
HALIDE_ALWAYS_INLINE bool match(const SelectOp< C2, T2, F2 > &instance, MatcherState &state) const noexcept
Definition: IRMatch.h:1697
Halide::Internal::IRMatcher::IsMaxValue::pattern_tag
Definition: IRMatch.h:2534
Halide::Internal::IRMatcher::CanProve::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:2339
Halide::Internal::IRNodeType::Call
@ Call
Halide::Internal::IRMatcher::WildConst::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:414
Halide::Internal::And
Logical and - are both expressions true.
Definition: IR.h:167
Halide::Internal::IRMatcher::operator==
HALIDE_ALWAYS_INLINE auto operator==(A &&a, B &&b) noexcept -> CmpOp< EQ, decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1232
Halide::Internal::IRMatcher::NegateOp::match
HALIDE_ALWAYS_INLINE bool match(const BaseExprNode &e, MatcherState &state) const noexcept
Definition: IRMatch.h:1968
Halide::Internal::IRMatcher::halving_sub
auto halving_sub(A &&a, B &&b) noexcept -> Intrin< decltype(pattern_arg(a)), decltype(pattern_arg(b))>
Definition: IRMatch.h:1582
Halide::Internal::Call::likely_if_innermost
@ likely_if_innermost
Definition: IR.h:555
Halide::Internal::IRMatcher::WildConstInt::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:240
Halide::Internal::Cast::_node_type
static const IRNodeType _node_type
Definition: IR.h:34
Halide::Internal::IRMatcher::NotOp::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:1640
Halide::Internal::IRMatcher::WildConstFloat
Definition: IRMatch.h:354
Halide::Internal::IRMatcher::CanProve::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:2340
Halide::Internal::IRMatcher::CastOp::t
Type t
Definition: IRMatch.h:2040
Halide::Internal::IRMatcher::IsConst::v
int64_t v
Definition: IRMatch.h:2290
Halide::Internal::Call::halving_add
@ halving_add
Definition: IR.h:543
Halide::Internal::IRMatcher::RampOp::match
HALIDE_ALWAYS_INLINE bool match(const RampOp< A2, B2, C2 > &op, MatcherState &state) const noexcept
Definition: IRMatch.h:1839
Halide::Internal::IRMatcher::WildConst::binds
constexpr static uint32_t binds
Definition: IRMatch.h:411
Halide::Internal::IRMatcher::Rewriter::operator()
HALIDE_ALWAYS_INLINE bool operator()(Before before, int64_t after) noexcept
Definition: IRMatch.h:2879
Halide::Internal::IRMatcher::WildConstUInt::canonical
constexpr static bool canonical
Definition: IRMatch.h:308
Halide::Internal::Not
Logical not - true if the expression false.
Definition: IR.h:185
Halide::Internal::IRMatcher::Intrin::print_args
HALIDE_ALWAYS_INLINE void print_args(double, std::ostream &s) const
Definition: IRMatch.h:1408
Halide::Internal::LT
Is the first expression less than the second.
Definition: IR.h:131
Halide::Internal::IRMatcher::Wild::binds
constexpr static uint32_t binds
Definition: IRMatch.h:469
Halide::Internal::IRMatcher::constant_fold_bin_op< And >
HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op< And >(halide_type_t &t, int64_t a, int64_t b) noexcept
Definition: IRMatch.h:1318
Halide::Internal::IRMatcher::equal
HALIDE_ALWAYS_INLINE bool equal(const BaseExprNode &a, const BaseExprNode &b) noexcept
Definition: IRMatch.h:195
halide_scalar_value_t::u64
uint64_t u64
Definition: HalideRuntime.h:1672
Halide::Internal::Mul
The product of two expressions.
Definition: IR.h:66
halide_type_t::code
uint8_t code
The basic type code: signed integer, unsigned integer, or floating point.
Definition: HalideRuntime.h:483
Halide::Internal::IRMatcher::add
HALIDE_ALWAYS_INLINE auto add(A &&a, B &&b) -> decltype(IRMatcher::operator+(a, b))
Definition: IRMatch.h:940
Halide::Internal::VectorReduce::Add
@ Add
Definition: IR.h:934
Halide::Internal::IRMatcher::Rewriter::build_replacement
HALIDE_NEVER_INLINE void build_replacement(After after)
Definition: IRMatch.h:2830
Halide::Internal::IRMatcher::IsConst::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:2285
Halide::Internal::IRMatcher::VectorReduceOp::make
HALIDE_ALWAYS_INLINE Expr make(MatcherState &state, halide_type_t type_hint) const
Definition: IRMatch.h:1908
Halide::Internal::IRMatcher::BroadcastOp::a
A a
Definition: IRMatch.h:1741
Halide::Internal::IRMatcher::Overflow::binds
constexpr static uint32_t binds
Definition: IRMatch.h:2241
Halide::Internal::IRMatcher::VectorReduceOp::min_node_type
constexpr static IRNodeType min_node_type
Definition: IRMatch.h:1883
Halide::Internal::IRMatcher::VectorReduceOp::max_node_type
constexpr static IRNodeType max_node_type
Definition: IRMatch.h:1884