Go to the documentation of this file. 1 #ifndef HALIDE_CODEGEN_LLVM_H
2 #define HALIDE_CODEGEN_LLVM_H
15 class IRBuilderDefaultInserter;
17 template<
typename,
typename>
24 class ExecutionEngine;
49 struct ExternSignature;
81 const std::string &suffix,
82 const std::vector<std::pair<std::string, ExternSignature>> &externs);
93 virtual void compile_func(
const LoweredFunc &func,
const std::string &simple_name,
const std::string &extern_name);
105 const std::string &extern_name,
const std::vector<LoweredArgument> &args);
106 virtual void end_func(
const std::vector<LoweredArgument> &args);
123 virtual std::string
mcpu_tune()
const = 0;
124 virtual std::string
mattrs()
const = 0;
125 virtual std::string
mabi()
const;
165 llvm::Function *
function;
167 std::unique_ptr<llvm::IRBuilder<llvm::ConstantFolder, llvm::IRBuilderDefaultInserter>>
builder;
194 void sym_pop(
const std::string &name);
199 llvm::Value *
sym_get(
const std::string &name,
200 bool must_succeed =
true)
const;
203 bool sym_exists(
const std::string &name)
const;
276 void create_assertion(llvm::Value *condition,
const Expr &message, llvm::Value *error_code =
nullptr);
290 llvm::Constant *
create_binary_blob(
const std::vector<char> &data,
const std::string &name,
bool constant =
true);
352 void visit(
const EQ *)
override;
353 void visit(
const NE *)
override;
354 void visit(
const LT *)
override;
355 void visit(
const LE *)
override;
356 void visit(
const GT *)
override;
357 void visit(
const GE *)
override;
359 void visit(
const Or *)
override;
414 int effective_vscale)
const;
419 bool zero_initialize =
false,
420 const std::string &name =
"");
461 llvm::Function *
get_llvm_intrin(
const Type &ret_type,
const std::string &name,
const std::vector<Type> &arg_types,
bool scalars_are_vectors =
false);
462 llvm::Function *
get_llvm_intrin(llvm::Type *ret_type,
const std::string &name,
const std::vector<llvm::Type *> &arg_types);
464 llvm::Function *
declare_intrin_overload(
const std::string &name,
const Type &ret_type,
const std::string &impl_name, std::vector<Type> arg_types,
bool scalars_are_vectors =
false);
479 const std::string &name, std::vector<Expr>);
481 llvm::Function *
intrin, std::vector<Expr>);
482 llvm::Value *
call_intrin(
const llvm::Type *t,
int intrin_lanes,
483 const std::string &name, std::vector<llvm::Value *>,
484 bool scalable_vector_result =
false,
bool is_reduction =
false);
485 llvm::Value *
call_intrin(
const llvm::Type *t,
int intrin_lanes,
486 llvm::Function *
intrin, std::vector<llvm::Value *>,
487 bool is_reduction =
false);
492 virtual llvm::Value *
slice_vector(llvm::Value *vec,
int start,
int extent);
495 virtual llvm::Value *
concat_vectors(
const std::vector<llvm::Value *> &);
499 const std::vector<int> &indices);
501 llvm::Value *
shuffle_vectors(llvm::Value *v,
const std::vector<int> &indices);
546 const std::vector<llvm::Value *> &args,
547 llvm::Type *result_type);
557 llvm::Type *desired_type);
615 using MaskVariant = std::variant<NoMask, AllEnabledMask, llvm::Value *>;
664 std::map<std::string, llvm::Constant *> string_constants;
669 llvm::BasicBlock *destructor_block;
675 bool llvm_large_code_model;
680 int effective_vscale;
686 int producer_consumer_id = 0;
694 llvm::Function *embed_metadata_getter(
const std::string &metadata_getter_name,
695 const std::string &function_name,
const std::vector<LoweredArgument> &args,
699 llvm::Constant *embed_constant_expr(
Expr e, llvm::Type *t);
700 llvm::Constant *embed_constant_scalar_value_t(
const Expr &e);
702 llvm::Function *add_argv_wrapper(llvm::Function *fn,
const std::string &name,
703 bool result_in_argv, std::vector<bool> &arg_is_buffer);
705 llvm::Value *codegen_vector_load(
const Type &type,
const std::string &name,
const Expr &base,
707 llvm::Value *vpred =
nullptr,
bool slice_to_native =
true, llvm::Value *stride =
nullptr);
709 virtual void codegen_predicated_load(
const Load *op);
710 virtual void codegen_predicated_store(
const Store *op);
712 void codegen_atomic_rmw(
const Store *op);
714 void init_codegen(
const std::string &name,
bool any_strict_float =
false);
715 std::unique_ptr<llvm::Module> finish_codegen();
718 template<
typename Op>
719 bool try_to_fold_vector_reduce(
const Expr &a,
Expr b);
726 std::map<llvm::Value *, llvm::Type *> struct_type_recovery;
733 llvm::LLVMContext &context);
signed __INT32_TYPE__ int32_t
virtual int maximum_vector_bits() const
Used to decide whether to break a vector up into multiple smaller operations.
Type indicating an intrinsic does not take a mask.
virtual Type upgrade_type_for_argument_passing(const Type &) const
Return the type that a Halide type should be passed in and out of functions as.
Allocate a scratch area called with the given name, type, and size.
HALIDE_ALWAYS_INLINE auto intrin(Call::IntrinsicOp intrinsic_op, Args... args) noexcept -> Intrin< decltype(pattern_arg(args))... >
The sum of two expressions.
virtual void prepare_for_early_exit()
If we have to bail out of a pipeline midway, this should inject the appropriate target-specific clean...
virtual std::string mcpu_target() const =0
What should be passed as -mcpu (warning: implies attrs!), -mattrs, and related for compilation.
A code generator abstract base class.
virtual void visit(const IntImm *)
void sym_pop(const std::string &name)
Remove an entry for the symbol table, revealing any previous entries with the same name.
virtual void compile_buffer(const Buffer<> &buffer)
void set_context(llvm::LLVMContext &context)
Tell the code generator which LLVM context to use.
llvm::FunctionType * signature_to_type(const ExternSignature &signature)
Given a Halide ExternSignature, return the equivalent llvm::FunctionType.
static void initialize_llvm()
Initialize internal llvm state for the enabled targets.
virtual llvm::Value * shuffle_vectors(llvm::Value *a, llvm::Value *b, const std::vector< int > &indices)
Create an LLVM shuffle vectors instruction.
Horizontally reduce a vector to a scalar or narrower vector using the given commutative and associati...
llvm::Value * convert_fixed_or_scalable_vector_type(llvm::Value *arg, llvm::Type *desired_type)
Convert between two LLVM vectors of potentially different scalable/fixed and size.
bool inside_atomic_mutex_node
Are we inside an atomic node that uses mutex locks? This is used for detecting deadlocks from nested ...
llvm::Value * sym_get(const std::string &name, bool must_succeed=true) const
Fetch an entry from the symbol table.
A reference-counted handle to a parameter to a halide pipeline.
llvm::Value * fixed_to_scalable_vector_type(llvm::Value *fixed)
Convert an LLVM fixed vector value to the corresponding vscale vector value.
llvm::Value * create_alloca_at_entry(llvm::Type *type, int n, bool zero_initialize=false, const std::string &name="")
Perform an alloca at the function entrypoint.
virtual void end_func(const std::vector< LoweredArgument > &args)
void optimize_module()
Run all of llvm's optimization passes on the module.
llvm::Value * make_halide_type_t(const Type &)
Turn a Halide Type into an llvm::Value representing a constant halide_type_t.
void function_does_not_access_memory(llvm::Function *fn)
Add the appropriate function attribute to tell LLVM that the function doesn't access memory.
Is the first expression greater than or equal to the second.
A base class for algorithms that need to recursively walk over the IR.
const Target & get_target() const
The target we're generating code for.
virtual void init_context()
Grab all the context specific internal state.
CodeGen_LLVM(const Target &t)
@ VScale
Force use of fixed size vectors.
Floating point constants.
virtual int target_vscale() const
For architectures that have vscale vectors, return the constant vscale to use.
llvm::Value * codegen_dense_vector_load(const Load *load, llvm::Value *vpred=nullptr, bool slice_to_native=true)
Generate a basic dense vector load, with an optional predicate and control over whether or not we sho...
A vector with 'lanes' elements, in which every element is 'value'.
The ratio of two expressions.
virtual Type upgrade_type_for_storage(const Type &) const
Return the type that a given Halide type should be stored/loaded from memory as.
std::pair< llvm::Function *, int > find_vector_runtime_function(const std::string &name, int lanes)
Go looking for a vector version of a runtime function.
llvm::Value * codegen(const Expr &)
Emit code that evaluates an expression, and return the llvm representation of the result of the expre...
The statement form of a let node.
virtual int native_vector_bits() const =0
What's the natural vector bit-width to use for loads, stores, etc.
void return_with_error_code(llvm::Value *error_code)
Return the the pipeline with the given error code.
virtual void compile_func(const LoweredFunc &func, const std::string &simple_name, const std::string &extern_name)
Compile a specific halide declaration into the llvm Module.
llvm::StructType * pseudostack_slot_t_type
llvm::Value * get_user_context() const
The user_context argument.
bool try_vector_predication_intrinsic(const std::string &name, VPResultType result_type, int32_t length, MaskVariant mask, std::vector< VPArg > args)
Generate an intrisic call if use_llvm_vp_intrinsics is true and length is greater than 1.
The actual IR nodes begin here.
Is the first expression less than or equal to the second.
virtual std::string get_allocation_name(const std::string &n)
Get a unique name for the actual block of memory that an allocate node uses.
Support for generating LLVM vector predication intrinsics ("@llvm.vp.*" and "@llvm....
void trigger_destructor(llvm::Function *destructor_fn, llvm::Value *stack_slot)
Call a destructor early.
llvm::Value * normalize_fixed_scalable_vector_type(llvm::Type *desired_type, llvm::Value *result)
Ensure that a vector value is either fixed or vscale depending to match desired_type.
Is the first expression not equal to the second.
llvm::Constant * get_splat(int lanes, llvm::Constant *value, VectorTypeConstraint type_constraint=VectorTypeConstraint::None) const
llvm::StructType * halide_buffer_t_type
std::optional< size_t > mangle_index
A reference-counted handle to a statement node.
std::unique_ptr< llvm::IRBuilder< llvm::ConstantFolder, llvm::IRBuilderDefaultInserter > > builder
llvm::Value * simple_call_intrin(const std::string &intrin, const std::vector< llvm::Value * > &args, llvm::Type *result_type)
call_intrin does far too much to be useful and generally breaks things when one has carefully set thi...
void create_assertion(llvm::Value *condition, const Expr &message, llvm::Value *error_code=nullptr)
Codegen an assertion.
VectorTypeConstraint
Interface to abstract vector code generation as LLVM is now providing multiple options to express eve...
bool emit_atomic_stores
Emit atomic store instructions?
Types in the halide type system.
Load a value from a named symbol if predicate is true.
Free the resources associated with the given buffer.
Allocate a multi-dimensional buffer of the given type and size.
This file defines the class FunctionDAG, which is our representation of a Halide pipeline,...
Logical or - is at least one of the expression true.
virtual bool supports_atomic_add(const Type &t) const
Is the first expression equal to the second.
This defines the value of a function at a multi-dimensional location.
std::unique_ptr< llvm::Module > module
@ Internal
Not visible externally, similar to 'static' linkage in C.
Description of an intrinsic function overload.
The greater of two values.
Intrinsic(Type result_type, std::vector< Type > arg_types, llvm::Function *impl)
llvm::Value * codegen_buffer_pointer(const std::string &buffer, Type type, llvm::Value *index)
Generate a pointer into a named buffer at a given index, of a given type.
llvm::BasicBlock * get_destructor_block()
Retrieves the block containing the error handling code.
std::optional< size_t > mangle_index
VPResultType(llvm::Type *type, std::optional< size_t > mangle_index=std::nullopt)
virtual llvm::Value * interleave_vectors(const std::vector< llvm::Value * > &)
Implementation of the intrinsic call to interleave_vectors.
llvm::Value * scalable_to_fixed_vector_type(llvm::Value *scalable)
Convert an LLVM vscale vector value to the corresponding fixed vector value.
virtual llvm::Value * slice_vector(llvm::Value *vec, int start, int extent)
Take a slice of lanes out of an llvm vector.
LinkageType
Type of linkage a function in a lowered Halide module can have.
WarningKind
Warning messages which we want to avoid displaying number of times.
size_t requested_alloca_total
A (very) conservative guess at the size of all alloca() storage requested (including alignment paddin...
std::unique_ptr< llvm::Module > codegen_llvm(const Module &module, llvm::LLVMContext &context)
Given a Halide module, generate an llvm::Module.
virtual bool promote_indices() const
Should indexing math be promoted to 64-bit on platforms with 64-bit pointers?
A let expression, like you might find in a functional language.
void sym_push(const std::string &name, llvm::Value *value)
Add an entry to the symbol table, hiding previous entries with the same name.
DestructorType
Some destructors should always be called.
std::variant< NoMask, AllEnabledMask, llvm::Value * > MaskVariant
Predication mask using the above two types for special cases and an llvm::Value for the general one.
bool try_vector_predication_comparison(const std::string &name, const Type &result_type, MaskVariant mask, llvm::Value *a, llvm::Value *b, const char *cmp_op)
Generate a vector predicated comparison intrinsic call if use_llvm_vp_intrinsics is true and result_t...
llvm::Value * create_broadcast(llvm::Value *, int lanes)
Widen an llvm scalar into an llvm vector with the given number of lanes.
Halide::Target target
The target we're generating code for.
@ Fixed
Use default for current target.
A linear ramp vector node.
llvm::StructType * scalar_value_t_type
virtual std::string mabi() const
llvm::Value * register_destructor(llvm::Function *destructor_fn, llvm::Value *obj, DestructorType when)
Evaluate and discard an expression, presumably because it has some side-effect.
static std::unique_ptr< CodeGen_LLVM > new_for_target(const Target &target, llvm::LLVMContext &context)
Create an instance of CodeGen_LLVM suitable for the target.
virtual void begin_func(LinkageType linkage, const std::string &simple_name, const std::string &extern_name, const std::vector< LoweredArgument > &args)
Helper functions for compiling Halide functions to llvm functions.
llvm::Value * call_overloaded_intrin(const Type &result_type, const std::string &name, const std::vector< Expr > &args)
Call an overloaded intrinsic function.
virtual bool use_pic() const
std::map< std::string, std::string > MetadataNameMap
Store a 'value' to the buffer called 'name' at a given 'index' if 'predicate' is true.
Definition of a lowered function.
std::map< WarningKind, std::string > onetime_warnings
The lesser of two values.
llvm::Constant * create_string_constant(const std::string &str)
Put a string constant in the module as a global variable and return a pointer to it.
This node is a helpful annotation to do with permissions.
void visit(const IntImm *) override
Generate code for various IR nodes.
virtual void codegen_vector_reduce(const VectorReduce *op, const Expr &init)
Compile a horizontal reduction that starts with an explicit initial value.
llvm::LLVMContext * context
static std::unique_ptr< llvm::Module > compile_trampolines(const Target &target, llvm::LLVMContext &context, const std::string &suffix, const std::vector< std::pair< std::string, ExternSignature >> &externs)
If the 'condition' is false, then evaluate and return the message, which should be a call to an error...
llvm::Function * get_llvm_intrin(const Type &ret_type, const std::string &name, const std::vector< Type > &arg_types, bool scalars_are_vectors=false)
Get an LLVM intrinsic declaration.
virtual std::string mattrs() const =0
llvm::Function * declare_intrin_overload(const std::string &name, const Type &ret_type, const std::string &impl_name, std::vector< Type > arg_types, bool scalars_are_vectors=false)
Declare an intrinsic function that participates in overload resolution.
int get_vector_num_elements(const llvm::Type *t)
Get number of vector elements, taking into account scalable vectors.
llvm::StructType * semaphore_t_type
virtual Type upgrade_type_for_arithmetic(const Type &) const
Return the type in which arithmetic should be done for the given storage type.
std::vector< LoweredArgument > current_function_args
llvm::Value * call_intrin(const Type &t, int intrin_lanes, const std::string &name, std::vector< Expr >)
Generate a call to a vector intrinsic or runtime inlined function.
Expr wild_u1x_
Some wildcard variables used for peephole optimizations in subclasses.
void scalarize(const Expr &)
Codegen a vector Expr by codegenning each lane and combining.
llvm::StructType * metadata_t_type
virtual bool supports_call_as_float16(const Call *op) const
Can we call this operation with float16 type? This is used to avoid "emulated" equivalent code-gen in...
bool sym_exists(const std::string &name) const
Test if an item exists in the symbol table.
llvm::Type * void_t
Some useful llvm types.
Reinterpret value as another type, without affecting any of the bits (on little-endian systems).
llvm::StructType * dimension_t_type
llvm::Constant * create_binary_blob(const std::vector< char > &data, const std::string &name, bool constant=true)
Put a binary blob in the module as a global variable and return a pointer to it.
llvm::MDNode * strict_fp_math_md
std::string mangle_llvm_type(llvm::Type *type)
Return type string for LLVM type using LLVM IR intrinsic type mangling.
llvm::MDNode * very_likely_branch
A fragment of Halide syntax.
llvm::StructType * argument_t_type
void codegen_asserts(const std::vector< const AssertStmt * > &asserts)
Codegen a block of asserts with pure conditions.
Represent a multi-dimensional region of a Func or an ImageParam that needs to be prefetched.
Is the first expression greater than the second.
Lock all the Store nodes in the body statement.
Construct a new vector by taking elements from another sequence of vectors.
virtual std::string mcpu_tune() const =0
virtual llvm::Value * concat_vectors(const std::vector< llvm::Value * > &)
Concatenate a bunch of llvm vectors.
Type indicating mask to use is all true – all lanes enabled.
void add_tbaa_metadata(llvm::Instruction *inst, std::string buffer, const Expr &index)
Mark a load or store with type-based-alias-analysis metadata so that llvm knows it can reorder loads ...
Unsigned integer constants.
virtual void init_module()
Initialize the CodeGen_LLVM internal state to compile a fresh module.
The result of modulus_remainder analysis.
The difference of two expressions.
llvm::MDNode * default_fp_math_md
virtual bool use_soft_float_abi() const =0
llvm::StructType * type_t_type
A struct representing a target machine and os to generate code for.
Logical and - are both expressions true.
std::map< std::string, std::vector< Intrinsic > > intrinsics
Mapping of intrinsic functions to the various overloads implementing it.
virtual llvm::Type * llvm_type_of(const Type &) const
Get the llvm type equivalent to the given halide type in the current context.
llvm::StructType * device_interface_t_type
llvm::Type * get_vector_type(llvm::Type *, int n, VectorTypeConstraint type_constraint=VectorTypeConstraint::None) const
VPArg(llvm::Value *value, std::optional< size_t > mangle_index=std::nullopt, int32_t alignment=0)
std::vector< Type > arg_types
Logical not - true if the expression false.
virtual std::unique_ptr< llvm::Module > compile(const Module &module)
Takes a halide Module and compiles it to an llvm Module.
Is the first expression less than the second.
size_t get_requested_alloca_total() const
bool use_llvm_vp_intrinsics
Controls use of vector predicated intrinsics for vector operations.
The product of two expressions.
A sequence of statements to be executed in-order.