amd64_transform.c

/*
 * This file is part of libFirm.
 * Copyright (C) 2012 University of Karlsruhe.
 */

/**
 * @file
 * @brief   code selection (transform FIRM into amd64 FIRM)
 */
#include "debug.h"
#include "panic.h"
#include "heights.h"
#include "ircons.h"
#include "irgmod.h"
#include "irgraph_t.h"
#include "irmode_t.h"
#include "irnode_t.h"
#include "iropt_t.h"
#include "tv_t.h"
#include "util.h"

#include "benode.h"
#include "betranshlp.h"
#include "beutil.h"
#include "bearch_amd64_t.h"
#include "beirg.h"
#include "beabihelper.h"
#include "besched.h"

#include "amd64_cconv.h"
#include "amd64_common_transform.h"
#include "amd64_new_nodes.h"
#include "amd64_nodes_attr.h"
#include "amd64_transform.h"
#include "../ia32/x86_address_mode.h"

#include "gen_amd64_regalloc_if.h"

DEBUG_ONLY(static firm_dbg_module_t *dbg = NULL;)

static ir_mode         *mode_gp;
static ir_mode         *mode_flags;
static amd64_cconv_t   *current_cconv = NULL;
static be_start_info_t  start_mem;
static be_start_info_t  start_val[N_AMD64_REGISTERS];
static size_t           start_callee_saves_offset;
static size_t           start_params_offset;
static pmap            *node_to_stack;
static be_stackorder_t *stackorder;

static const arch_register_req_t amd64_requirement_gp = {
	.cls             = &amd64_reg_classes[CLASS_amd64_gp],
	.limited         = NULL,
	.type            = arch_register_req_type_normal,
	.other_same      = 0,
	.other_different = 0,
	.width           = 1,
};

static const arch_register_req_t amd64_requirement_flags = {
	.cls             = &amd64_reg_classes[CLASS_amd64_flags],
	.limited         = NULL,
	.type            = arch_register_req_type_normal,
	.other_same      = 0,
	.other_different = 0,
	.width           = 1,
};

static const arch_register_req_t amd64_requirement_xmm = {
	.cls             = &amd64_reg_classes[CLASS_amd64_xmm],
	.limited         = NULL,
	.type            = arch_register_req_type_normal,
	.other_same      = 0,
	.other_different = 0,
	.width           = 1,
};

#define BIT(x)    (1u << x)

static const arch_register_req_t amd64_requirement_gp_same_0 = {
	.cls             = &amd64_reg_classes[CLASS_amd64_gp],
	.limited         = NULL,
	.type            = arch_register_req_type_normal
	                   | arch_register_req_type_should_be_same,
	.other_same      = BIT(0),
	.other_different = 0,
	.width           = 1,
};

static const arch_register_req_t amd64_requirement_xmm_same_0 = {
	.cls             = &amd64_reg_classes[CLASS_amd64_xmm],
	.limited         = NULL,
	.type            = arch_register_req_type_normal
	                   | arch_register_req_type_should_be_same,
	.other_same      = BIT(0),
	.other_different = 0,
	.width           = 1,
};

static const arch_register_req_t amd64_requirement_gp_same_0_not_1 = {
	.cls             = &amd64_reg_classes[CLASS_amd64_gp],
	.limited         = NULL,
	.type            = arch_register_req_type_normal
	                   | arch_register_req_type_should_be_same
	                   | arch_register_req_type_must_be_different,
	.other_same      = BIT(0),
	.other_different = BIT(1),
	.width           = 1,
};

static const unsigned amd64_limited_gp_rcx [] = { BIT(REG_GP_RCX) };
static const arch_register_req_t amd64_requirement_rcx = {
	.cls             = &amd64_reg_classes[CLASS_amd64_gp],
	.limited         = amd64_limited_gp_rcx,
	.type            = arch_register_req_type_limited,
	.other_same      = 0,
	.other_different = 0,
	.width           = 1,
};

static const unsigned amd64_limited_gp_rax [] = { BIT(REG_GP_RAX) };
static const arch_register_req_t amd64_requirement_rax = {
	.cls             = &amd64_reg_classes[CLASS_amd64_gp],
	.limited         = amd64_limited_gp_rax,
	.type            = arch_register_req_type_limited,
	.other_same      = 0,
	.other_different = 0,
	.width           = 1,
};

static const unsigned amd64_limited_gp_rdx [] = { BIT(REG_GP_RDX) };
static const arch_register_req_t amd64_requirement_rdx = {
	.cls             = &amd64_reg_classes[CLASS_amd64_gp],
	.limited         = amd64_limited_gp_rdx,
	.type            = arch_register_req_type_limited,
	.other_same      = 0,
	.other_different = 0,
	.width           = 1,
};

static const arch_register_req_t *mem_reqs[] = {
	&arch_no_requirement,
};

static const arch_register_req_t *reg_mem_reqs[] = {
	&amd64_requirement_gp,
	&arch_no_requirement,
};

static const arch_register_req_t *xmm_mem_reqs[] = {
	&amd64_requirement_xmm,
	&arch_no_requirement,
};

static const arch_register_req_t *reg_reg_mem_reqs[] = {
	&amd64_requirement_gp,
	&amd64_requirement_gp,
	&arch_no_requirement,
};

static const arch_register_req_t *xmm_reg_mem_reqs[] = {
	&amd64_requirement_xmm,
	&amd64_requirement_gp,
	&arch_no_requirement,
};

static const arch_register_req_t *reg_reg_reg_mem_reqs[] = {
	&amd64_requirement_gp,
	&amd64_requirement_gp,
	&amd64_requirement_gp,
	&arch_no_requirement,
};

static const arch_register_req_t *xmm_reg_reg_mem_reqs[] = {
	&amd64_requirement_xmm,
	&amd64_requirement_gp,
	&amd64_requirement_gp,
	&arch_no_requirement,
};

static const arch_register_req_t *reg_flags_reqs[] = {
	&amd64_requirement_gp,
	&amd64_requirement_flags,
};

static const arch_register_req_t *reg_reg_reqs[] = {
	&amd64_requirement_gp,
	&amd64_requirement_gp,
};

static const arch_register_req_t *rax_reg_reqs[] = {
	&amd64_requirement_rax,
	&amd64_requirement_gp,
};

static const arch_register_req_t *rax_reg_rdx_mem_reqs[] = {
	&amd64_requirement_rax,
	&amd64_requirement_gp,
	&amd64_requirement_rdx,
	&arch_no_requirement,
};

static const arch_register_req_t *reg_reqs[] = {
	&amd64_requirement_gp,
};

static const arch_register_req_t *xmm_reqs[] = {
	&amd64_requirement_xmm,
};

static const arch_register_req_t *reg_rcx_reqs[] = {
	&amd64_requirement_gp,
	&amd64_requirement_rcx,
};

static const arch_register_req_t *no_reqs[] = {
};

static const arch_register_req_t *xmm_xmm_reqs[] = {
	&amd64_requirement_xmm,
	&amd64_requirement_xmm,
};

static inline bool mode_needs_gp_reg(ir_mode *mode)
{
	return get_mode_arithmetic(mode) == irma_twos_complement;
}

static bool is_downconv(const ir_node *node)
{
	if (!is_Conv(node))
		return false;

	ir_mode *dest_mode = get_irn_mode(node);
	if (!mode_needs_gp_reg(dest_mode))
		return false;
	ir_mode *src_mode = get_irn_mode(get_Conv_op(node));
	if (!mode_needs_gp_reg(src_mode))
		return false;
	return get_mode_size_bits(dest_mode) <= get_mode_size_bits(src_mode);
}

static ir_node *skip_downconv(ir_node *node)
{
	while (is_downconv(node)) {
		if (get_irn_n_edges(node) > 1)
			break;
		node = get_Conv_op(node);
	}
	return node;
}

static bool is_sameconv(const ir_node *node)
{
	if (!is_Conv(node))
		return false;
	ir_mode *dest_mode = get_irn_mode(node);
	if (!mode_needs_gp_reg(dest_mode))
		return false;
	ir_mode *src_mode = get_irn_mode(get_Conv_op(node));
	if (!mode_needs_gp_reg(src_mode))
		return false;
	return get_mode_size_bits(dest_mode) == get_mode_size_bits(src_mode);
}

static ir_node *skip_sameconv(ir_node *node)
{
	while (is_sameconv(node)) {
		if (get_irn_n_edges(node) > 1)
			break;
		node = get_Conv_op(node);
	}
	return node;
}

static ir_node *get_initial_sp(ir_graph *irg)
{
	return be_get_start_proj(irg, &start_val[REG_RSP]);
}

static ir_node *get_initial_fp(ir_graph *irg)
{
	return be_get_start_proj(irg, &start_val[REG_RBP]);
}

static ir_node *get_initial_mem(ir_graph *irg)
{
	return be_get_start_proj(irg, &start_mem);
}

static ir_node *get_frame_base(ir_graph *irg)
{
	if (current_cconv->omit_fp) {
		return get_initial_sp(irg);
	} else {
		return get_initial_fp(irg);
	}
}

static ir_node *gen_Const(ir_node *node)
{
	ir_node  *block = be_transform_node(get_nodes_block(node));
	dbg_info *dbgi  = get_irn_dbg_info(node);
	ir_mode  *mode  = get_irn_mode(node);
	ir_tarval *tv = get_Const_tarval(node);

	if (!mode_needs_gp_reg(mode)) {
		if (tarval_is_null(tv)) {
			return new_bd_amd64_xXorp0(dbgi, block);
		}

		return create_float_const(dbgi, block, tv);
	}

	uint64_t val = get_tarval_uint64(tv);
	amd64_insn_mode_t imode = val > UINT32_MAX ? INSN_MODE_64 : INSN_MODE_32;
	return new_bd_amd64_MovImm(dbgi, block, imode, val, NULL);
}

typedef enum reference_mode_t {
	REFERENCE_DIRECT,
	REFERENCE_IP_RELATIVE,
	REFERENCE_GOT,
} reference_mode_t;

static reference_mode_t need_relative_addressing(const ir_entity *entity)
{
	if (!be_options.pic)
		return REFERENCE_DIRECT;

	/* simply everything is instruction pointer relative, external functions
	 * use a global offset table */
	return entity_has_definition(entity)
	   && (get_entity_linkage(entity) & IR_LINKAGE_MERGE) == 0
	    ? REFERENCE_IP_RELATIVE : REFERENCE_GOT;
}

static ir_node *gen_Address(ir_node *node)
{
	ir_node   *block  = be_transform_node(get_nodes_block(node));
	dbg_info  *dbgi   = get_irn_dbg_info(node);
	ir_entity *entity = get_Address_entity(node);

	/* do we need RIP-relative addressing because of PIC? */
	reference_mode_t mode = need_relative_addressing(entity);
	if (mode == REFERENCE_DIRECT)
		return new_bd_amd64_MovImm(dbgi, block, INSN_MODE_64, 0, entity);

	amd64_addr_t addr;
	memset(&addr, 0, sizeof(addr));
	addr.base_input  = NO_INPUT;
	addr.index_input = NO_INPUT;
	addr.mem_input   = NO_INPUT;

	if (mode == REFERENCE_IP_RELATIVE) {
		addr.immediate.entity = entity;
		return new_bd_amd64_Lea(dbgi, block, 0, NULL, INSN_MODE_64, addr);
	} else {
		assert(mode == REFERENCE_GOT);
		addr.immediate.entity = new_got_entry_entity(entity);
		ir_node *load = new_bd_amd64_Mov(dbgi, block, 0, NULL, INSN_MODE_64,
										 AMD64_OP_ADDR, addr);
		return new_r_Proj(load, mode_gp, pn_amd64_Mov_res);
	}
}

ir_node *amd64_new_IncSP(ir_node *block, ir_node *old_sp, int offset,
                         unsigned align)
{
	ir_node *incsp = be_new_IncSP(&amd64_registers[REG_RSP], block, old_sp,
	                              offset, align);
	arch_add_irn_flags(incsp, arch_irn_flag_modify_flags);
	return incsp;
}

typedef ir_node *(*construct_binop_func)(dbg_info *dbgi, ir_node *block,
	int arity, ir_node *in[], const amd64_binop_addr_attr_t *attr_init);

typedef ir_node *(*construct_rax_binop_func)(dbg_info *dbgi, ir_node *block,
	int arity, ir_node *in[], amd64_insn_mode_t insn_mode,
	amd64_op_mode_t op_mode, amd64_addr_t addr);

typedef enum match_flags_t {
	match_am           = 1 << 0,
	match_mode_neutral = 1 << 1,
	match_immediate    = 1 << 2,
	match_commutative  = 1 << 3,
} match_flags_t;

typedef struct amd64_args_t {
	amd64_binop_addr_attr_t     attr;
	ir_node                    *mem_proj;
	ir_node                    *in[4];
	int                         arity;
	const arch_register_req_t **reqs;
} amd64_args_t;

static bool match_immediate_32(amd64_imm32_t *imm, const ir_node *op,
                               bool can_match_ip_relative,
                               bool upper32_dont_care)
{
	assert(mode_needs_gp_reg(get_irn_mode(op)));
	assert(imm->offset == 0 && imm->entity == NULL);
	if (is_Const(op)) {
		ir_tarval *tv = get_Const_tarval(op);
		if (!tarval_is_long(tv))
			return false;
		long    lval = get_tarval_long(tv);
		int32_t val  = (int32_t)lval;
		if ((long)val != lval)
			return false;
		/** the immediate value is signed extended to 64bit, sometimes
		 * this is not what we want. */
		if (!upper32_dont_care && val < 0
		    && !mode_is_signed(get_tarval_mode(tv)))
		    return false;
		imm->offset = val;
		return true;
	} else if (can_match_ip_relative && is_Address(op)) {
		/* TODO: check if entity is in lower 4GB address space/relative */
		ir_entity *entity = get_Address_entity(op);
		imm->entity = entity;
		return true;
	}
	/* TODO: SymConst, Add(SymConst, Const) ... */
	return false;
}

static ir_heights_t *heights;

static bool input_depends_on_load(ir_node *load, ir_node *input)
{
	ir_node *block = get_nodes_block(load);
	/* if the dependency is in another block, then we ignore it for now
	   as we only match address mode loads in the same block. */
	return get_nodes_block(input) == block
	    && heights_reachable_in_block(heights, input, load);
}

static void fix_node_mem_proj(ir_node *node, ir_node *mem_proj)
{
	if (mem_proj == NULL)
		return;

	ir_node *load = get_Proj_pred(mem_proj);
	be_set_transformed_node(load, node);
}

static ir_node *source_am_possible(ir_node *block, ir_node *node)
{
	if (!is_Proj(node))
		return NULL;
	ir_node *load = get_Proj_pred(node);
	if (!is_Load(load))
		return NULL;
	assert(get_Proj_num(node) == pn_Load_res);
	if (get_nodes_block(load) != block)
		return NULL;
	/* make sure we are the only user */
	if (get_irn_n_edges(node) != 1)
		return NULL;
	/* ia32 backend claims this can happen, use an assert for now and see
	 * if we hit it :) */
	assert(!be_is_transformed(node));
	return load;
}

static bool needs_extension(ir_node *op)
{
	ir_mode *mode = get_irn_mode(op);
	if (get_mode_size_bits(mode) >= 32)
		return false;
	return !be_upper_bits_clean(op, mode);
}

static ir_node *create_sext(ir_node *block, ir_node *const node, ir_mode *mode)
{
	amd64_insn_mode_t insn_mode = get_insn_mode_from_mode(mode);
	dbg_info *const   dbgi      = get_irn_dbg_info(node);
	ir_node  *const   new_node  = be_transform_node(node);
	ir_node  *const   new_block = be_transform_node(block);

	amd64_shift_attr_t attr;
	memset(&attr, 0, sizeof(attr));
	attr.base.op_mode = AMD64_OP_SHIFT_IMM;
	attr.insn_mode    = insn_mode;
	attr.immediate    = get_mode_size_bits(mode) - 1;
	ir_node *in[1]    = { new_node };
	ir_node *sext     = new_bd_amd64_Sar(dbgi, new_block, ARRAY_SIZE(in),
	                                     in, &attr);

	arch_set_irn_register_reqs_in(sext, reg_reqs);
	arch_set_irn_register_req_out(sext, 0, &amd64_requirement_gp_same_0);

	return sext;
}

static ir_node *create_zext(ir_node *block, ir_node *const node)
{
	dbg_info *const dbgi      = get_irn_dbg_info(node);
	ir_node  *const new_block = be_transform_node(block);
	ir_node  *const zero      = new_bd_amd64_Xor0(dbgi, new_block);

	arch_set_irn_register_reqs_in(zero, reg_reqs);
	return zero;
}

static bool use_address_matching(ir_mode *mode, match_flags_t flags,
                                 ir_node *block,
                                 ir_node *op1, ir_node *op2,
                                 ir_node **out_load, ir_node **out_op)
{
	if (! (flags & match_am))
		return false;

	unsigned mode_bits = get_mode_size_bits(mode);
	if (mode_bits == 8 || mode_bits == 16)
		return false;

	ir_node *load2 = source_am_possible(block, op2);
	if (load2 != NULL && !input_depends_on_load(load2, op1)) {
		(*out_load) = load2;
		(*out_op)   = op1;
		return true;
	}

	if (flags & match_commutative) {
		ir_node *load1 = source_am_possible(block, op1);
		if (load1 != NULL && !input_depends_on_load(load1, op2)) {
			(*out_load) = load1;
			(*out_op)   = op2;
			return true;
		}
	}
	return false;
}

static void perform_address_matching(ir_node *ptr, int *arity,
                                     ir_node **in, amd64_addr_t *addr)
{
	x86_address_t maddr;
	memset(&maddr, 0, sizeof(maddr));
	x86_create_address_mode(&maddr, ptr, x86_create_am_normal);

	if (maddr.base != NULL) {
		int base_input   = (*arity)++;
		addr->base_input = base_input;
		in[base_input]   = be_transform_node(maddr.base);
	} else {
		addr->base_input = NO_INPUT;
	}
	if (maddr.index != NULL) {
		int index_input = (*arity)++;
		addr->index_input = index_input;
		in[index_input]  = be_transform_node(maddr.index);
	} else {
		addr->index_input = NO_INPUT;
	}
	if (maddr.frame_entity != NULL) {
		assert(maddr.entity == NULL);
		addr->immediate.entity = maddr.frame_entity;
		/* not supported yet */
		assert(!is_parameter_entity(maddr.frame_entity)
		       || get_entity_parameter_number(maddr.frame_entity)
		          != IR_VA_START_PARAMETER_NUMBER);
	} else {
		addr->immediate.entity = maddr.entity;
	}
	addr->immediate.offset = maddr.offset;
	addr->log_scale        = maddr.scale;
}

static void match_binop(amd64_args_t *args, ir_node *block,
                        ir_mode *mode, ir_node *op1, ir_node *op2,
                        match_flags_t flags)
{
	memset(args, 0, sizeof(*args));

	bool use_am;
	bool use_xmm       = mode_is_float(mode);
	bool use_immediate = flags & match_immediate;
	bool mode_neutral  = flags & match_mode_neutral;

	args->attr.base.insn_mode = get_insn_mode_from_mode(mode);

	/* TODO: legalize phase */
	if (mode_neutral) {
		op1 = skip_downconv(op1);
		op2 = skip_downconv(op2);
	} else {
		/* TODO: extend inputs? */
		(void)needs_extension;
	}

	ir_node *load;
	ir_node *op;

	use_am = use_address_matching(mode, flags, block, op1, op2, &load, &op);

	if (use_immediate
	    && match_immediate_32(&args->attr.u.immediate, op2, false, mode_neutral)) {
		assert(!use_xmm && "Can't (yet) match binop with xmm immediate");
		/* fine, we found an immediate */
		args->attr.base.base.op_mode = AMD64_OP_REG_IMM;
		args->in[args->arity++]      = be_transform_node(op1);
		args->reqs                   = reg_reqs;
	} else if (use_am) {
		ir_node *new_op        = be_transform_node(op);
		int      reg_input     = args->arity++;
		args->attr.u.reg_input = reg_input;
		args->in[reg_input]    = new_op;
		amd64_addr_t *addr     = &args->attr.base.addr;

		ir_node *ptr = get_Load_ptr(load);
		perform_address_matching(ptr, &(args->arity), args->in, addr);

		args->reqs = use_xmm ? xmm_mem_reqs : reg_mem_reqs;
		if (addr->base_input != NO_INPUT && addr->index_input != NO_INPUT) {
			args->reqs = use_xmm ? xmm_reg_reg_mem_reqs
			             : reg_reg_reg_mem_reqs;
		} else if (addr->base_input != NO_INPUT || addr->index_input != NO_INPUT) {
			args->reqs = use_xmm ? xmm_reg_mem_reqs
			             : reg_reg_mem_reqs;
		}
		ir_node *new_mem    = be_transform_node(get_Load_mem(load));
		int mem_input       = args->arity++;
		args->in[mem_input] = new_mem;
		addr->mem_input     = mem_input;

		args->mem_proj      = be_get_Proj_for_pn(load, pn_Load_M);
		args->attr.base.base.op_mode = AMD64_OP_ADDR_REG;
	} else {
		/* simply transform the arguments */
		args->in[args->arity++] = be_transform_node(op1);
		args->in[args->arity++] = be_transform_node(op2);
		args->attr.base.base.op_mode = AMD64_OP_REG_REG;

		args->reqs = use_xmm ? xmm_xmm_reqs : reg_reg_reqs;
	}
}

static ir_node *gen_binop_am(ir_node *node, ir_node *op1, ir_node *op2,
                             construct_binop_func func, match_flags_t flags)
{
	ir_node *block = get_nodes_block(node);
	ir_mode *mode  = get_irn_mode(node);
	amd64_args_t args;
	match_binop(&args, block, mode, op1, op2, flags);

	dbg_info *const dbgi      = get_irn_dbg_info(node);
	ir_node  *const new_block = be_transform_node(block);

	ir_node *new_node = func(dbgi, new_block, args.arity, args.in, &args.attr);
	arch_set_irn_register_reqs_in(new_node, args.reqs);

	fix_node_mem_proj(new_node, args.mem_proj);

	if (mode_is_float(mode)) {
		arch_set_irn_register_req_out(new_node, 0,
		                              &amd64_requirement_xmm_same_0);
		return new_r_Proj(new_node, mode_D, pn_amd64_xSubs_res);
	} else {
		arch_set_irn_register_req_out(new_node, 0,
		                              &amd64_requirement_gp_same_0);
		return new_r_Proj(new_node, mode_gp, pn_amd64_Sub_res);
	}
}

static ir_node *gen_binop_rax(ir_node *node, ir_node *op1, ir_node *op2,
                              construct_rax_binop_func make_node,
                              match_flags_t flags)
{
	bool use_am;
	bool mode_neutral  = flags & match_mode_neutral;
	assert(! (flags & match_immediate));

	ir_mode *mode = get_irn_mode(op1);
	amd64_insn_mode_t insn_mode = get_insn_mode_from_mode(mode);

	/* TODO: legalize phase */
	if (mode_neutral) {
		op1 = skip_downconv(op1);
		op2 = skip_downconv(op2);
	} else {
		/* TODO: extend inputs? */
		(void)needs_extension;
	}

	ir_node *load;
	ir_node *op;
	ir_node *block = get_nodes_block(node);
	ir_node *in[4];
	int      arity = 0;
	const arch_register_req_t **reqs;
	amd64_op_mode_t op_mode;
	amd64_addr_t    addr;
	memset(&addr, 0, sizeof(addr));

	use_am = use_address_matching(mode, flags, block, op1, op2, &load, &op);

	ir_node *mem_proj = NULL;
	if (use_am) {
		ir_node *new_op    = be_transform_node(op);
		int      reg_input = arity++;
		in[reg_input]      = new_op;

		ir_node *ptr = get_Load_ptr(load);
		perform_address_matching(ptr, &arity, in, &addr);

		reqs = reg_mem_reqs;
		if (addr.base_input != NO_INPUT && addr.index_input != NO_INPUT) {
			reqs = reg_reg_reg_mem_reqs;
		} else if (addr.base_input != NO_INPUT || addr.index_input != NO_INPUT) {
			reqs = reg_reg_mem_reqs;
		}

		ir_node *new_mem = be_transform_node(get_Load_mem(load));
		int mem_input    = arity++;
		in[mem_input]    = new_mem;
		addr.mem_input   = mem_input;

		mem_proj                = be_get_Proj_for_pn(load, pn_Load_M);
		op_mode                 = AMD64_OP_RAX_ADDR;
	} else {
		/* simply transform the arguments */
		in[arity++] = be_transform_node(op1);
		in[arity++] = be_transform_node(op2);
		reqs        = rax_reg_reqs;
		op_mode     = AMD64_OP_RAX_REG;
	}

	assert((size_t)arity <= ARRAY_SIZE(in));
	dbg_info *dbgi      = get_irn_dbg_info(node);
	ir_node  *new_block = be_transform_node(block);
	ir_node  *new_node  = make_node(dbgi, new_block, arity, in, insn_mode,
	                                op_mode, addr);
	arch_set_irn_register_reqs_in(new_node, reqs);
	if (mem_proj != NULL) {
		be_set_transformed_node(load, new_node);
	}
	return new_node;
}

typedef ir_node *(*construct_shift_func)(dbg_info *dbgi, ir_node *block,
	int arity, ir_node *in[], const amd64_shift_attr_t *attr_init);

static ir_node *gen_shift_binop(ir_node *node, ir_node *op1, ir_node *op2,
                                construct_shift_func func, match_flags_t flags)
{
	ir_mode *mode = get_irn_mode(node);
	assert(!mode_is_float(mode));

	if (get_mode_modulo_shift(mode) != 32 && get_mode_size_bits(mode) != 64)
		panic("insupported modulo shift used");

	ir_node *in[3];
	int      arity = 0;
	if (flags & match_mode_neutral) {
		op1 = skip_downconv(op1);
		in[arity++] = be_transform_node(op1);
		mode = get_mode_size_bits(mode) > 32 ? mode_gp : mode_Iu;
	} else {
		op1 = skip_sameconv(op1);

		/* Use 8/16bit operations instead of doing zext/upconv */
		in[arity++] = be_transform_node(op1);
	}

	/* we can skip any convs for the shift count, as it only uses the lowest
	 * 5/6 bits anyway */
	while (is_Conv(op2) && get_irn_n_edges(op2) == 1) {
		ir_node *const op = get_Conv_op(op2);
		if (get_mode_arithmetic(get_irn_mode(op)) != irma_twos_complement)
			break;
		op2 = op;
	}

	amd64_shift_attr_t attr;
	memset(&attr, 0, sizeof(attr));
	const arch_register_req_t **reqs;
	const arch_register_req_t  *out_req0;
	if (is_Const(op2)) {
		attr.base.op_mode = AMD64_OP_SHIFT_IMM;
		reqs              = reg_reqs;
		out_req0          = &amd64_requirement_gp_same_0;
		attr.immediate    = get_tarval_long(get_Const_tarval(op2));
	} else {
		attr.base.op_mode = AMD64_OP_SHIFT_REG;
		in[arity++]       = be_transform_node(op2);
		reqs              = reg_rcx_reqs;
		out_req0          = &amd64_requirement_gp_same_0_not_1;
	}
	attr.insn_mode = get_insn_mode_from_mode(mode);

	ir_node  *const block     = get_nodes_block(node);
	dbg_info *const dbgi      = get_irn_dbg_info(node);
	ir_node  *const new_block = be_transform_node(block);
	ir_node  *const new_node  = func(dbgi, new_block, arity, in, &attr);
	arch_set_irn_register_reqs_in(new_node, reqs);
	arch_set_irn_register_req_out(new_node, 0, out_req0);
	return new_node;
}

static ir_node *create_Lea_as_Add(ir_node *node, ir_node *op1, ir_node *op2)
{
	dbg_info *const dbgi = get_irn_dbg_info(node);
	ir_node  *block      = get_nodes_block(node);
	ir_node  *new_block  = be_transform_node(block);
	ir_mode *mode        = get_irn_mode(node);

	amd64_insn_mode_t insn_mode;
	if (get_mode_size_bits(mode) <= 32)
		insn_mode = INSN_MODE_32;
	else
		insn_mode = INSN_MODE_64;

	const arch_register_req_t **reqs;
	amd64_addr_t addr;
	memset(&addr, 0, sizeof(addr));

	ir_node *in[2];
	int arity = 0;

	if (match_immediate_32(&addr.immediate, op2, false, true)) {
		in[arity++]      = be_transform_node(op1);
		reqs             = reg_reqs;
		addr.index_input = NO_INPUT;
	} else {
		in[arity++]      = be_transform_node(op1);
		in[arity++]      = be_transform_node(op2);
		addr.index_input = 0;
		addr.base_input  = 1;
		reqs             = reg_reg_reqs;
	}

	ir_node *res = new_bd_amd64_Lea(dbgi, new_block, arity, in, insn_mode, addr);
	arch_set_irn_register_reqs_in(res, reqs);
	return res;
}

static ir_node *gen_Add(ir_node *const node)
{
	match_flags_t flags = match_immediate | match_am | match_mode_neutral
	                      | match_commutative;

	ir_node *op1 = get_Add_left(node);
	ir_node *op2 = get_Add_right(node);

	ir_mode *mode  = get_irn_mode(node);
	ir_node *block = get_nodes_block(node);
	ir_node *load, *op;

	if (mode_is_float(mode)) {
		return gen_binop_am(node, op1, op2, new_bd_amd64_xAdds,
		                    match_commutative | match_am);
	}

	bool use_am = use_address_matching(mode, flags, block, op1, op2, &load, &op);

	ir_node *res;
	if (use_am)
		res = gen_binop_am(node, op1, op2, new_bd_amd64_Add, flags);
	else
		res = create_Lea_as_Add(node, op1, op2);

	x86_mark_non_am(node);
	return res;
}

static ir_node *gen_Sub(ir_node *const node)
{
	ir_node  *const op1     = get_Sub_left(node);
	ir_node  *const op2     = get_Sub_right(node);
	ir_mode  *const mode    = get_irn_mode(node);

	if (mode_is_float(mode)) {
		return gen_binop_am(node, op1, op2, new_bd_amd64_xSubs, match_am);
	} else {
		/* do not match AM yet until we have a sub->neg+add rule
		 * in amd64_finish */
		return gen_binop_am(node, op1, op2, new_bd_amd64_Sub,
		                    /* match_am | */ match_immediate);
	}
}

static ir_node *gen_And(ir_node *const node)
{
	ir_node *op1 = get_And_left(node);
	ir_node *op2 = get_And_right(node);
	return gen_binop_am(node, op1, op2, new_bd_amd64_And,
	                    match_immediate | match_am | match_mode_neutral
	                    | match_commutative);
}

static ir_node *gen_Eor(ir_node *const node)
{
	ir_node *op1 = get_Eor_left(node);
	ir_node *op2 = get_Eor_right(node);
	return gen_binop_am(node, op1, op2, new_bd_amd64_Xor,
	                    match_immediate | match_am | match_mode_neutral
	                    | match_commutative);
}

static ir_node *gen_Or(ir_node *const node)
{
	ir_node *op1 = get_Or_left(node);
	ir_node *op2 = get_Or_right(node);
	return gen_binop_am(node, op1, op2, new_bd_amd64_Or,
	                    match_immediate | match_am | match_mode_neutral
	                    | match_commutative);
}

static ir_node *gen_Mul(ir_node *const node)
{
	ir_node *op1  = get_Mul_left(node);
	ir_node *op2  = get_Mul_right(node);
	ir_mode *mode = get_irn_mode(node);

	if (get_mode_size_bits(mode) < 16) {
		/* imulb only supports rax - reg form */
		ir_node *new_node =
		            gen_binop_rax(node, op1, op2, new_bd_amd64_IMul1Op,
		                          match_mode_neutral
		                          | match_commutative);
		return new_r_Proj(new_node, mode_gp, pn_amd64_IMul1Op_res_low);
	} else if (mode_is_float(mode)) {
		return gen_binop_am(node, op1, op2, new_bd_amd64_xMuls,
		                    match_commutative | match_am);
	} else {
		return gen_binop_am(node, op1, op2, new_bd_amd64_IMul,
		                    match_immediate | match_am
		                    | match_mode_neutral | match_commutative);
	}
}

static ir_node *gen_Mulh(ir_node *const node)
{
	ir_node *op1  = get_Mulh_left(node);
	ir_node *op2  = get_Mulh_right(node);
	ir_mode *mode = get_irn_mode(op1);
	ir_node *new_node;

	if (mode_is_signed(mode)) {
		new_node = gen_binop_rax(node, op1, op2, new_bd_amd64_IMul1Op,
                        /* match_am TODO */
                        match_mode_neutral | match_commutative);
		return new_r_Proj(new_node, mode_gp, pn_amd64_IMul1Op_res_high);
	} else {
		new_node = gen_binop_rax(node, op1, op2, new_bd_amd64_Mul,
                         /* match_am TODO */
                         match_mode_neutral | match_commutative);
		return new_r_Proj(new_node, mode_gp, pn_amd64_Mul_res_high);
	}
}

static ir_node *gen_Shl(ir_node *const node)
{
	ir_node *op1 = get_Shl_left(node);
	ir_node *op2 = get_Shl_right(node);
	return gen_shift_binop(node, op1, op2, new_bd_amd64_Shl,
	                       match_immediate | match_mode_neutral);
}

static ir_node *gen_Shr(ir_node *const node)
{
	ir_node *op1 = get_Shr_left(node);
	ir_node *op2 = get_Shr_right(node);
	return gen_shift_binop(node, op1, op2, new_bd_amd64_Shr,
	                       match_immediate);
}

static ir_node *gen_Shrs(ir_node *const node)
{
	ir_node *op1 = get_Shrs_left(node);
	ir_node *op2 = get_Shrs_right(node);
	return gen_shift_binop(node, op1, op2, new_bd_amd64_Sar,
	                       match_immediate);
}

static ir_node *create_div(ir_node *const node, ir_mode *const mode,
                           ir_node *const op1, ir_node *const op2,
                           ir_node *const mem)
{
	dbg_info *const dbgi      = get_irn_dbg_info(node);
	ir_node  *const block     = get_nodes_block(node);
	ir_node  *const new_block = be_transform_node(block);
	amd64_insn_mode_t insn_mode
		= get_mode_size_bits(mode) > 32 ? INSN_MODE_64 : INSN_MODE_32;

	ir_node *new_op1 = be_transform_node(op1);
	ir_node *new_op2 = be_transform_node(op2);
	ir_node *new_mem = be_transform_node(mem);
	ir_node *upper_value;
	ir_node *(*constructor)(dbg_info*,ir_node*,int,ir_node**,amd64_insn_mode_t);
	if (mode_is_signed(mode)) {
		upper_value = create_sext(block, op1, mode);
		constructor = new_bd_amd64_IDiv;
	} else {
		upper_value = create_zext(block, node);
		constructor = new_bd_amd64_Div;
	}

	ir_node *in[] = { new_op1, new_op2, upper_value, new_mem };
	ir_node *res = constructor(dbgi, new_block, ARRAY_SIZE(in), in, insn_mode);
	arch_set_irn_register_reqs_in(res, rax_reg_rdx_mem_reqs);
	return res;
}

static ir_node *create_sse_div(ir_node *const node, ir_mode *const mode,
                               ir_node *const op1, ir_node *const op2)
{
	dbg_info *const dbgi      = get_irn_dbg_info(node);
	ir_node  *const block     = get_nodes_block(node);
	ir_node  *const new_block = be_transform_node(block);

	amd64_args_t args;
	match_binop(&args, block, mode, op1, op2, match_am);

	ir_node *const new_node = new_bd_amd64_xDivs(dbgi, new_block, args.arity,
	                                             args.in, &args.attr);
	arch_set_irn_register_reqs_in(new_node, args.reqs);

	fix_node_mem_proj(new_node, args.mem_proj);

	arch_set_irn_register_req_out(new_node, 0,
	                              &amd64_requirement_xmm_same_0);
	return new_node;
}

static ir_node *gen_Div(ir_node *const node)
{
	ir_mode *const mode = get_Div_resmode(node);
	ir_node *const op1  = get_Div_left(node);
	ir_node *const op2  = get_Div_right(node);
	ir_node *const mem  = get_Div_mem(node);

	if (mode_is_float(mode))
		return create_sse_div(node, mode, op1, op2);
	else
		return create_div(node, mode, op1, op2, mem);
}

static ir_node *gen_Proj_Div(ir_node *const node)
{
	ir_node *pred     = get_Proj_pred(node);
	ir_node *new_pred = be_transform_node(pred);
	unsigned pn       = get_Proj_num(node);

	assert((unsigned)pn_amd64_Div_M == (unsigned)pn_amd64_IDiv_M);
	assert((unsigned)pn_amd64_Div_res_div == (unsigned)pn_amd64_IDiv_res_div);
	assert((unsigned)pn_amd64_xDivs_M == (unsigned)pn_amd64_IDiv_M);
	assert((unsigned)pn_amd64_xDivs_res_div == (unsigned)pn_amd64_IDiv_res_div);

	ir_mode *mode;
	if (mode_is_float(get_Div_resmode(pred)))
		mode = mode_D;
	else
		mode = mode_gp;

	switch ((pn_Div)pn) {
	case pn_Div_M:
		return new_r_Proj(new_pred, mode_M, pn_amd64_Div_M);
	case pn_Div_res:
		return new_r_Proj(new_pred, mode, pn_amd64_Div_res_div);
	case pn_Div_X_except:
	case pn_Div_X_regular:
		panic("amd64 exception NIY");
	}
	panic("invalid Div Proj");
}

static ir_node *gen_Mod(ir_node *const node)
{
	ir_mode *const mode = get_Mod_resmode(node);
	ir_node *const op1  = get_Mod_left(node);
	ir_node *const op2  = get_Mod_right(node);
	ir_node *const mem  = get_Mod_mem(node);
	assert(mode_needs_gp_reg(mode));
	return create_div(node, mode, op1, op2, mem);
}

static ir_node *gen_Proj_Mod(ir_node *const node)
{
	ir_node *pred     = get_Proj_pred(node);
	ir_node *new_pred = be_transform_node(pred);
	unsigned pn       = get_Proj_num(node);

	assert((unsigned)pn_amd64_Div_M == (unsigned)pn_amd64_IDiv_M);
	assert((unsigned)pn_amd64_Div_res_mod == (unsigned)pn_amd64_IDiv_res_mod);
	switch ((pn_Mod)pn) {
	case pn_Mod_M:
		return new_r_Proj(new_pred, mode_M, pn_amd64_Div_M);
	case pn_Mod_res:
		return new_r_Proj(new_pred, mode_gp, pn_amd64_Div_res_mod);
	case pn_Mod_X_except:
	case pn_Mod_X_regular:
		panic("amd64 exception NIY");
	}
	panic("invalid Mod Proj");
}

typedef ir_node* (*unop_constructor)(dbg_info*,ir_node*block,ir_node*op,amd64_insn_mode_t insn_mode);

static ir_node *gen_unop(ir_node *const node, int op_pos, unop_constructor gen)
{
	dbg_info *const dbgi   = get_irn_dbg_info(node);
	ir_node  *const block  = be_transform_node(get_nodes_block(node));
	ir_node  *const op     = get_irn_n(node, op_pos);
	ir_node  *const new_op = be_transform_node(op);
	ir_mode  *const mode   = get_irn_mode(node);

	amd64_insn_mode_t insn_mode
		= get_mode_size_bits(mode) > 32 ? INSN_MODE_64 : INSN_MODE_32;
	return gen(dbgi, block, new_op, insn_mode);
}

/** Create a floating point negation by switching the sign bit using a Xor.
  */
static ir_node *gen_float_neg(ir_node *const node)
{
	dbg_info  *const dbgi = get_irn_dbg_info(node);
	ir_node   *block      = get_nodes_block(node);
	ir_node   *new_block  = be_transform_node(block);
	ir_node   *op         = get_irn_n(node, n_Minus_op);
	ir_node   *new_op     = be_transform_node(op);
	ir_mode   *mode       = get_irn_mode(node);
	ir_tarval *tv         = create_sign_tv(mode);
	ir_node   *load       = create_float_const(dbgi, new_block, tv);
	ir_node   *in[]       = { new_op, load };

	amd64_binop_addr_attr_t attr;
	memset(&attr, 0, sizeof(attr));
	attr.base.base.op_mode = AMD64_OP_REG_REG;

	ir_node *xor = new_bd_amd64_xXorp(dbgi, new_block, ARRAY_SIZE(in),
	                                  in, &attr);
	arch_set_irn_register_reqs_in(xor, xmm_xmm_reqs);
	arch_set_irn_register_req_out(xor, 0, &amd64_requirement_xmm_same_0);

	return new_r_Proj(xor, mode, pn_amd64_xXorp_res);
}

static ir_node *gen_Minus(ir_node *const node)
{
	ir_mode *mode = get_irn_mode(node);

	if (mode_is_float(mode)) {
		return gen_float_neg(node);
	} else {
		return gen_unop(node, n_Minus_op, &new_bd_amd64_Neg);
	}
}
static ir_node *gen_Not(ir_node *const node)
{
	return gen_unop(node, n_Not_op, &new_bd_amd64_Not);
}

static ir_node *gen_Member(ir_node *const node)
{
	ir_node   *block     = get_nodes_block(node);
	ir_node   *new_block = be_transform_node(block);
	dbg_info  *dbgi      = get_irn_dbg_info(node);
	ir_node   *ptr       = get_Member_ptr(node);
	ir_graph  *irg       = get_irn_irg(node);
	ir_node   *base      = get_frame_base(irg);
	ir_entity *entity    = get_Member_entity(node);
	if (!is_Proj(ptr) || !is_Start(get_Proj_pred(ptr)))