bearch_ia32.c 49.3 KB
Newer Older
Christian Würdig's avatar
Christian Würdig committed
1
2
/*
 * This file is part of libFirm.
3
 * Copyright (C) 2012 University of Karlsruhe.
Christian Würdig's avatar
Christian Würdig committed
4
5
 */

Christian Würdig's avatar
Christian Würdig committed
6
/**
Christian Würdig's avatar
Christian Würdig committed
7
8
9
 * @file
 * @brief       This is the main ia32 firm backend driver.
 * @author      Christian Wuerdig
Christian Würdig's avatar
Christian Würdig committed
10
 */
11
#include "be_t.h"
Matthias Braun's avatar
Matthias Braun committed
12
13
14
15
#include "bearch_ia32_t.h"
#include "beflags.h"
#include "begnuas.h"
#include "bemodule.h"
16
#include "bera.h"
Matthias Braun's avatar
Matthias Braun committed
17
18
19
20
#include "besched.h"
#include "bespillslots.h"
#include "bestack.h"
#include "beutil.h"
21
#include "bevarargs.h"
22
23
24
25
26
27
28
29
30
#include "gen_ia32_regalloc_if.h"
#include "ia32_architecture.h"
#include "ia32_emitter.h"
#include "ia32_finish.h"
#include "ia32_fpu.h"
#include "ia32_new_nodes.h"
#include "ia32_optimize.h"
#include "ia32_transform.h"
#include "ia32_x87.h"
31
#include "ident_t.h"
Matthias Braun's avatar
Matthias Braun committed
32
#include "instrument.h"
33
#include "ircons.h"
Christian Würdig's avatar
Christian Würdig committed
34
#include "irgmod.h"
Christian Würdig's avatar
Christian Würdig committed
35
#include "irgopt.h"
Matthias Braun's avatar
Matthias Braun committed
36
#include "irgwalk.h"
37
#include "iropt_t.h"
Matthias Braun's avatar
Matthias Braun committed
38
39
#include "irtools.h"
#include "lc_opts_enum.h"
40
#include "lower_alloc.h"
Matthias Braun's avatar
Matthias Braun committed
41
#include "lower_builtins.h"
42
#include "lower_calls.h"
43
#include "lower_mode_b.h"
44
#include "lower_softfloat.h"
45
#include "lowering.h"
Matthias Braun's avatar
Matthias Braun committed
46
#include "panic.h"
47

48
49
pmap *ia32_tv_ent; /**< A map of entities that store const tarvals */

50
ir_mode *ia32_mode_fpcw;
51
ir_mode *ia32_mode_flags;
52
53
ir_mode *ia32_mode_E;
ir_type *ia32_type_E;
54
55
56
ir_mode *ia32_mode_gp;
ir_mode *ia32_mode_float64;
ir_mode *ia32_mode_float32;
57

58
/** The current omit-fp state */
Matthias Braun's avatar
Matthias Braun committed
59
60
61
62
63
static ir_type   *omit_fp_between_type;
static ir_type   *between_type;
static ir_entity *old_bp_ent;
static ir_entity *ret_addr_ent;
static ir_entity *omit_fp_ret_addr_ent;
64
static bool       precise_x87_spills;
65
static bool       return_small_struct_in_regs;
66

67
typedef ir_node *(*create_const_node_func) (dbg_info *dbgi, ir_node *block);
68

69
/**
70
 * Used to create per-graph unique pseudo nodes.
71
 */
72
static inline ir_node *create_const(ir_graph *irg, ir_node **place,
73
                                    create_const_node_func func,
74
                                    const arch_register_t* reg)
75
{
76
	if (*place != NULL)
77
78
		return *place;

Matthias Braun's avatar
Matthias Braun committed
79
80
	ir_node *block = get_irg_start_block(irg);
	ir_node *res   = func(NULL, block);
81
	arch_set_irn_register(res, reg);
82
83
84
85
	*place = res;
	return res;
}

86
ir_node *ia32_new_NoReg_gp(ir_graph *irg)
87
{
88
89
	ia32_irg_data_t *irg_data = ia32_get_irg_data(irg);
	return create_const(irg, &irg_data->noreg_gp, new_bd_ia32_NoReg_GP,
90
	                    &ia32_registers[REG_GP_NOREG]);
91
92
}

93
ir_node *ia32_new_NoReg_fp(ir_graph *irg)
94
{
95
	ia32_irg_data_t *irg_data = ia32_get_irg_data(irg);
96
97
	return create_const(irg, &irg_data->noreg_fp, new_bd_ia32_NoReg_FP,
	                    &ia32_registers[REG_FP_NOREG]);
98
99
}

100
ir_node *ia32_new_NoReg_xmm(ir_graph *irg)
101
{
102
103
	ia32_irg_data_t *irg_data = ia32_get_irg_data(irg);
	return create_const(irg, &irg_data->noreg_xmm, new_bd_ia32_NoReg_XMM,
104
	                    &ia32_registers[REG_XMM_NOREG]);
105
106
}

107
ir_node *ia32_new_Fpu_truncate(ir_graph *irg)
108
{
109
110
	ia32_irg_data_t *irg_data = ia32_get_irg_data(irg);
	return create_const(irg, &irg_data->fpu_trunc_mode, new_bd_ia32_ChangeCW,
111
                        &ia32_registers[REG_FPCW]);
112
113
}

114
/**
115
 * Returns the admissible noreg register node for input register pos of node irn.
116
 */
117
static ir_node *ia32_get_admissible_noreg(ir_node *irn, int pos)
118
{
119
	ir_graph                  *irg = get_irn_irg(irn);
120
	const arch_register_req_t *req = arch_get_irn_register_req_in(irn, pos);
Matthias Braun's avatar
Matthias Braun committed
121
	if (req->cls == &ia32_reg_classes[CLASS_ia32_gp])
122
		return ia32_new_NoReg_gp(irg);
Matthias Braun's avatar
Matthias Braun committed
123

124
	if (ia32_cg_config.use_sse2) {
125
		return ia32_new_NoReg_xmm(irg);
126
	} else {
127
		return ia32_new_NoReg_fp(irg);
128
	}
129
130
}

131
static ir_entity *ia32_get_frame_entity(const ir_node *node)
132
{
133
134
135
136
137
138
139
140
141
	if (!is_ia32_irn(node))
		return NULL;
	ia32_attr_t const *const attr = get_ia32_attr_const(node);
	if (attr->am_imm.kind == X86_IMM_FRAMEOFFSET) {
		assert(get_ia32_frame_use(node) != IA32_FRAME_USE_NONE);
		return attr->am_imm.entity;
	}
	assert(get_ia32_frame_use(node) == IA32_FRAME_USE_NONE);
	return NULL;
142
143
}

144
145
static void ia32_set_frame_entity(ir_node *node, ir_entity *entity,
                                  const ir_type *type)
146
{
147
148
149
150
151
152
153
	ia32_attr_t *const attr = get_ia32_attr(node);
	attr->am_imm = (x86_imm32_t) {
		.kind   = X86_IMM_FRAMEOFFSET,
		.entity = entity,
		.offset = attr->am_imm.offset,
	};
	assert(get_ia32_frame_use(node) != IA32_FRAME_USE_NONE);
154

yb9976's avatar
yb9976 committed
155
	/* set ls_mode based on entity unless we explicitly requested
156
	 * a certain mode */
yb9976's avatar
yb9976 committed
157
158
	if (get_ia32_frame_use(node) != IA32_FRAME_USE_AUTO
	    || is_ia32_Cmp(node) || is_ia32_Conv_I2I(node))
159
160
161
162
163
164
165
166
		return;
	ir_mode *mode = get_type_mode(type);
	/** we 8bit stores have a special register requirement, so we can't simply
	 * change the ls_mode to 8bit here. The "hack" in
	 * ia32_collect_frame_entity_nodes() should take care that it never happens
	 */
	assert(!is_ia32_Store(node) || get_mode_size_bits(mode) > 8);
	set_ia32_ls_mode(node, mode);
167
168
}

169
static void ia32_set_frame_offset(ir_node *node, int bias)
170
{
171
172
	ia32_attr_t *const attr = get_ia32_attr(node);
	assert(attr->am_imm.kind == X86_IMM_FRAMEOFFSET);
173

174
175
176
177
178
179
	/* Pop nodes modify the stack pointer before calculating the
	 * destination address, fix this here */
	if (is_ia32_PopMem(node)) {
		ir_node *base = get_irn_n(node, n_ia32_PopMem_base);
		if (arch_get_irn_register(base) == &ia32_registers[REG_ESP]) {
			ir_mode *mode = get_ia32_ls_mode(node);
Matthias Braun's avatar
Matthias Braun committed
180
			bias -= get_mode_size_bytes(mode);
181
		}
182
	}
183
184
185
186
187
188
189
190

#ifndef NDEBUG
	attr->old_frame_ent = attr->am_imm.entity;
#endif
	/* This is just a simple 32bit value now */
	attr->am_imm.offset += bias;
	attr->am_imm.entity = NULL;
	attr->am_imm.kind   = X86_IMM_VALUE;
191
192
}

193
int ia32_get_sp_bias(const ir_node *node)
194
{
195
196
197
	if (is_ia32_Call(node))
		return -(int)get_ia32_call_attr_const(node)->pop;

Matthias Braun's avatar
Matthias Braun committed
198
199
200
201
	if (is_ia32_Push(node)) {
		ir_mode *ls_mode = get_ia32_ls_mode(node);
		return get_mode_size_bytes(ls_mode);
	}
202

Matthias Braun's avatar
Matthias Braun committed
203
204
205
206
	if (is_ia32_Pop(node) || is_ia32_PopMem(node)) {
		ir_mode *ls_mode = get_ia32_ls_mode(node);
		return -get_mode_size_bytes(ls_mode);
	}
207

Matthias Braun's avatar
Matthias Braun committed
208
	if (is_ia32_Leave(node) || is_ia32_CopyEbpEsp(node))
209
210
		return SP_BIAS_RESET;

211
212
213
	return 0;
}

Sebastian Hack's avatar
Sebastian Hack committed
214
/**
215
 * Build the between type and entities if not already built.
Sebastian Hack's avatar
Sebastian Hack committed
216
 */
217
218
static void ia32_build_between_type(void)
{
219
	if (between_type == NULL) {
220
221
		ir_type *old_bp_type   = new_type_primitive(ia32_mode_gp);
		ir_type *ret_addr_type = new_type_primitive(ia32_mode_gp);
Sebastian Hack's avatar
Sebastian Hack committed
222

223
224
225
		between_type = new_type_struct(NEW_IDENT("ia32_between_type"));
		old_bp_ent   = new_entity(between_type, NEW_IDENT("old_bp"), old_bp_type);
		ret_addr_ent = new_entity(between_type, NEW_IDENT("ret_addr"), ret_addr_type);
Sebastian Hack's avatar
Sebastian Hack committed
226

227
228
229
		set_entity_offset(old_bp_ent, 0);
		set_entity_offset(ret_addr_ent, get_type_size_bytes(old_bp_type));
		set_type_size_bytes(between_type, get_type_size_bytes(old_bp_type) + get_type_size_bytes(ret_addr_type));
230
		set_type_state(between_type, layout_fixed);
231

232
233
		omit_fp_between_type = new_type_struct(NEW_IDENT("ia32_between_type_omit_fp"));
		omit_fp_ret_addr_ent = new_entity(omit_fp_between_type, NEW_IDENT("ret_addr"), ret_addr_type);
234
235
236
237

		set_entity_offset(omit_fp_ret_addr_ent, 0);
		set_type_size_bytes(omit_fp_between_type, get_type_size_bytes(ret_addr_type));
		set_type_state(omit_fp_between_type, layout_fixed);
Sebastian Hack's avatar
Sebastian Hack committed
238
	}
239
240
241
242
243
}

/**
 * Return the stack entity that contains the return address.
 */
244
ir_entity *ia32_get_return_address_entity(ir_graph *irg)
245
{
246
	const be_stack_layout_t *layout = be_get_irg_stack_layout(irg);
247
	ia32_build_between_type();
248
	return layout->sp_relative ? omit_fp_ret_addr_ent : ret_addr_ent;
249
250
251
252
253
}

/**
 * Return the stack entity that contains the frame address.
 */
254
ir_entity *ia32_get_frame_address_entity(ir_graph *irg)
255
{
256
	const be_stack_layout_t *layout = be_get_irg_stack_layout(irg);
257
	ia32_build_between_type();
258
	return layout->sp_relative ? NULL : old_bp_ent;
Sebastian Hack's avatar
Sebastian Hack committed
259
260
}

261
262
263
264
265
266
267
/**
 * Get the estimated cycle count for @p irn.
 *
 * @param irn  The node.
 *
 * @return     The estimated cycle count for this operation
 */
268
static unsigned ia32_get_op_estimated_cost(ir_node const *const irn)
269
{
270
271
272
	if (!is_ia32_irn(irn))
		return 1;

273
	if (is_ia32_CopyB_i(irn)) {
274
275
		unsigned const size = get_ia32_copyb_size(irn);
		return 20 + size * 4 / 3;
Christian Würdig's avatar
Christian Würdig committed
276
	}
277
278
279

	unsigned cost = get_ia32_latency(irn);

Christian Würdig's avatar
Christian Würdig committed
280
	/* in case of address mode operations add additional cycles */
281
	if (get_ia32_op_type(irn) != ia32_Normal) {
282
		if (get_ia32_frame_use(irn) != IA32_FRAME_USE_NONE || (
283
284
		      is_ia32_NoReg_GP(get_irn_n(irn, n_ia32_base)) &&
		      is_ia32_NoReg_GP(get_irn_n(irn, n_ia32_index))
285
		    )) {
286
			/* Stack access, assume it is cached. */
287
288
			cost += 5;
		} else {
289
			/* Access probably elsewhere. */
290
291
			cost += 20;
		}
Christian Würdig's avatar
Christian Würdig committed
292
293
294
	}

	return cost;
295
296
}

297
static ir_mode *get_spill_mode(const ir_node *value)
298
{
299
300
301
302
303
	/* determine a sensible spill mode and try to make it small */
	const ir_node *skipped = skip_Proj_const(value);
	if (is_ia32_fld(skipped) || is_ia32_Load(skipped))
		return get_ia32_ls_mode(skipped);

304
	return get_irn_mode(value);
305
306
}

307
/**
Christoph Mallon's avatar
Christoph Mallon committed
308
 * Check if irn can load its operand at position i from memory (source addressmode).
309
310
 * @param irn    The irn to be checked
 * @param i      The operands position
311
 * @return whether operand can be loaded
312
 */
313
static bool ia32_possible_memory_operand(const ir_node *irn, unsigned int i)
314
{
315
316
	if (!is_ia32_irn(irn)                    || /* must be an ia32 irn */
	    get_ia32_op_type(irn) != ia32_Normal || /* must not already be a addressmode irn */
317
	    get_ia32_frame_use(irn) != IA32_FRAME_USE_NONE) /* must not already use frame */
Matthias Braun's avatar
Matthias Braun committed
318
		return false;
319

320
	switch (get_ia32_am_support(irn)) {
Matthias Braun's avatar
cleanup    
Matthias Braun committed
321
	case ia32_am_none:
Matthias Braun's avatar
Matthias Braun committed
322
		return false;
Matthias Braun's avatar
cleanup    
Matthias Braun committed
323
324
325

	case ia32_am_unary:
		if (i != n_ia32_unary_op)
Matthias Braun's avatar
Matthias Braun committed
326
			return false;
Matthias Braun's avatar
cleanup    
Matthias Braun committed
327
		break;
328

Matthias Braun's avatar
cleanup    
Matthias Braun committed
329
330
331
332
	case ia32_am_binary:
		switch (i) {
		case n_ia32_binary_left: {
			if (!is_ia32_commutative(irn))
Matthias Braun's avatar
Matthias Braun committed
333
				return false;
334

Matthias Braun's avatar
cleanup    
Matthias Braun committed
335
336
337
			/* we can't swap left/right for limited registers
			 * (As this (currently) breaks constraint handling copies) */
			arch_register_req_t const *const req = arch_get_irn_register_req_in(irn, n_ia32_binary_left);
338
			if (req->limited != NULL)
Matthias Braun's avatar
Matthias Braun committed
339
				return false;
Matthias Braun's avatar
cleanup    
Matthias Braun committed
340
341
			break;
		}
342

Matthias Braun's avatar
cleanup    
Matthias Braun committed
343
		case n_ia32_binary_right:
344
			break;
345
346

		default:
Matthias Braun's avatar
Matthias Braun committed
347
			return false;
Matthias Braun's avatar
cleanup    
Matthias Braun committed
348
349
350
351
		}
		break;

	default:
352
		panic("unknown AM type");
353
	}
354
355
356
357

	/* HACK: must not already use "real" memory.
	 * This can happen for Call and Div */
	if (!is_NoMem(get_irn_n(irn, n_ia32_mem)))
Matthias Braun's avatar
Matthias Braun committed
358
		return false;
359

360
361
362
363
364
365
366
367
368
369
370
371
372
	ir_node       *const op   = get_irn_n(irn, i);
	ir_node const *const load = get_Proj_pred(op);
	ir_mode const *const mode = get_ia32_ls_mode(load);
	if (mode_is_float(mode)) {
		if (mode != ia32_mode_float64 && mode != mode_F)
			return false;
		/* Don't do reload folding for x87 nodes for now, as we can't predict
		 * yet wether the spillslot must be widened to 80bit for which no AM
		 * operations exist. */
		if (is_ia32_fld(load))
			return false;
	}

Matthias Braun's avatar
Matthias Braun committed
373
	return true;
374
375
}

376
static void ia32_perform_memory_operand(ir_node *irn, unsigned int i)
377
{
378
379
	if (!ia32_possible_memory_operand(irn, i))
		return;
380

381
382
383
384
	ir_node *op           = get_irn_n(irn, i);
	ir_node *load         = get_Proj_pred(op);
	ir_mode *load_mode    = get_ia32_ls_mode(load);
	ir_node *spill        = get_irn_n(load, n_ia32_mem);
385
	ir_mode *dest_op_mode = get_ia32_ls_mode(irn);
Matthias Braun's avatar
Matthias Braun committed
386
	if (get_mode_size_bits(load_mode) <= get_mode_size_bits(dest_op_mode))
387
		set_ia32_ls_mode(irn, load_mode);
388
	set_ia32_op_type(irn, ia32_AddrModeS);
389
	set_ia32_frame_use(irn, IA32_FRAME_USE_AUTO);
390

391
392
	if (i == n_ia32_binary_left                    &&
	    get_ia32_am_support(irn) == ia32_am_binary &&
393
394
	    /* immediates are only allowed on the right side */
	    !is_ia32_Immediate(get_irn_n(irn, n_ia32_binary_right))) {
395
		ia32_swap_left_right(irn);
396
		i = n_ia32_binary_right;
397
	}
398

399
400
	assert(is_NoMem(get_irn_n(irn, n_ia32_mem)));

401
402
	set_irn_n(irn, n_ia32_base, get_irg_frame(get_irn_irg(irn)));
	set_irn_n(irn, n_ia32_mem,  spill);
403
	set_irn_n(irn, i,           ia32_get_admissible_noreg(irn, i));
404
	set_ia32_is_reload(irn);
405
406
407
408
409
410
411

	/* kill the reload */
	assert(get_irn_n_edges(op) == 0);
	assert(get_irn_n_edges(load) == 1);
	sched_remove(load);
	kill_node(op);
	kill_node(load);
412
413
}

Matthias Braun's avatar
Matthias Braun committed
414
static bool gprof;
Michael Beck's avatar
Michael Beck committed
415

416
417
418
419
420
421
422
423
static ir_node *ia32_turn_back_dest_am(ir_node *node)
{
	typedef ir_node *construct_binop_func(
		dbg_info *db, ir_node *block,
		ir_node *base, ir_node *index, ir_node *mem,
		ir_node *op1, ir_node *op2);

	construct_binop_func *func;
424
425
426
427
428
429
430
	switch (get_ia32_irn_opcode(node)) {
	case iro_ia32_AddMem: func = new_bd_ia32_Add; break;
	case iro_ia32_AndMem: func = new_bd_ia32_And; break;
	case iro_ia32_OrMem:  func = new_bd_ia32_Or;  break;
	case iro_ia32_SubMem: func = new_bd_ia32_Sub; break;
	case iro_ia32_XorMem: func = new_bd_ia32_Xor; break;
	default: panic("cannot turn back DestAM for %+F", node);
431
432
	}

433
434
435
436
437
438
	dbg_info *const dbgi  = get_irn_dbg_info(node);
	ir_node  *const block = get_nodes_block(node);
	ir_node  *const base  = get_irn_n(node, n_ia32_base);
	ir_node  *const idx   = get_irn_n(node, n_ia32_index);
	ir_node  *const mem   = get_irn_n(node, n_ia32_mem);
	ir_node  *const load  = new_bd_ia32_Load(dbgi, block, base, idx, mem);
439
440
441
	ia32_copy_am_attrs(load, node);
	if (is_ia32_is_reload(node))
		set_ia32_is_reload(load);
442
	sched_add_before(node, load);
443
444
	ir_node *const load_res = be_new_Proj(load, pn_ia32_Load_res);
	ir_node *const load_mem = be_new_Proj(load, pn_ia32_Load_M);
445
446
447
448
449
450

	ir_graph *const irg      = get_irn_irg(node);
	ir_node  *const noreg    = ia32_new_NoReg_gp(irg);
	ir_node  *const nomem    = get_irg_no_mem(irg);
	ir_node  *const operand  = get_irn_n(node, n_ia32_binary_left);
	ir_node  *const new_node = func(dbgi, block, noreg, noreg, nomem, load_res, operand);
451
452
	set_ia32_ls_mode(new_node, get_ia32_ls_mode(node));
	set_irn_mode(new_node, mode_T);
453
	ir_node *const res_proj = be_new_Proj(new_node, pn_ia32_res);
454

455
	ir_node *const store = new_bd_ia32_Store(dbgi, block, base, idx, load_mem, res_proj);
456
457
	ia32_copy_am_attrs(store, node);
	set_ia32_op_type(store, ia32_AddrModeD);
458
	sched_add_after(node, store);
459

460
461
462
	ir_node *const mem_proj = get_Proj_for_pn(node, pn_ia32_M);
	set_Proj_pred(mem_proj, store);
	set_Proj_num(mem_proj, pn_ia32_Store_M);
463

464
465
	sched_replace(node, new_node);
	exchange(node, new_node);
466
467
468
	return new_node;
}

469
ir_node *ia32_turn_back_am(ir_node *node)
470
{
Matthias Braun's avatar
Matthias Braun committed
471
472
473
474
475
476
	dbg_info *dbgi     = get_irn_dbg_info(node);
	ir_graph *irg      = get_irn_irg(node);
	ir_node  *block    = get_nodes_block(node);
	ir_node  *base     = get_irn_n(node, n_ia32_base);
	ir_node  *idx      = get_irn_n(node, n_ia32_index);
	ir_node  *mem      = get_irn_n(node, n_ia32_mem);
477
	ir_node  *load     = new_bd_ia32_Load(dbgi, block, base, idx, mem);
478
	ir_node  *load_res = be_new_Proj(load, pn_ia32_Load_res);
479
480

	ia32_copy_am_attrs(load, node);
481
482
	if (is_ia32_is_reload(node))
		set_ia32_is_reload(load);
483
	set_irn_n(node, n_ia32_mem, get_irg_no_mem(irg));
484

485
	switch (get_ia32_am_support(node)) {
Matthias Braun's avatar
cleanup    
Matthias Braun committed
486
487
488
	case ia32_am_unary:
		set_irn_n(node, n_ia32_unary_op, load_res);
		break;
489

Matthias Braun's avatar
cleanup    
Matthias Braun committed
490
491
492
493
494
495
496
	case ia32_am_binary:
		if (is_ia32_Immediate(get_irn_n(node, n_ia32_binary_right))) {
			set_irn_n(node, n_ia32_binary_left, load_res);
		} else {
			set_irn_n(node, n_ia32_binary_right, load_res);
		}
		break;
497

Matthias Braun's avatar
cleanup    
Matthias Braun committed
498
	default:
499
		panic("unknown AM type");
500
	}
Matthias Braun's avatar
Matthias Braun committed
501
	ir_node *noreg = ia32_new_NoReg_gp(irg);
502
	set_irn_n(node, n_ia32_base,  noreg);
503
	set_irn_n(node, n_ia32_index, noreg);
504
505
	ia32_attr_t *const attr = get_ia32_attr(node);
	attr->am_imm = (x86_imm32_t) { .kind = X86_IMM_VALUE, .offset = 0 };
506
	attr->frame_use = IA32_FRAME_USE_NONE;
507
	set_ia32_am_scale(node, 0);
508
509

	/* rewire mem-proj */
Michael Beck's avatar
Michael Beck committed
510
	if (get_irn_mode(node) == mode_T) {
511
512
		foreach_out_edge(node, edge) {
			ir_node *out = get_edge_src_irn(edge);
513
514
			if (get_irn_mode(out) == mode_M) {
				set_Proj_pred(out, load);
515
				set_Proj_num(out, pn_ia32_Load_M);
516
				break;
517
518
519
520
521
			}
		}
	}

	set_ia32_op_type(node, ia32_Normal);
Michael Beck's avatar
Michael Beck committed
522
	if (sched_is_scheduled(node))
523
		sched_add_before(node, load);
524
525

	return load_res;
526
527
528
529
}

static ir_node *flags_remat(ir_node *node, ir_node *after)
{
530
	/* we should turn back address modes when rematerializing nodes */
Christoph Mallon's avatar
Christoph Mallon committed
531
	ir_node *const block = get_block(after);
532

Matthias Braun's avatar
Matthias Braun committed
533
	ia32_op_type_t type = get_ia32_op_type(node);
534
	switch (type) {
Matthias Braun's avatar
cleanup    
Matthias Braun committed
535
536
537
	case ia32_AddrModeS:
		ia32_turn_back_am(node);
		break;
538

Matthias Braun's avatar
cleanup    
Matthias Braun committed
539
	case ia32_AddrModeD:
540
541
		node = ia32_turn_back_dest_am(node);
		break;
542

Matthias Braun's avatar
Matthias Braun committed
543
544
545
	default:
		assert(type == ia32_Normal);
		break;
546
547
	}

Matthias Braun's avatar
Matthias Braun committed
548
	ir_node *copy = exact_copy(node);
549
	set_nodes_block(copy, block);
550
551
552
553
	sched_add_after(after, copy);
	return copy;
}

554
555
COMPILETIME_ASSERT((int)(n_ia32_Sub_minuend)    == (int)(n_ia32_Cmp_left) &&
                   (int)(n_ia32_Sub_subtrahend) == (int)(n_ia32_Cmp_right),
Matthias Braun's avatar
Matthias Braun committed
556
                   Cmp_and_Sub_operand_numbers_equal)
557

558
static bool ia32_try_replace_flags(ir_node *consumers, ir_node *flags, ir_node *available, unsigned pn)
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
{
	if ((is_ia32_Sub(flags) || is_ia32_Cmp(flags)) &&
	    (is_ia32_Sub(available) || is_ia32_Cmp(available))) {

		ir_node *flags_left  = get_irn_n(flags,     n_ia32_Cmp_left);
		ir_node *flags_right = get_irn_n(flags,     n_ia32_Cmp_right);
		ir_node *avail_left  = get_irn_n(available, n_ia32_Cmp_left);
		ir_node *avail_right = get_irn_n(available, n_ia32_Cmp_right);

		/* Assuming CSE would have found the more obvious case */
		if (flags_left == avail_right && avail_left == flags_right) {
			/* We can use available if we reverse the
			 * consumers' condition codes. */
			ir_mode               *flag_mode =  ia32_reg_classes[CLASS_ia32_flags].mode;
			const arch_register_t *flag_reg  = &ia32_reg_classes[CLASS_ia32_flags].regs[0];

			for (ir_node *c = consumers; c != NULL; c = get_irn_link(c)) {
				x86_condition_code_t cc = get_ia32_condcode(c);
				set_ia32_condcode(c, x86_invert_condition_code(cc));

				foreach_irn_in(c, i, in) {
					if (get_irn_mode(in) == flag_mode) {
581
						ir_node *const proj = be_new_Proj(available, pn);
582
583
584
585
586
587
588
589
590
591
592
						arch_set_irn_register(proj, flag_reg);
						set_irn_n(c, i, proj);
					}
				}
			}
			return true;
		}
	}
	return false;
}

593
static void remat_simplifier(ir_node *node, void *env)
594
595
596
597
{
	(void)env;
	const arch_register_t *flags_reg = &(ia32_reg_classes[CLASS_ia32_flags].regs[0]);

598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
	/* A Sub with unused result is a Cmp. */
	if (is_ia32_Sub(node) && get_irn_mode(node) == mode_T) {
		bool      has_res_users  = false;
		ir_node  *res_keep       = NULL;
		ir_node **flag_users     = NEW_ARR_F(ir_node*, 0);
		int      *flag_users_pos = NEW_ARR_F(int, 0);
		ir_node **mem_users      = NEW_ARR_F(ir_node*, 0);
		int      *mem_users_pos  = NEW_ARR_F(int, 0);

		foreach_out_edge(node, out) {
			ir_node *proj = get_edge_src_irn(out);
			ir_mode *proj_mode = get_irn_mode(proj);
			if (proj_mode == ia32_mode_flags) {
				foreach_out_edge(proj, out2) {
					ir_node *user = get_edge_src_irn(out2);
					ARR_APP1(ir_node*, flag_users, user);
					ARR_APP1(int, flag_users_pos, get_edge_src_pos(out2));
				}
			} else if (proj_mode == ia32_mode_gp) {
				foreach_out_edge(proj, out2) {
					ir_node *user = get_edge_src_irn(out2);
					if (!be_is_Keep(user)) {
						has_res_users = true;
					} else {
						assert(res_keep == NULL && "Proj has two be_Keep");
						res_keep = user;
624
					}
625
				}
626
627
628
629
630
			} else if (proj_mode == mode_M) {
				foreach_out_edge(proj, out2) {
					ir_node *user = get_edge_src_irn(out2);
					ARR_APP1(ir_node*, mem_users, user);
					ARR_APP1(int, mem_users_pos, get_edge_src_pos(out2));
631
				}
yb9976's avatar
yb9976 committed
632
633
			} else {
				panic("unexpected mode for %+F", proj);
634
635
			}
		}
636

637
638
639
640
641
642
643
644
645
646
647
		if (!has_res_users) {
			ir_node *cmp = new_bd_ia32_Cmp(
				get_irn_dbg_info(node),
				get_nodes_block(node),
				get_irn_n(node, n_ia32_Sub_base),
				get_irn_n(node, n_ia32_Sub_index),
				get_irn_n(node, n_ia32_Sub_mem),
				get_irn_n(node, n_ia32_Sub_minuend),
				get_irn_n(node, n_ia32_Sub_subtrahend),
				false);
			arch_set_irn_register(cmp, flags_reg);
648
			ia32_copy_am_attrs(cmp, node);
649
650
651
652

			sched_replace(node, cmp);

			if (get_ia32_op_type(node) == ia32_AddrModeD) {
653
				panic("unexpected DestAM node %+F", node);
654
655
656
657
			}
			if (get_ia32_op_type(node) == ia32_AddrModeS) {
				set_ia32_op_type(cmp, ia32_AddrModeS);
				set_irn_mode(cmp, mode_T);
658

659
				ir_node *const proj_M = be_new_Proj(cmp, pn_ia32_Cmp_M);
660
661
				for(unsigned i = 0; i < ARR_LEN(mem_users); i++) {
					set_irn_n(mem_users[i], mem_users_pos[i], proj_M);
662
663
				}

664
				cmp = be_new_Proj(cmp, pn_ia32_Cmp_eflags);
665
			}
666

667
668
			for (unsigned i = 0; i < ARR_LEN(flag_users); i++) {
				set_irn_n(flag_users[i], flag_users_pos[i], cmp);
669
670
			}

671
672
673
674
675
676
677
			if (res_keep) {
				sched_remove(res_keep);
				remove_keep_alive(res_keep);
				kill_node(res_keep);
			}
			kill_node(node);
			node = cmp;
678
679
		}

680
681
682
683
684
		DEL_ARR_F(flag_users);
		DEL_ARR_F(flag_users_pos);
		DEL_ARR_F(mem_users);
		DEL_ARR_F(mem_users_pos);
	}
685
686
687
688
}

static void simplify_remat_nodes(ir_graph *irg)
{
689
690
	irg_walk_graph(irg, remat_simplifier, NULL, NULL);
	remove_End_Bads_and_doublets(get_irg_end(irg));
691
692
}

693
static ir_node *ia32_new_spill(ir_node *value, ir_node *after)
694
{
695
696
697
698
699
700
	ir_graph *irg   = get_irn_irg(value);
	ir_node  *block = get_block(after);
	ir_node  *frame = get_irg_frame(irg);
	ir_mode  *mode  = get_spill_mode(value);
	ir_node  *noreg = ia32_new_NoReg_gp(irg);
	ir_node  *nomem = get_irg_no_mem(irg);
Christian Würdig's avatar
Christian Würdig committed
701

702
	ir_node *res;
703
	ir_node *store;
Christian Würdig's avatar
Christian Würdig committed
704
	if (mode_is_float(mode)) {
705
		if (ia32_cg_config.use_sse2) {
706
			store = new_bd_ia32_xStore(NULL, block, frame, noreg, nomem, value);
707
			res   = be_new_Proj(store, pn_ia32_xStore_M);
708
		} else {
709
			store = new_bd_ia32_fst(NULL, block, frame, noreg, nomem, value, mode);
710
			res   = be_new_Proj(store, pn_ia32_fst_M);
711
		}
712
	} else if (get_mode_size_bits(mode) == 128) {
713
		/* Spill 128 bit SSE registers */
714
		store = new_bd_ia32_xxStore(NULL, block, frame, noreg, nomem, value);
715
		res   = be_new_Proj(store, pn_ia32_xxStore_M);
716
	} else {
717
		store = get_mode_size_bits(mode) == 8
718
719
			? new_bd_ia32_Store_8bit(NULL, block, frame, noreg, nomem, value)
			: new_bd_ia32_Store     (NULL, block, frame, noreg, nomem, value);
720
		res   = be_new_Proj(store, pn_ia32_Store_M);
Christian Würdig's avatar
Christian Würdig committed
721
	}
722
723
	set_ia32_op_type(store, ia32_AddrModeD);
	set_ia32_ls_mode(store, mode);
724
	set_ia32_frame_use(store, IA32_FRAME_USE_AUTO);
725
	set_ia32_is_spill(store);
726
	sched_add_after(after, store);
Christian Würdig's avatar
Christian Würdig committed
727

728
729
730
731
732
733
734
	return res;
}

static ir_node *ia32_new_reload(ir_node *value, ir_node *spill, ir_node *before)
{
	ir_graph *irg       = get_irn_irg(before);
	ir_node  *block     = get_block(before);
735
	ir_mode  *spillmode = get_spill_mode(value);
736
737
738
	ir_node  *noreg     = ia32_new_NoReg_gp(irg);
	ir_node  *frame     = get_irg_frame(irg);

739
	ir_node *load;
740
	if (mode_is_float(spillmode)) {
741
		if (ia32_cg_config.use_sse2) {
742
			load = new_bd_ia32_xLoad(NULL, block, frame, noreg, spill, spillmode);
743
		} else {
744
			load = new_bd_ia32_fld(NULL, block, frame, noreg, spill, spillmode);
745
		}
746
747
748
749
750
751
752
753
	} else if (get_mode_size_bits(spillmode) == 128) {
		/* Reload 128 bit SSE registers */
		load = new_bd_ia32_xxLoad(NULL, block, frame, noreg, spill);
	} else {
		load = new_bd_ia32_Load(NULL, block, frame, noreg, spill);
	}
	set_ia32_op_type(load, ia32_AddrModeS);
	set_ia32_ls_mode(load, spillmode);
754
	set_ia32_frame_use(load, IA32_FRAME_USE_AUTO);
755
756
757
758
	set_ia32_is_reload(load);
	arch_add_irn_flags(load, arch_irn_flag_reload);
	sched_add_before(before, load);

759
	return be_new_Proj(load, pn_ia32_res);
Christian Würdig's avatar
Christian Würdig committed
760
761
}

Matthias Braun's avatar
Matthias Braun committed
762
763
static ir_node *create_push(ir_node *node, ir_node *schedpoint, ir_node *sp,
                            ir_node *mem, ir_entity *ent, ir_mode *mode)
764
{
765
766
767
768
769
	dbg_info *dbgi  = get_irn_dbg_info(node);
	ir_node  *block = get_nodes_block(node);
	ir_graph *irg   = get_irn_irg(node);
	ir_node  *noreg = ia32_new_NoReg_gp(irg);
	ir_node  *frame = get_irg_frame(irg);
770

771
772
773
774
775
776
777
778
	ir_node *const push = new_bd_ia32_Push(dbgi, block, frame, noreg, mem,
	                                       noreg, sp, mode);
	ia32_attr_t *const attr = get_ia32_attr(push);
	attr->am_imm = (x86_imm32_t) {
		.kind   = X86_IMM_FRAMEOFFSET,
		.entity = ent,
	};
	set_ia32_frame_use(push, IA32_FRAME_USE_AUTO);
779
	set_ia32_op_type(push, ia32_AddrModeS);
780
	set_ia32_is_spill(push);
781
782
783
784
785

	sched_add_before(schedpoint, push);
	return push;
}

Matthias Braun's avatar
Matthias Braun committed
786
787
static ir_node *create_pop(ir_node *node, ir_node *schedpoint, ir_node *sp,
                           ir_entity *ent, ir_mode *mode)
788
{
789
	dbg_info *dbgi  = get_irn_dbg_info(node);
790
791
792
793
	ir_node  *block = get_nodes_block(node);
	ir_graph *irg   = get_irn_irg(node);
	ir_node  *noreg = ia32_new_NoReg_gp(irg);
	ir_node  *frame = get_irg_frame(irg);
Matthias Braun's avatar
Matthias Braun committed
794
795
	ir_node  *pop   = new_bd_ia32_PopMem(dbgi, block, frame, noreg,
	                                     get_irg_no_mem(irg), sp);
796
797
798
799
800
801
	ia32_attr_t *const attr = get_ia32_attr(pop);
	attr->am_imm = (x86_imm32_t) {
		.kind   = X86_IMM_FRAMEOFFSET,
		.entity = ent,
	};
	set_ia32_frame_use(pop, IA32_FRAME_USE_AUTO);
802
	set_ia32_op_type(pop, ia32_AddrModeD);
Matthias Braun's avatar
Matthias Braun committed
803
	set_ia32_ls_mode(pop, mode);
804
	set_ia32_is_reload(pop);
805
806
807
808
	sched_add_before(schedpoint, pop);
	return pop;
}

809
static ir_node *create_spproj(ir_node *const pred, unsigned const pos)
810
{
811
812
	ir_node *const sp = be_new_Proj(pred, pos);
	arch_set_irn_register(sp, &ia32_registers[REG_ESP]);
813
814
815
	return sp;
}

816
/**
817
 * Transform MemPerm, currently we do this the ugly way and produce
818
819
820
 * push/pop into/from memory cascades. This is possible without using
 * any registers.
 */
821
static void transform_MemPerm(ir_node *node)
822
{
823
	ir_graph *irg   = get_irn_irg(node);
Christoph Mallon's avatar
Christoph Mallon committed
824
	ir_node  *sp    = be_get_Start_proj(irg, &ia32_registers[REG_ESP]);
825
826
	int       arity = be_get_MemPerm_entity_arity(node);
	ir_node **pops  = ALLOCAN(ir_node*, arity);
827

828
	/* create Pushs */
Matthias Braun's avatar
Matthias Braun committed
829
	for (int i = 0; i < arity; ++i) {
830
831
832
		ir_entity *inent = be_get_MemPerm_in_entity(node, i);
		ir_entity *outent = be_get_MemPerm_out_entity(node, i);
		ir_type *enttype = get_entity_type(inent);
833
834
		unsigned entsize = get_type_size_bytes(enttype);
		unsigned entsize2 = get_type_size_bytes(get_entity_type(outent));
835
		ir_node *mem = get_irn_n(node, i);
836

837
		/* work around cases where entities have different sizes */
838
		if (entsize2 < entsize)
839
			entsize = entsize2;
840
841
842
843
844
845
846
847
848
849
850

		int offset = 0;
		do {
			ir_mode *mode;
			if (entsize%2 == 1) {
				mode = mode_Bu;
			} else if (entsize % 4 == 2) {
				mode = mode_Hu;
			} else {
				assert(entsize%4 == 0);
				mode = ia32_mode_gp;
Matthias Braun's avatar
Matthias Braun committed
851
			}
852

853
			ir_node *push = create_push(node, node, sp, mem, inent, mode);
854
			sp = create_spproj(push, pn_ia32_Push_stack);
855
856
857
858
859
860
			add_ia32_am_offs_int(push, offset);

			unsigned size = get_mode_size_bytes(mode);
			offset  += size;
			entsize -= size;
		} while(entsize > 0);
Matthias Braun's avatar
Matthias Braun committed
861
		set_irn_n(node, i, new_r_Bad(irg, mode_X));
862
863
	}

864
	/* create pops */
Matthias Braun's avatar
Matthias Braun committed
865
	for (int i = arity; i-- > 0; ) {
866
867
868
		ir_entity *inent = be_get_MemPerm_in_entity(node, i);
		ir_entity *outent = be_get_MemPerm_out_entity(node, i);
		ir_type *enttype = get_entity_type(outent);
869
870
		unsigned entsize = get_type_size_bytes(enttype);
		unsigned entsize2 = get_type_size_bytes(get_entity_type(inent));
871

872
		/* work around cases where entities have different sizes */
873
		if (entsize2 < entsize)
874
			entsize = entsize2;
875

876
877
878
879
880 <