1 /* Subroutines used for code generation on IA-32.
2 Copyright (C) 1988-2015 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 3, or (at your option)
11 GCC is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
28 #include "fold-const.h"
29 #include "stringpool.h"
32 #include "stor-layout.h"
36 #include "hard-reg-set.h"
37 #include "insn-config.h"
38 #include "conditions.h"
40 #include "insn-codes.h"
41 #include "insn-attr.h"
53 #include "diagnostic-core.h"
56 #include "dominance.h"
62 #include "cfgcleanup.h"
63 #include "basic-block.h"
65 #include "target-def.h"
66 #include "common/common-target.h"
67 #include "langhooks.h"
69 #include "plugin-api.h"
72 #include "tree-ssa-alias.h"
73 #include "internal-fn.h"
74 #include "gimple-fold.h"
76 #include "gimple-expr.h"
82 #include "tm-constrs.h"
86 #include "sched-int.h"
90 #include "diagnostic.h"
92 #include "tree-pass.h"
94 #include "pass_manager.h"
95 #include "target-globals.h"
96 #include "tree-vectorizer.h"
97 #include "shrink-wrap.h"
100 #include "tree-iterator.h"
101 #include "tree-chkp.h"
102 #include "rtl-chkp.h"
104 static rtx legitimize_dllimport_symbol (rtx, bool);
105 static rtx legitimize_pe_coff_extern_decl (rtx, bool);
106 static rtx legitimize_pe_coff_symbol (rtx, bool);
108 #ifndef CHECK_STACK_LIMIT
109 #define CHECK_STACK_LIMIT (-1)
112 /* Return index of given mode in mult and division cost tables. */
113 #define MODE_INDEX(mode) \
114 ((mode) == QImode ? 0 \
115 : (mode) == HImode ? 1 \
116 : (mode) == SImode ? 2 \
117 : (mode) == DImode ? 3 \
120 /* Processor costs (relative to an add) */
121 /* We assume COSTS_N_INSNS is defined as (N)*4 and an addition is 2 bytes. */
122 #define COSTS_N_BYTES(N) ((N) * 2)
124 #define DUMMY_STRINGOP_ALGS {libcall, {{-1, libcall, false}}}
126 static stringop_algs ix86_size_memcpy[2] = {
127 {rep_prefix_1_byte, {{-1, rep_prefix_1_byte, false}}},
128 {rep_prefix_1_byte, {{-1, rep_prefix_1_byte, false}}}};
129 static stringop_algs ix86_size_memset[2] = {
130 {rep_prefix_1_byte, {{-1, rep_prefix_1_byte, false}}},
131 {rep_prefix_1_byte, {{-1, rep_prefix_1_byte, false}}}};
134 struct processor_costs ix86_size_cost = {/* costs for tuning for size */
135 COSTS_N_BYTES (2), /* cost of an add instruction */
136 COSTS_N_BYTES (3), /* cost of a lea instruction */
137 COSTS_N_BYTES (2), /* variable shift costs */
138 COSTS_N_BYTES (3), /* constant shift costs */
139 {COSTS_N_BYTES (3), /* cost of starting multiply for QI */
140 COSTS_N_BYTES (3), /* HI */
141 COSTS_N_BYTES (3), /* SI */
142 COSTS_N_BYTES (3), /* DI */
143 COSTS_N_BYTES (5)}, /* other */
144 0, /* cost of multiply per each bit set */
145 {COSTS_N_BYTES (3), /* cost of a divide/mod for QI */
146 COSTS_N_BYTES (3), /* HI */
147 COSTS_N_BYTES (3), /* SI */
148 COSTS_N_BYTES (3), /* DI */
149 COSTS_N_BYTES (5)}, /* other */
150 COSTS_N_BYTES (3), /* cost of movsx */
151 COSTS_N_BYTES (3), /* cost of movzx */
152 0, /* "large" insn */
154 2, /* cost for loading QImode using movzbl */
155 {2, 2, 2}, /* cost of loading integer registers
156 in QImode, HImode and SImode.
157 Relative to reg-reg move (2). */
158 {2, 2, 2}, /* cost of storing integer registers */
159 2, /* cost of reg,reg fld/fst */
160 {2, 2, 2}, /* cost of loading fp registers
161 in SFmode, DFmode and XFmode */
162 {2, 2, 2}, /* cost of storing fp registers
163 in SFmode, DFmode and XFmode */
164 3, /* cost of moving MMX register */
165 {3, 3}, /* cost of loading MMX registers
166 in SImode and DImode */
167 {3, 3}, /* cost of storing MMX registers
168 in SImode and DImode */
169 3, /* cost of moving SSE register */
170 {3, 3, 3}, /* cost of loading SSE registers
171 in SImode, DImode and TImode */
172 {3, 3, 3}, /* cost of storing SSE registers
173 in SImode, DImode and TImode */
174 3, /* MMX or SSE register to integer */
175 0, /* size of l1 cache */
176 0, /* size of l2 cache */
177 0, /* size of prefetch block */
178 0, /* number of parallel prefetches */
180 COSTS_N_BYTES (2), /* cost of FADD and FSUB insns. */
181 COSTS_N_BYTES (2), /* cost of FMUL instruction. */
182 COSTS_N_BYTES (2), /* cost of FDIV instruction. */
183 COSTS_N_BYTES (2), /* cost of FABS instruction. */
184 COSTS_N_BYTES (2), /* cost of FCHS instruction. */
185 COSTS_N_BYTES (2), /* cost of FSQRT instruction. */
188 1, /* scalar_stmt_cost. */
189 1, /* scalar load_cost. */
190 1, /* scalar_store_cost. */
191 1, /* vec_stmt_cost. */
192 1, /* vec_to_scalar_cost. */
193 1, /* scalar_to_vec_cost. */
194 1, /* vec_align_load_cost. */
195 1, /* vec_unalign_load_cost. */
196 1, /* vec_store_cost. */
197 1, /* cond_taken_branch_cost. */
198 1, /* cond_not_taken_branch_cost. */
201 /* Processor costs (relative to an add) */
202 static stringop_algs i386_memcpy[2] = {
203 {rep_prefix_1_byte, {{-1, rep_prefix_1_byte, false}}},
204 DUMMY_STRINGOP_ALGS};
205 static stringop_algs i386_memset[2] = {
206 {rep_prefix_1_byte, {{-1, rep_prefix_1_byte, false}}},
207 DUMMY_STRINGOP_ALGS};
210 struct processor_costs i386_cost = { /* 386 specific costs */
211 COSTS_N_INSNS (1), /* cost of an add instruction */
212 COSTS_N_INSNS (1), /* cost of a lea instruction */
213 COSTS_N_INSNS (3), /* variable shift costs */
214 COSTS_N_INSNS (2), /* constant shift costs */
215 {COSTS_N_INSNS (6), /* cost of starting multiply for QI */
216 COSTS_N_INSNS (6), /* HI */
217 COSTS_N_INSNS (6), /* SI */
218 COSTS_N_INSNS (6), /* DI */
219 COSTS_N_INSNS (6)}, /* other */
220 COSTS_N_INSNS (1), /* cost of multiply per each bit set */
221 {COSTS_N_INSNS (23), /* cost of a divide/mod for QI */
222 COSTS_N_INSNS (23), /* HI */
223 COSTS_N_INSNS (23), /* SI */
224 COSTS_N_INSNS (23), /* DI */
225 COSTS_N_INSNS (23)}, /* other */
226 COSTS_N_INSNS (3), /* cost of movsx */
227 COSTS_N_INSNS (2), /* cost of movzx */
228 15, /* "large" insn */
230 4, /* cost for loading QImode using movzbl */
231 {2, 4, 2}, /* cost of loading integer registers
232 in QImode, HImode and SImode.
233 Relative to reg-reg move (2). */
234 {2, 4, 2}, /* cost of storing integer registers */
235 2, /* cost of reg,reg fld/fst */
236 {8, 8, 8}, /* cost of loading fp registers
237 in SFmode, DFmode and XFmode */
238 {8, 8, 8}, /* cost of storing fp registers
239 in SFmode, DFmode and XFmode */
240 2, /* cost of moving MMX register */
241 {4, 8}, /* cost of loading MMX registers
242 in SImode and DImode */
243 {4, 8}, /* cost of storing MMX registers
244 in SImode and DImode */
245 2, /* cost of moving SSE register */
246 {4, 8, 16}, /* cost of loading SSE registers
247 in SImode, DImode and TImode */
248 {4, 8, 16}, /* cost of storing SSE registers
249 in SImode, DImode and TImode */
250 3, /* MMX or SSE register to integer */
251 0, /* size of l1 cache */
252 0, /* size of l2 cache */
253 0, /* size of prefetch block */
254 0, /* number of parallel prefetches */
256 COSTS_N_INSNS (23), /* cost of FADD and FSUB insns. */
257 COSTS_N_INSNS (27), /* cost of FMUL instruction. */
258 COSTS_N_INSNS (88), /* cost of FDIV instruction. */
259 COSTS_N_INSNS (22), /* cost of FABS instruction. */
260 COSTS_N_INSNS (24), /* cost of FCHS instruction. */
261 COSTS_N_INSNS (122), /* cost of FSQRT instruction. */
264 1, /* scalar_stmt_cost. */
265 1, /* scalar load_cost. */
266 1, /* scalar_store_cost. */
267 1, /* vec_stmt_cost. */
268 1, /* vec_to_scalar_cost. */
269 1, /* scalar_to_vec_cost. */
270 1, /* vec_align_load_cost. */
271 2, /* vec_unalign_load_cost. */
272 1, /* vec_store_cost. */
273 3, /* cond_taken_branch_cost. */
274 1, /* cond_not_taken_branch_cost. */
277 static stringop_algs i486_memcpy[2] = {
278 {rep_prefix_4_byte, {{-1, rep_prefix_4_byte, false}}},
279 DUMMY_STRINGOP_ALGS};
280 static stringop_algs i486_memset[2] = {
281 {rep_prefix_4_byte, {{-1, rep_prefix_4_byte, false}}},
282 DUMMY_STRINGOP_ALGS};
285 struct processor_costs i486_cost = { /* 486 specific costs */
286 COSTS_N_INSNS (1), /* cost of an add instruction */
287 COSTS_N_INSNS (1), /* cost of a lea instruction */
288 COSTS_N_INSNS (3), /* variable shift costs */
289 COSTS_N_INSNS (2), /* constant shift costs */
290 {COSTS_N_INSNS (12), /* cost of starting multiply for QI */
291 COSTS_N_INSNS (12), /* HI */
292 COSTS_N_INSNS (12), /* SI */
293 COSTS_N_INSNS (12), /* DI */
294 COSTS_N_INSNS (12)}, /* other */
295 1, /* cost of multiply per each bit set */
296 {COSTS_N_INSNS (40), /* cost of a divide/mod for QI */
297 COSTS_N_INSNS (40), /* HI */
298 COSTS_N_INSNS (40), /* SI */
299 COSTS_N_INSNS (40), /* DI */
300 COSTS_N_INSNS (40)}, /* other */
301 COSTS_N_INSNS (3), /* cost of movsx */
302 COSTS_N_INSNS (2), /* cost of movzx */
303 15, /* "large" insn */
305 4, /* cost for loading QImode using movzbl */
306 {2, 4, 2}, /* cost of loading integer registers
307 in QImode, HImode and SImode.
308 Relative to reg-reg move (2). */
309 {2, 4, 2}, /* cost of storing integer registers */
310 2, /* cost of reg,reg fld/fst */
311 {8, 8, 8}, /* cost of loading fp registers
312 in SFmode, DFmode and XFmode */
313 {8, 8, 8}, /* cost of storing fp registers
314 in SFmode, DFmode and XFmode */
315 2, /* cost of moving MMX register */
316 {4, 8}, /* cost of loading MMX registers
317 in SImode and DImode */
318 {4, 8}, /* cost of storing MMX registers
319 in SImode and DImode */
320 2, /* cost of moving SSE register */
321 {4, 8, 16}, /* cost of loading SSE registers
322 in SImode, DImode and TImode */
323 {4, 8, 16}, /* cost of storing SSE registers
324 in SImode, DImode and TImode */
325 3, /* MMX or SSE register to integer */
326 4, /* size of l1 cache. 486 has 8kB cache
327 shared for code and data, so 4kB is
328 not really precise. */
329 4, /* size of l2 cache */
330 0, /* size of prefetch block */
331 0, /* number of parallel prefetches */
333 COSTS_N_INSNS (8), /* cost of FADD and FSUB insns. */
334 COSTS_N_INSNS (16), /* cost of FMUL instruction. */
335 COSTS_N_INSNS (73), /* cost of FDIV instruction. */
336 COSTS_N_INSNS (3), /* cost of FABS instruction. */
337 COSTS_N_INSNS (3), /* cost of FCHS instruction. */
338 COSTS_N_INSNS (83), /* cost of FSQRT instruction. */
341 1, /* scalar_stmt_cost. */
342 1, /* scalar load_cost. */
343 1, /* scalar_store_cost. */
344 1, /* vec_stmt_cost. */
345 1, /* vec_to_scalar_cost. */
346 1, /* scalar_to_vec_cost. */
347 1, /* vec_align_load_cost. */
348 2, /* vec_unalign_load_cost. */
349 1, /* vec_store_cost. */
350 3, /* cond_taken_branch_cost. */
351 1, /* cond_not_taken_branch_cost. */
354 static stringop_algs pentium_memcpy[2] = {
355 {libcall, {{256, rep_prefix_4_byte, false}, {-1, libcall, false}}},
356 DUMMY_STRINGOP_ALGS};
357 static stringop_algs pentium_memset[2] = {
358 {libcall, {{-1, rep_prefix_4_byte, false}}},
359 DUMMY_STRINGOP_ALGS};
362 struct processor_costs pentium_cost = {
363 COSTS_N_INSNS (1), /* cost of an add instruction */
364 COSTS_N_INSNS (1), /* cost of a lea instruction */
365 COSTS_N_INSNS (4), /* variable shift costs */
366 COSTS_N_INSNS (1), /* constant shift costs */
367 {COSTS_N_INSNS (11), /* cost of starting multiply for QI */
368 COSTS_N_INSNS (11), /* HI */
369 COSTS_N_INSNS (11), /* SI */
370 COSTS_N_INSNS (11), /* DI */
371 COSTS_N_INSNS (11)}, /* other */
372 0, /* cost of multiply per each bit set */
373 {COSTS_N_INSNS (25), /* cost of a divide/mod for QI */
374 COSTS_N_INSNS (25), /* HI */
375 COSTS_N_INSNS (25), /* SI */
376 COSTS_N_INSNS (25), /* DI */
377 COSTS_N_INSNS (25)}, /* other */
378 COSTS_N_INSNS (3), /* cost of movsx */
379 COSTS_N_INSNS (2), /* cost of movzx */
380 8, /* "large" insn */
382 6, /* cost for loading QImode using movzbl */
383 {2, 4, 2}, /* cost of loading integer registers
384 in QImode, HImode and SImode.
385 Relative to reg-reg move (2). */
386 {2, 4, 2}, /* cost of storing integer registers */
387 2, /* cost of reg,reg fld/fst */
388 {2, 2, 6}, /* cost of loading fp registers
389 in SFmode, DFmode and XFmode */
390 {4, 4, 6}, /* cost of storing fp registers
391 in SFmode, DFmode and XFmode */
392 8, /* cost of moving MMX register */
393 {8, 8}, /* cost of loading MMX registers
394 in SImode and DImode */
395 {8, 8}, /* cost of storing MMX registers
396 in SImode and DImode */
397 2, /* cost of moving SSE register */
398 {4, 8, 16}, /* cost of loading SSE registers
399 in SImode, DImode and TImode */
400 {4, 8, 16}, /* cost of storing SSE registers
401 in SImode, DImode and TImode */
402 3, /* MMX or SSE register to integer */
403 8, /* size of l1 cache. */
404 8, /* size of l2 cache */
405 0, /* size of prefetch block */
406 0, /* number of parallel prefetches */
408 COSTS_N_INSNS (3), /* cost of FADD and FSUB insns. */
409 COSTS_N_INSNS (3), /* cost of FMUL instruction. */
410 COSTS_N_INSNS (39), /* cost of FDIV instruction. */
411 COSTS_N_INSNS (1), /* cost of FABS instruction. */
412 COSTS_N_INSNS (1), /* cost of FCHS instruction. */
413 COSTS_N_INSNS (70), /* cost of FSQRT instruction. */
416 1, /* scalar_stmt_cost. */
417 1, /* scalar load_cost. */
418 1, /* scalar_store_cost. */
419 1, /* vec_stmt_cost. */
420 1, /* vec_to_scalar_cost. */
421 1, /* scalar_to_vec_cost. */
422 1, /* vec_align_load_cost. */
423 2, /* vec_unalign_load_cost. */
424 1, /* vec_store_cost. */
425 3, /* cond_taken_branch_cost. */
426 1, /* cond_not_taken_branch_cost. */
429 /* PentiumPro has optimized rep instructions for blocks aligned by 8 bytes
430 (we ensure the alignment). For small blocks inline loop is still a
431 noticeable win, for bigger blocks either rep movsl or rep movsb is
432 way to go. Rep movsb has apparently more expensive startup time in CPU,
433 but after 4K the difference is down in the noise. */
434 static stringop_algs pentiumpro_memcpy[2] = {
435 {rep_prefix_4_byte, {{128, loop, false}, {1024, unrolled_loop, false},
436 {8192, rep_prefix_4_byte, false},
437 {-1, rep_prefix_1_byte, false}}},
438 DUMMY_STRINGOP_ALGS};
439 static stringop_algs pentiumpro_memset[2] = {
440 {rep_prefix_4_byte, {{1024, unrolled_loop, false},
441 {8192, rep_prefix_4_byte, false},
442 {-1, libcall, false}}},
443 DUMMY_STRINGOP_ALGS};
445 struct processor_costs pentiumpro_cost = {
446 COSTS_N_INSNS (1), /* cost of an add instruction */
447 COSTS_N_INSNS (1), /* cost of a lea instruction */
448 COSTS_N_INSNS (1), /* variable shift costs */
449 COSTS_N_INSNS (1), /* constant shift costs */
450 {COSTS_N_INSNS (4), /* cost of starting multiply for QI */
451 COSTS_N_INSNS (4), /* HI */
452 COSTS_N_INSNS (4), /* SI */
453 COSTS_N_INSNS (4), /* DI */
454 COSTS_N_INSNS (4)}, /* other */
455 0, /* cost of multiply per each bit set */
456 {COSTS_N_INSNS (17), /* cost of a divide/mod for QI */
457 COSTS_N_INSNS (17), /* HI */
458 COSTS_N_INSNS (17), /* SI */
459 COSTS_N_INSNS (17), /* DI */
460 COSTS_N_INSNS (17)}, /* other */
461 COSTS_N_INSNS (1), /* cost of movsx */
462 COSTS_N_INSNS (1), /* cost of movzx */
463 8, /* "large" insn */
465 2, /* cost for loading QImode using movzbl */
466 {4, 4, 4}, /* cost of loading integer registers
467 in QImode, HImode and SImode.
468 Relative to reg-reg move (2). */
469 {2, 2, 2}, /* cost of storing integer registers */
470 2, /* cost of reg,reg fld/fst */
471 {2, 2, 6}, /* cost of loading fp registers
472 in SFmode, DFmode and XFmode */
473 {4, 4, 6}, /* cost of storing fp registers
474 in SFmode, DFmode and XFmode */
475 2, /* cost of moving MMX register */
476 {2, 2}, /* cost of loading MMX registers
477 in SImode and DImode */
478 {2, 2}, /* cost of storing MMX registers
479 in SImode and DImode */
480 2, /* cost of moving SSE register */
481 {2, 2, 8}, /* cost of loading SSE registers
482 in SImode, DImode and TImode */
483 {2, 2, 8}, /* cost of storing SSE registers
484 in SImode, DImode and TImode */
485 3, /* MMX or SSE register to integer */
486 8, /* size of l1 cache. */
487 256, /* size of l2 cache */
488 32, /* size of prefetch block */
489 6, /* number of parallel prefetches */
491 COSTS_N_INSNS (3), /* cost of FADD and FSUB insns. */
492 COSTS_N_INSNS (5), /* cost of FMUL instruction. */
493 COSTS_N_INSNS (56), /* cost of FDIV instruction. */
494 COSTS_N_INSNS (2), /* cost of FABS instruction. */
495 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
496 COSTS_N_INSNS (56), /* cost of FSQRT instruction. */
499 1, /* scalar_stmt_cost. */
500 1, /* scalar load_cost. */
501 1, /* scalar_store_cost. */
502 1, /* vec_stmt_cost. */
503 1, /* vec_to_scalar_cost. */
504 1, /* scalar_to_vec_cost. */
505 1, /* vec_align_load_cost. */
506 2, /* vec_unalign_load_cost. */
507 1, /* vec_store_cost. */
508 3, /* cond_taken_branch_cost. */
509 1, /* cond_not_taken_branch_cost. */
512 static stringop_algs geode_memcpy[2] = {
513 {libcall, {{256, rep_prefix_4_byte, false}, {-1, libcall, false}}},
514 DUMMY_STRINGOP_ALGS};
515 static stringop_algs geode_memset[2] = {
516 {libcall, {{256, rep_prefix_4_byte, false}, {-1, libcall, false}}},
517 DUMMY_STRINGOP_ALGS};
519 struct processor_costs geode_cost = {
520 COSTS_N_INSNS (1), /* cost of an add instruction */
521 COSTS_N_INSNS (1), /* cost of a lea instruction */
522 COSTS_N_INSNS (2), /* variable shift costs */
523 COSTS_N_INSNS (1), /* constant shift costs */
524 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
525 COSTS_N_INSNS (4), /* HI */
526 COSTS_N_INSNS (7), /* SI */
527 COSTS_N_INSNS (7), /* DI */
528 COSTS_N_INSNS (7)}, /* other */
529 0, /* cost of multiply per each bit set */
530 {COSTS_N_INSNS (15), /* cost of a divide/mod for QI */
531 COSTS_N_INSNS (23), /* HI */
532 COSTS_N_INSNS (39), /* SI */
533 COSTS_N_INSNS (39), /* DI */
534 COSTS_N_INSNS (39)}, /* other */
535 COSTS_N_INSNS (1), /* cost of movsx */
536 COSTS_N_INSNS (1), /* cost of movzx */
537 8, /* "large" insn */
539 1, /* cost for loading QImode using movzbl */
540 {1, 1, 1}, /* cost of loading integer registers
541 in QImode, HImode and SImode.
542 Relative to reg-reg move (2). */
543 {1, 1, 1}, /* cost of storing integer registers */
544 1, /* cost of reg,reg fld/fst */
545 {1, 1, 1}, /* cost of loading fp registers
546 in SFmode, DFmode and XFmode */
547 {4, 6, 6}, /* cost of storing fp registers
548 in SFmode, DFmode and XFmode */
550 1, /* cost of moving MMX register */
551 {1, 1}, /* cost of loading MMX registers
552 in SImode and DImode */
553 {1, 1}, /* cost of storing MMX registers
554 in SImode and DImode */
555 1, /* cost of moving SSE register */
556 {1, 1, 1}, /* cost of loading SSE registers
557 in SImode, DImode and TImode */
558 {1, 1, 1}, /* cost of storing SSE registers
559 in SImode, DImode and TImode */
560 1, /* MMX or SSE register to integer */
561 64, /* size of l1 cache. */
562 128, /* size of l2 cache. */
563 32, /* size of prefetch block */
564 1, /* number of parallel prefetches */
566 COSTS_N_INSNS (6), /* cost of FADD and FSUB insns. */
567 COSTS_N_INSNS (11), /* cost of FMUL instruction. */
568 COSTS_N_INSNS (47), /* cost of FDIV instruction. */
569 COSTS_N_INSNS (1), /* cost of FABS instruction. */
570 COSTS_N_INSNS (1), /* cost of FCHS instruction. */
571 COSTS_N_INSNS (54), /* cost of FSQRT instruction. */
574 1, /* scalar_stmt_cost. */
575 1, /* scalar load_cost. */
576 1, /* scalar_store_cost. */
577 1, /* vec_stmt_cost. */
578 1, /* vec_to_scalar_cost. */
579 1, /* scalar_to_vec_cost. */
580 1, /* vec_align_load_cost. */
581 2, /* vec_unalign_load_cost. */
582 1, /* vec_store_cost. */
583 3, /* cond_taken_branch_cost. */
584 1, /* cond_not_taken_branch_cost. */
587 static stringop_algs k6_memcpy[2] = {
588 {libcall, {{256, rep_prefix_4_byte, false}, {-1, libcall, false}}},
589 DUMMY_STRINGOP_ALGS};
590 static stringop_algs k6_memset[2] = {
591 {libcall, {{256, rep_prefix_4_byte, false}, {-1, libcall, false}}},
592 DUMMY_STRINGOP_ALGS};
594 struct processor_costs k6_cost = {
595 COSTS_N_INSNS (1), /* cost of an add instruction */
596 COSTS_N_INSNS (2), /* cost of a lea instruction */
597 COSTS_N_INSNS (1), /* variable shift costs */
598 COSTS_N_INSNS (1), /* constant shift costs */
599 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
600 COSTS_N_INSNS (3), /* HI */
601 COSTS_N_INSNS (3), /* SI */
602 COSTS_N_INSNS (3), /* DI */
603 COSTS_N_INSNS (3)}, /* other */
604 0, /* cost of multiply per each bit set */
605 {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */
606 COSTS_N_INSNS (18), /* HI */
607 COSTS_N_INSNS (18), /* SI */
608 COSTS_N_INSNS (18), /* DI */
609 COSTS_N_INSNS (18)}, /* other */
610 COSTS_N_INSNS (2), /* cost of movsx */
611 COSTS_N_INSNS (2), /* cost of movzx */
612 8, /* "large" insn */
614 3, /* cost for loading QImode using movzbl */
615 {4, 5, 4}, /* cost of loading integer registers
616 in QImode, HImode and SImode.
617 Relative to reg-reg move (2). */
618 {2, 3, 2}, /* cost of storing integer registers */
619 4, /* cost of reg,reg fld/fst */
620 {6, 6, 6}, /* cost of loading fp registers
621 in SFmode, DFmode and XFmode */
622 {4, 4, 4}, /* cost of storing fp registers
623 in SFmode, DFmode and XFmode */
624 2, /* cost of moving MMX register */
625 {2, 2}, /* cost of loading MMX registers
626 in SImode and DImode */
627 {2, 2}, /* cost of storing MMX registers
628 in SImode and DImode */
629 2, /* cost of moving SSE register */
630 {2, 2, 8}, /* cost of loading SSE registers
631 in SImode, DImode and TImode */
632 {2, 2, 8}, /* cost of storing SSE registers
633 in SImode, DImode and TImode */
634 6, /* MMX or SSE register to integer */
635 32, /* size of l1 cache. */
636 32, /* size of l2 cache. Some models
637 have integrated l2 cache, but
638 optimizing for k6 is not important
639 enough to worry about that. */
640 32, /* size of prefetch block */
641 1, /* number of parallel prefetches */
643 COSTS_N_INSNS (2), /* cost of FADD and FSUB insns. */
644 COSTS_N_INSNS (2), /* cost of FMUL instruction. */
645 COSTS_N_INSNS (56), /* cost of FDIV instruction. */
646 COSTS_N_INSNS (2), /* cost of FABS instruction. */
647 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
648 COSTS_N_INSNS (56), /* cost of FSQRT instruction. */
651 1, /* scalar_stmt_cost. */
652 1, /* scalar load_cost. */
653 1, /* scalar_store_cost. */
654 1, /* vec_stmt_cost. */
655 1, /* vec_to_scalar_cost. */
656 1, /* scalar_to_vec_cost. */
657 1, /* vec_align_load_cost. */
658 2, /* vec_unalign_load_cost. */
659 1, /* vec_store_cost. */
660 3, /* cond_taken_branch_cost. */
661 1, /* cond_not_taken_branch_cost. */
664 /* For some reason, Athlon deals better with REP prefix (relative to loops)
665 compared to K8. Alignment becomes important after 8 bytes for memcpy and
666 128 bytes for memset. */
667 static stringop_algs athlon_memcpy[2] = {
668 {libcall, {{2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
669 DUMMY_STRINGOP_ALGS};
670 static stringop_algs athlon_memset[2] = {
671 {libcall, {{2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
672 DUMMY_STRINGOP_ALGS};
674 struct processor_costs athlon_cost = {
675 COSTS_N_INSNS (1), /* cost of an add instruction */
676 COSTS_N_INSNS (2), /* cost of a lea instruction */
677 COSTS_N_INSNS (1), /* variable shift costs */
678 COSTS_N_INSNS (1), /* constant shift costs */
679 {COSTS_N_INSNS (5), /* cost of starting multiply for QI */
680 COSTS_N_INSNS (5), /* HI */
681 COSTS_N_INSNS (5), /* SI */
682 COSTS_N_INSNS (5), /* DI */
683 COSTS_N_INSNS (5)}, /* other */
684 0, /* cost of multiply per each bit set */
685 {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */
686 COSTS_N_INSNS (26), /* HI */
687 COSTS_N_INSNS (42), /* SI */
688 COSTS_N_INSNS (74), /* DI */
689 COSTS_N_INSNS (74)}, /* other */
690 COSTS_N_INSNS (1), /* cost of movsx */
691 COSTS_N_INSNS (1), /* cost of movzx */
692 8, /* "large" insn */
694 4, /* cost for loading QImode using movzbl */
695 {3, 4, 3}, /* cost of loading integer registers
696 in QImode, HImode and SImode.
697 Relative to reg-reg move (2). */
698 {3, 4, 3}, /* cost of storing integer registers */
699 4, /* cost of reg,reg fld/fst */
700 {4, 4, 12}, /* cost of loading fp registers
701 in SFmode, DFmode and XFmode */
702 {6, 6, 8}, /* cost of storing fp registers
703 in SFmode, DFmode and XFmode */
704 2, /* cost of moving MMX register */
705 {4, 4}, /* cost of loading MMX registers
706 in SImode and DImode */
707 {4, 4}, /* cost of storing MMX registers
708 in SImode and DImode */
709 2, /* cost of moving SSE register */
710 {4, 4, 6}, /* cost of loading SSE registers
711 in SImode, DImode and TImode */
712 {4, 4, 5}, /* cost of storing SSE registers
713 in SImode, DImode and TImode */
714 5, /* MMX or SSE register to integer */
715 64, /* size of l1 cache. */
716 256, /* size of l2 cache. */
717 64, /* size of prefetch block */
718 6, /* number of parallel prefetches */
720 COSTS_N_INSNS (4), /* cost of FADD and FSUB insns. */
721 COSTS_N_INSNS (4), /* cost of FMUL instruction. */
722 COSTS_N_INSNS (24), /* cost of FDIV instruction. */
723 COSTS_N_INSNS (2), /* cost of FABS instruction. */
724 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
725 COSTS_N_INSNS (35), /* cost of FSQRT instruction. */
728 1, /* scalar_stmt_cost. */
729 1, /* scalar load_cost. */
730 1, /* scalar_store_cost. */
731 1, /* vec_stmt_cost. */
732 1, /* vec_to_scalar_cost. */
733 1, /* scalar_to_vec_cost. */
734 1, /* vec_align_load_cost. */
735 2, /* vec_unalign_load_cost. */
736 1, /* vec_store_cost. */
737 3, /* cond_taken_branch_cost. */
738 1, /* cond_not_taken_branch_cost. */
741 /* K8 has optimized REP instruction for medium sized blocks, but for very
742 small blocks it is better to use loop. For large blocks, libcall can
743 do nontemporary accesses and beat inline considerably. */
744 static stringop_algs k8_memcpy[2] = {
745 {libcall, {{6, loop, false}, {14, unrolled_loop, false},
746 {-1, rep_prefix_4_byte, false}}},
747 {libcall, {{16, loop, false}, {8192, rep_prefix_8_byte, false},
748 {-1, libcall, false}}}};
749 static stringop_algs k8_memset[2] = {
750 {libcall, {{8, loop, false}, {24, unrolled_loop, false},
751 {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
752 {libcall, {{48, unrolled_loop, false},
753 {8192, rep_prefix_8_byte, false}, {-1, libcall, false}}}};
755 struct processor_costs k8_cost = {
756 COSTS_N_INSNS (1), /* cost of an add instruction */
757 COSTS_N_INSNS (2), /* cost of a lea instruction */
758 COSTS_N_INSNS (1), /* variable shift costs */
759 COSTS_N_INSNS (1), /* constant shift costs */
760 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
761 COSTS_N_INSNS (4), /* HI */
762 COSTS_N_INSNS (3), /* SI */
763 COSTS_N_INSNS (4), /* DI */
764 COSTS_N_INSNS (5)}, /* other */
765 0, /* cost of multiply per each bit set */
766 {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */
767 COSTS_N_INSNS (26), /* HI */
768 COSTS_N_INSNS (42), /* SI */
769 COSTS_N_INSNS (74), /* DI */
770 COSTS_N_INSNS (74)}, /* other */
771 COSTS_N_INSNS (1), /* cost of movsx */
772 COSTS_N_INSNS (1), /* cost of movzx */
773 8, /* "large" insn */
775 4, /* cost for loading QImode using movzbl */
776 {3, 4, 3}, /* cost of loading integer registers
777 in QImode, HImode and SImode.
778 Relative to reg-reg move (2). */
779 {3, 4, 3}, /* cost of storing integer registers */
780 4, /* cost of reg,reg fld/fst */
781 {4, 4, 12}, /* cost of loading fp registers
782 in SFmode, DFmode and XFmode */
783 {6, 6, 8}, /* cost of storing fp registers
784 in SFmode, DFmode and XFmode */
785 2, /* cost of moving MMX register */
786 {3, 3}, /* cost of loading MMX registers
787 in SImode and DImode */
788 {4, 4}, /* cost of storing MMX registers
789 in SImode and DImode */
790 2, /* cost of moving SSE register */
791 {4, 3, 6}, /* cost of loading SSE registers
792 in SImode, DImode and TImode */
793 {4, 4, 5}, /* cost of storing SSE registers
794 in SImode, DImode and TImode */
795 5, /* MMX or SSE register to integer */
796 64, /* size of l1 cache. */
797 512, /* size of l2 cache. */
798 64, /* size of prefetch block */
799 /* New AMD processors never drop prefetches; if they cannot be performed
800 immediately, they are queued. We set number of simultaneous prefetches
801 to a large constant to reflect this (it probably is not a good idea not
802 to limit number of prefetches at all, as their execution also takes some
804 100, /* number of parallel prefetches */
806 COSTS_N_INSNS (4), /* cost of FADD and FSUB insns. */
807 COSTS_N_INSNS (4), /* cost of FMUL instruction. */
808 COSTS_N_INSNS (19), /* cost of FDIV instruction. */
809 COSTS_N_INSNS (2), /* cost of FABS instruction. */
810 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
811 COSTS_N_INSNS (35), /* cost of FSQRT instruction. */
815 4, /* scalar_stmt_cost. */
816 2, /* scalar load_cost. */
817 2, /* scalar_store_cost. */
818 5, /* vec_stmt_cost. */
819 0, /* vec_to_scalar_cost. */
820 2, /* scalar_to_vec_cost. */
821 2, /* vec_align_load_cost. */
822 3, /* vec_unalign_load_cost. */
823 3, /* vec_store_cost. */
824 3, /* cond_taken_branch_cost. */
825 2, /* cond_not_taken_branch_cost. */
828 /* AMDFAM10 has optimized REP instruction for medium sized blocks, but for
829 very small blocks it is better to use loop. For large blocks, libcall can
830 do nontemporary accesses and beat inline considerably. */
831 static stringop_algs amdfam10_memcpy[2] = {
832 {libcall, {{6, loop, false}, {14, unrolled_loop, false},
833 {-1, rep_prefix_4_byte, false}}},
834 {libcall, {{16, loop, false}, {8192, rep_prefix_8_byte, false},
835 {-1, libcall, false}}}};
836 static stringop_algs amdfam10_memset[2] = {
837 {libcall, {{8, loop, false}, {24, unrolled_loop, false},
838 {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
839 {libcall, {{48, unrolled_loop, false}, {8192, rep_prefix_8_byte, false},
840 {-1, libcall, false}}}};
841 struct processor_costs amdfam10_cost = {
842 COSTS_N_INSNS (1), /* cost of an add instruction */
843 COSTS_N_INSNS (2), /* cost of a lea instruction */
844 COSTS_N_INSNS (1), /* variable shift costs */
845 COSTS_N_INSNS (1), /* constant shift costs */
846 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
847 COSTS_N_INSNS (4), /* HI */
848 COSTS_N_INSNS (3), /* SI */
849 COSTS_N_INSNS (4), /* DI */
850 COSTS_N_INSNS (5)}, /* other */
851 0, /* cost of multiply per each bit set */
852 {COSTS_N_INSNS (19), /* cost of a divide/mod for QI */
853 COSTS_N_INSNS (35), /* HI */
854 COSTS_N_INSNS (51), /* SI */
855 COSTS_N_INSNS (83), /* DI */
856 COSTS_N_INSNS (83)}, /* other */
857 COSTS_N_INSNS (1), /* cost of movsx */
858 COSTS_N_INSNS (1), /* cost of movzx */
859 8, /* "large" insn */
861 4, /* cost for loading QImode using movzbl */
862 {3, 4, 3}, /* cost of loading integer registers
863 in QImode, HImode and SImode.
864 Relative to reg-reg move (2). */
865 {3, 4, 3}, /* cost of storing integer registers */
866 4, /* cost of reg,reg fld/fst */
867 {4, 4, 12}, /* cost of loading fp registers
868 in SFmode, DFmode and XFmode */
869 {6, 6, 8}, /* cost of storing fp registers
870 in SFmode, DFmode and XFmode */
871 2, /* cost of moving MMX register */
872 {3, 3}, /* cost of loading MMX registers
873 in SImode and DImode */
874 {4, 4}, /* cost of storing MMX registers
875 in SImode and DImode */
876 2, /* cost of moving SSE register */
877 {4, 4, 3}, /* cost of loading SSE registers
878 in SImode, DImode and TImode */
879 {4, 4, 5}, /* cost of storing SSE registers
880 in SImode, DImode and TImode */
881 3, /* MMX or SSE register to integer */
883 MOVD reg64, xmmreg Double FSTORE 4
884 MOVD reg32, xmmreg Double FSTORE 4
886 MOVD reg64, xmmreg Double FADD 3
888 MOVD reg32, xmmreg Double FADD 3
890 64, /* size of l1 cache. */
891 512, /* size of l2 cache. */
892 64, /* size of prefetch block */
893 /* New AMD processors never drop prefetches; if they cannot be performed
894 immediately, they are queued. We set number of simultaneous prefetches
895 to a large constant to reflect this (it probably is not a good idea not
896 to limit number of prefetches at all, as their execution also takes some
898 100, /* number of parallel prefetches */
900 COSTS_N_INSNS (4), /* cost of FADD and FSUB insns. */
901 COSTS_N_INSNS (4), /* cost of FMUL instruction. */
902 COSTS_N_INSNS (19), /* cost of FDIV instruction. */
903 COSTS_N_INSNS (2), /* cost of FABS instruction. */
904 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
905 COSTS_N_INSNS (35), /* cost of FSQRT instruction. */
909 4, /* scalar_stmt_cost. */
910 2, /* scalar load_cost. */
911 2, /* scalar_store_cost. */
912 6, /* vec_stmt_cost. */
913 0, /* vec_to_scalar_cost. */
914 2, /* scalar_to_vec_cost. */
915 2, /* vec_align_load_cost. */
916 2, /* vec_unalign_load_cost. */
917 2, /* vec_store_cost. */
918 2, /* cond_taken_branch_cost. */
919 1, /* cond_not_taken_branch_cost. */
922 /* BDVER1 has optimized REP instruction for medium sized blocks, but for
923 very small blocks it is better to use loop. For large blocks, libcall
924 can do nontemporary accesses and beat inline considerably. */
925 static stringop_algs bdver1_memcpy[2] = {
926 {libcall, {{6, loop, false}, {14, unrolled_loop, false},
927 {-1, rep_prefix_4_byte, false}}},
928 {libcall, {{16, loop, false}, {8192, rep_prefix_8_byte, false},
929 {-1, libcall, false}}}};
930 static stringop_algs bdver1_memset[2] = {
931 {libcall, {{8, loop, false}, {24, unrolled_loop, false},
932 {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
933 {libcall, {{48, unrolled_loop, false}, {8192, rep_prefix_8_byte, false},
934 {-1, libcall, false}}}};
936 const struct processor_costs bdver1_cost = {
937 COSTS_N_INSNS (1), /* cost of an add instruction */
938 COSTS_N_INSNS (1), /* cost of a lea instruction */
939 COSTS_N_INSNS (1), /* variable shift costs */
940 COSTS_N_INSNS (1), /* constant shift costs */
941 {COSTS_N_INSNS (4), /* cost of starting multiply for QI */
942 COSTS_N_INSNS (4), /* HI */
943 COSTS_N_INSNS (4), /* SI */
944 COSTS_N_INSNS (6), /* DI */
945 COSTS_N_INSNS (6)}, /* other */
946 0, /* cost of multiply per each bit set */
947 {COSTS_N_INSNS (19), /* cost of a divide/mod for QI */
948 COSTS_N_INSNS (35), /* HI */
949 COSTS_N_INSNS (51), /* SI */
950 COSTS_N_INSNS (83), /* DI */
951 COSTS_N_INSNS (83)}, /* other */
952 COSTS_N_INSNS (1), /* cost of movsx */
953 COSTS_N_INSNS (1), /* cost of movzx */
954 8, /* "large" insn */
956 4, /* cost for loading QImode using movzbl */
957 {5, 5, 4}, /* cost of loading integer registers
958 in QImode, HImode and SImode.
959 Relative to reg-reg move (2). */
960 {4, 4, 4}, /* cost of storing integer registers */
961 2, /* cost of reg,reg fld/fst */
962 {5, 5, 12}, /* cost of loading fp registers
963 in SFmode, DFmode and XFmode */
964 {4, 4, 8}, /* cost of storing fp registers
965 in SFmode, DFmode and XFmode */
966 2, /* cost of moving MMX register */
967 {4, 4}, /* cost of loading MMX registers
968 in SImode and DImode */
969 {4, 4}, /* cost of storing MMX registers
970 in SImode and DImode */
971 2, /* cost of moving SSE register */
972 {4, 4, 4}, /* cost of loading SSE registers
973 in SImode, DImode and TImode */
974 {4, 4, 4}, /* cost of storing SSE registers
975 in SImode, DImode and TImode */
976 2, /* MMX or SSE register to integer */
978 MOVD reg64, xmmreg Double FSTORE 4
979 MOVD reg32, xmmreg Double FSTORE 4
981 MOVD reg64, xmmreg Double FADD 3
983 MOVD reg32, xmmreg Double FADD 3
985 16, /* size of l1 cache. */
986 2048, /* size of l2 cache. */
987 64, /* size of prefetch block */
988 /* New AMD processors never drop prefetches; if they cannot be performed
989 immediately, they are queued. We set number of simultaneous prefetches
990 to a large constant to reflect this (it probably is not a good idea not
991 to limit number of prefetches at all, as their execution also takes some
993 100, /* number of parallel prefetches */
995 COSTS_N_INSNS (6), /* cost of FADD and FSUB insns. */
996 COSTS_N_INSNS (6), /* cost of FMUL instruction. */
997 COSTS_N_INSNS (42), /* cost of FDIV instruction. */
998 COSTS_N_INSNS (2), /* cost of FABS instruction. */
999 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
1000 COSTS_N_INSNS (52), /* cost of FSQRT instruction. */
1004 6, /* scalar_stmt_cost. */
1005 4, /* scalar load_cost. */
1006 4, /* scalar_store_cost. */
1007 6, /* vec_stmt_cost. */
1008 0, /* vec_to_scalar_cost. */
1009 2, /* scalar_to_vec_cost. */
1010 4, /* vec_align_load_cost. */
1011 4, /* vec_unalign_load_cost. */
1012 4, /* vec_store_cost. */
1013 4, /* cond_taken_branch_cost. */
1014 2, /* cond_not_taken_branch_cost. */
1017 /* BDVER2 has optimized REP instruction for medium sized blocks, but for
1018 very small blocks it is better to use loop. For large blocks, libcall
1019 can do nontemporary accesses and beat inline considerably. */
1021 static stringop_algs bdver2_memcpy[2] = {
1022 {libcall, {{6, loop, false}, {14, unrolled_loop, false},
1023 {-1, rep_prefix_4_byte, false}}},
1024 {libcall, {{16, loop, false}, {8192, rep_prefix_8_byte, false},
1025 {-1, libcall, false}}}};
1026 static stringop_algs bdver2_memset[2] = {
1027 {libcall, {{8, loop, false}, {24, unrolled_loop, false},
1028 {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
1029 {libcall, {{48, unrolled_loop, false}, {8192, rep_prefix_8_byte, false},
1030 {-1, libcall, false}}}};
1032 const struct processor_costs bdver2_cost = {
1033 COSTS_N_INSNS (1), /* cost of an add instruction */
1034 COSTS_N_INSNS (1), /* cost of a lea instruction */
1035 COSTS_N_INSNS (1), /* variable shift costs */
1036 COSTS_N_INSNS (1), /* constant shift costs */
1037 {COSTS_N_INSNS (4), /* cost of starting multiply for QI */
1038 COSTS_N_INSNS (4), /* HI */
1039 COSTS_N_INSNS (4), /* SI */
1040 COSTS_N_INSNS (6), /* DI */
1041 COSTS_N_INSNS (6)}, /* other */
1042 0, /* cost of multiply per each bit set */
1043 {COSTS_N_INSNS (19), /* cost of a divide/mod for QI */
1044 COSTS_N_INSNS (35), /* HI */
1045 COSTS_N_INSNS (51), /* SI */
1046 COSTS_N_INSNS (83), /* DI */
1047 COSTS_N_INSNS (83)}, /* other */
1048 COSTS_N_INSNS (1), /* cost of movsx */
1049 COSTS_N_INSNS (1), /* cost of movzx */
1050 8, /* "large" insn */
1052 4, /* cost for loading QImode using movzbl */
1053 {5, 5, 4}, /* cost of loading integer registers
1054 in QImode, HImode and SImode.
1055 Relative to reg-reg move (2). */
1056 {4, 4, 4}, /* cost of storing integer registers */
1057 2, /* cost of reg,reg fld/fst */
1058 {5, 5, 12}, /* cost of loading fp registers
1059 in SFmode, DFmode and XFmode */
1060 {4, 4, 8}, /* cost of storing fp registers
1061 in SFmode, DFmode and XFmode */
1062 2, /* cost of moving MMX register */
1063 {4, 4}, /* cost of loading MMX registers
1064 in SImode and DImode */
1065 {4, 4}, /* cost of storing MMX registers
1066 in SImode and DImode */
1067 2, /* cost of moving SSE register */
1068 {4, 4, 4}, /* cost of loading SSE registers
1069 in SImode, DImode and TImode */
1070 {4, 4, 4}, /* cost of storing SSE registers
1071 in SImode, DImode and TImode */
1072 2, /* MMX or SSE register to integer */
1074 MOVD reg64, xmmreg Double FSTORE 4
1075 MOVD reg32, xmmreg Double FSTORE 4
1077 MOVD reg64, xmmreg Double FADD 3
1079 MOVD reg32, xmmreg Double FADD 3
1081 16, /* size of l1 cache. */
1082 2048, /* size of l2 cache. */
1083 64, /* size of prefetch block */
1084 /* New AMD processors never drop prefetches; if they cannot be performed
1085 immediately, they are queued. We set number of simultaneous prefetches
1086 to a large constant to reflect this (it probably is not a good idea not
1087 to limit number of prefetches at all, as their execution also takes some
1089 100, /* number of parallel prefetches */
1090 2, /* Branch cost */
1091 COSTS_N_INSNS (6), /* cost of FADD and FSUB insns. */
1092 COSTS_N_INSNS (6), /* cost of FMUL instruction. */
1093 COSTS_N_INSNS (42), /* cost of FDIV instruction. */
1094 COSTS_N_INSNS (2), /* cost of FABS instruction. */
1095 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
1096 COSTS_N_INSNS (52), /* cost of FSQRT instruction. */
1100 6, /* scalar_stmt_cost. */
1101 4, /* scalar load_cost. */
1102 4, /* scalar_store_cost. */
1103 6, /* vec_stmt_cost. */
1104 0, /* vec_to_scalar_cost. */
1105 2, /* scalar_to_vec_cost. */
1106 4, /* vec_align_load_cost. */
1107 4, /* vec_unalign_load_cost. */
1108 4, /* vec_store_cost. */
1109 4, /* cond_taken_branch_cost. */
1110 2, /* cond_not_taken_branch_cost. */
1114 /* BDVER3 has optimized REP instruction for medium sized blocks, but for
1115 very small blocks it is better to use loop. For large blocks, libcall
1116 can do nontemporary accesses and beat inline considerably. */
1117 static stringop_algs bdver3_memcpy[2] = {
1118 {libcall, {{6, loop, false}, {14, unrolled_loop, false},
1119 {-1, rep_prefix_4_byte, false}}},
1120 {libcall, {{16, loop, false}, {8192, rep_prefix_8_byte, false},
1121 {-1, libcall, false}}}};
1122 static stringop_algs bdver3_memset[2] = {
1123 {libcall, {{8, loop, false}, {24, unrolled_loop, false},
1124 {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
1125 {libcall, {{48, unrolled_loop, false}, {8192, rep_prefix_8_byte, false},
1126 {-1, libcall, false}}}};
1127 struct processor_costs bdver3_cost = {
1128 COSTS_N_INSNS (1), /* cost of an add instruction */
1129 COSTS_N_INSNS (1), /* cost of a lea instruction */
1130 COSTS_N_INSNS (1), /* variable shift costs */
1131 COSTS_N_INSNS (1), /* constant shift costs */
1132 {COSTS_N_INSNS (4), /* cost of starting multiply for QI */
1133 COSTS_N_INSNS (4), /* HI */
1134 COSTS_N_INSNS (4), /* SI */
1135 COSTS_N_INSNS (6), /* DI */
1136 COSTS_N_INSNS (6)}, /* other */
1137 0, /* cost of multiply per each bit set */
1138 {COSTS_N_INSNS (19), /* cost of a divide/mod for QI */
1139 COSTS_N_INSNS (35), /* HI */
1140 COSTS_N_INSNS (51), /* SI */
1141 COSTS_N_INSNS (83), /* DI */
1142 COSTS_N_INSNS (83)}, /* other */
1143 COSTS_N_INSNS (1), /* cost of movsx */
1144 COSTS_N_INSNS (1), /* cost of movzx */
1145 8, /* "large" insn */
1147 4, /* cost for loading QImode using movzbl */
1148 {5, 5, 4}, /* cost of loading integer registers
1149 in QImode, HImode and SImode.
1150 Relative to reg-reg move (2). */
1151 {4, 4, 4}, /* cost of storing integer registers */
1152 2, /* cost of reg,reg fld/fst */
1153 {5, 5, 12}, /* cost of loading fp registers
1154 in SFmode, DFmode and XFmode */
1155 {4, 4, 8}, /* cost of storing fp registers
1156 in SFmode, DFmode and XFmode */
1157 2, /* cost of moving MMX register */
1158 {4, 4}, /* cost of loading MMX registers
1159 in SImode and DImode */
1160 {4, 4}, /* cost of storing MMX registers
1161 in SImode and DImode */
1162 2, /* cost of moving SSE register */
1163 {4, 4, 4}, /* cost of loading SSE registers
1164 in SImode, DImode and TImode */
1165 {4, 4, 4}, /* cost of storing SSE registers
1166 in SImode, DImode and TImode */
1167 2, /* MMX or SSE register to integer */
1168 16, /* size of l1 cache. */
1169 2048, /* size of l2 cache. */
1170 64, /* size of prefetch block */
1171 /* New AMD processors never drop prefetches; if they cannot be performed
1172 immediately, they are queued. We set number of simultaneous prefetches
1173 to a large constant to reflect this (it probably is not a good idea not
1174 to limit number of prefetches at all, as their execution also takes some
1176 100, /* number of parallel prefetches */
1177 2, /* Branch cost */
1178 COSTS_N_INSNS (6), /* cost of FADD and FSUB insns. */
1179 COSTS_N_INSNS (6), /* cost of FMUL instruction. */
1180 COSTS_N_INSNS (42), /* cost of FDIV instruction. */
1181 COSTS_N_INSNS (2), /* cost of FABS instruction. */
1182 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
1183 COSTS_N_INSNS (52), /* cost of FSQRT instruction. */
1187 6, /* scalar_stmt_cost. */
1188 4, /* scalar load_cost. */
1189 4, /* scalar_store_cost. */
1190 6, /* vec_stmt_cost. */
1191 0, /* vec_to_scalar_cost. */
1192 2, /* scalar_to_vec_cost. */
1193 4, /* vec_align_load_cost. */
1194 4, /* vec_unalign_load_cost. */
1195 4, /* vec_store_cost. */
1196 4, /* cond_taken_branch_cost. */
1197 2, /* cond_not_taken_branch_cost. */
1200 /* BDVER4 has optimized REP instruction for medium sized blocks, but for
1201 very small blocks it is better to use loop. For large blocks, libcall
1202 can do nontemporary accesses and beat inline considerably. */
1203 static stringop_algs bdver4_memcpy[2] = {
1204 {libcall, {{6, loop, false}, {14, unrolled_loop, false},
1205 {-1, rep_prefix_4_byte, false}}},
1206 {libcall, {{16, loop, false}, {8192, rep_prefix_8_byte, false},
1207 {-1, libcall, false}}}};
1208 static stringop_algs bdver4_memset[2] = {
1209 {libcall, {{8, loop, false}, {24, unrolled_loop, false},
1210 {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
1211 {libcall, {{48, unrolled_loop, false}, {8192, rep_prefix_8_byte, false},
1212 {-1, libcall, false}}}};
1213 struct processor_costs bdver4_cost = {
1214 COSTS_N_INSNS (1), /* cost of an add instruction */
1215 COSTS_N_INSNS (1), /* cost of a lea instruction */
1216 COSTS_N_INSNS (1), /* variable shift costs */
1217 COSTS_N_INSNS (1), /* constant shift costs */
1218 {COSTS_N_INSNS (4), /* cost of starting multiply for QI */
1219 COSTS_N_INSNS (4), /* HI */
1220 COSTS_N_INSNS (4), /* SI */
1221 COSTS_N_INSNS (6), /* DI */
1222 COSTS_N_INSNS (6)}, /* other */
1223 0, /* cost of multiply per each bit set */
1224 {COSTS_N_INSNS (19), /* cost of a divide/mod for QI */
1225 COSTS_N_INSNS (35), /* HI */
1226 COSTS_N_INSNS (51), /* SI */
1227 COSTS_N_INSNS (83), /* DI */
1228 COSTS_N_INSNS (83)}, /* other */
1229 COSTS_N_INSNS (1), /* cost of movsx */
1230 COSTS_N_INSNS (1), /* cost of movzx */
1231 8, /* "large" insn */
1233 4, /* cost for loading QImode using movzbl */
1234 {5, 5, 4}, /* cost of loading integer registers
1235 in QImode, HImode and SImode.
1236 Relative to reg-reg move (2). */
1237 {4, 4, 4}, /* cost of storing integer registers */
1238 2, /* cost of reg,reg fld/fst */
1239 {5, 5, 12}, /* cost of loading fp registers
1240 in SFmode, DFmode and XFmode */
1241 {4, 4, 8}, /* cost of storing fp registers
1242 in SFmode, DFmode and XFmode */
1243 2, /* cost of moving MMX register */
1244 {4, 4}, /* cost of loading MMX registers
1245 in SImode and DImode */
1246 {4, 4}, /* cost of storing MMX registers
1247 in SImode and DImode */
1248 2, /* cost of moving SSE register */
1249 {4, 4, 4}, /* cost of loading SSE registers
1250 in SImode, DImode and TImode */
1251 {4, 4, 4}, /* cost of storing SSE registers
1252 in SImode, DImode and TImode */
1253 2, /* MMX or SSE register to integer */
1254 16, /* size of l1 cache. */
1255 2048, /* size of l2 cache. */
1256 64, /* size of prefetch block */
1257 /* New AMD processors never drop prefetches; if they cannot be performed
1258 immediately, they are queued. We set number of simultaneous prefetches
1259 to a large constant to reflect this (it probably is not a good idea not
1260 to limit number of prefetches at all, as their execution also takes some
1262 100, /* number of parallel prefetches */
1263 2, /* Branch cost */
1264 COSTS_N_INSNS (6), /* cost of FADD and FSUB insns. */
1265 COSTS_N_INSNS (6), /* cost of FMUL instruction. */
1266 COSTS_N_INSNS (42), /* cost of FDIV instruction. */
1267 COSTS_N_INSNS (2), /* cost of FABS instruction. */
1268 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
1269 COSTS_N_INSNS (52), /* cost of FSQRT instruction. */
1273 6, /* scalar_stmt_cost. */
1274 4, /* scalar load_cost. */
1275 4, /* scalar_store_cost. */
1276 6, /* vec_stmt_cost. */
1277 0, /* vec_to_scalar_cost. */
1278 2, /* scalar_to_vec_cost. */
1279 4, /* vec_align_load_cost. */
1280 4, /* vec_unalign_load_cost. */
1281 4, /* vec_store_cost. */
1282 4, /* cond_taken_branch_cost. */
1283 2, /* cond_not_taken_branch_cost. */
1286 /* BTVER1 has optimized REP instruction for medium sized blocks, but for
1287 very small blocks it is better to use loop. For large blocks, libcall can
1288 do nontemporary accesses and beat inline considerably. */
1289 static stringop_algs btver1_memcpy[2] = {
1290 {libcall, {{6, loop, false}, {14, unrolled_loop, false},
1291 {-1, rep_prefix_4_byte, false}}},
1292 {libcall, {{16, loop, false}, {8192, rep_prefix_8_byte, false},
1293 {-1, libcall, false}}}};
1294 static stringop_algs btver1_memset[2] = {
1295 {libcall, {{8, loop, false}, {24, unrolled_loop, false},
1296 {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
1297 {libcall, {{48, unrolled_loop, false}, {8192, rep_prefix_8_byte, false},
1298 {-1, libcall, false}}}};
1299 const struct processor_costs btver1_cost = {
1300 COSTS_N_INSNS (1), /* cost of an add instruction */
1301 COSTS_N_INSNS (2), /* cost of a lea instruction */
1302 COSTS_N_INSNS (1), /* variable shift costs */
1303 COSTS_N_INSNS (1), /* constant shift costs */
1304 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
1305 COSTS_N_INSNS (4), /* HI */
1306 COSTS_N_INSNS (3), /* SI */
1307 COSTS_N_INSNS (4), /* DI */
1308 COSTS_N_INSNS (5)}, /* other */
1309 0, /* cost of multiply per each bit set */
1310 {COSTS_N_INSNS (19), /* cost of a divide/mod for QI */
1311 COSTS_N_INSNS (35), /* HI */
1312 COSTS_N_INSNS (51), /* SI */
1313 COSTS_N_INSNS (83), /* DI */
1314 COSTS_N_INSNS (83)}, /* other */
1315 COSTS_N_INSNS (1), /* cost of movsx */
1316 COSTS_N_INSNS (1), /* cost of movzx */
1317 8, /* "large" insn */
1319 4, /* cost for loading QImode using movzbl */
1320 {3, 4, 3}, /* cost of loading integer registers
1321 in QImode, HImode and SImode.
1322 Relative to reg-reg move (2). */
1323 {3, 4, 3}, /* cost of storing integer registers */
1324 4, /* cost of reg,reg fld/fst */
1325 {4, 4, 12}, /* cost of loading fp registers
1326 in SFmode, DFmode and XFmode */
1327 {6, 6, 8}, /* cost of storing fp registers
1328 in SFmode, DFmode and XFmode */
1329 2, /* cost of moving MMX register */
1330 {3, 3}, /* cost of loading MMX registers
1331 in SImode and DImode */
1332 {4, 4}, /* cost of storing MMX registers
1333 in SImode and DImode */
1334 2, /* cost of moving SSE register */
1335 {4, 4, 3}, /* cost of loading SSE registers
1336 in SImode, DImode and TImode */
1337 {4, 4, 5}, /* cost of storing SSE registers
1338 in SImode, DImode and TImode */
1339 3, /* MMX or SSE register to integer */
1341 MOVD reg64, xmmreg Double FSTORE 4
1342 MOVD reg32, xmmreg Double FSTORE 4
1344 MOVD reg64, xmmreg Double FADD 3
1346 MOVD reg32, xmmreg Double FADD 3
1348 32, /* size of l1 cache. */
1349 512, /* size of l2 cache. */
1350 64, /* size of prefetch block */
1351 100, /* number of parallel prefetches */
1352 2, /* Branch cost */
1353 COSTS_N_INSNS (4), /* cost of FADD and FSUB insns. */
1354 COSTS_N_INSNS (4), /* cost of FMUL instruction. */
1355 COSTS_N_INSNS (19), /* cost of FDIV instruction. */
1356 COSTS_N_INSNS (2), /* cost of FABS instruction. */
1357 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
1358 COSTS_N_INSNS (35), /* cost of FSQRT instruction. */
1362 4, /* scalar_stmt_cost. */
1363 2, /* scalar load_cost. */
1364 2, /* scalar_store_cost. */
1365 6, /* vec_stmt_cost. */
1366 0, /* vec_to_scalar_cost. */
1367 2, /* scalar_to_vec_cost. */
1368 2, /* vec_align_load_cost. */
1369 2, /* vec_unalign_load_cost. */
1370 2, /* vec_store_cost. */
1371 2, /* cond_taken_branch_cost. */
1372 1, /* cond_not_taken_branch_cost. */
1375 static stringop_algs btver2_memcpy[2] = {
1376 {libcall, {{6, loop, false}, {14, unrolled_loop, false},
1377 {-1, rep_prefix_4_byte, false}}},
1378 {libcall, {{16, loop, false}, {8192, rep_prefix_8_byte, false},
1379 {-1, libcall, false}}}};
1380 static stringop_algs btver2_memset[2] = {
1381 {libcall, {{8, loop, false}, {24, unrolled_loop, false},
1382 {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
1383 {libcall, {{48, unrolled_loop, false}, {8192, rep_prefix_8_byte, false},
1384 {-1, libcall, false}}}};
1385 const struct processor_costs btver2_cost = {
1386 COSTS_N_INSNS (1), /* cost of an add instruction */
1387 COSTS_N_INSNS (2), /* cost of a lea instruction */
1388 COSTS_N_INSNS (1), /* variable shift costs */
1389 COSTS_N_INSNS (1), /* constant shift costs */
1390 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
1391 COSTS_N_INSNS (4), /* HI */
1392 COSTS_N_INSNS (3), /* SI */
1393 COSTS_N_INSNS (4), /* DI */
1394 COSTS_N_INSNS (5)}, /* other */
1395 0, /* cost of multiply per each bit set */
1396 {COSTS_N_INSNS (19), /* cost of a divide/mod for QI */
1397 COSTS_N_INSNS (35), /* HI */
1398 COSTS_N_INSNS (51), /* SI */
1399 COSTS_N_INSNS (83), /* DI */
1400 COSTS_N_INSNS (83)}, /* other */
1401 COSTS_N_INSNS (1), /* cost of movsx */
1402 COSTS_N_INSNS (1), /* cost of movzx */
1403 8, /* "large" insn */
1405 4, /* cost for loading QImode using movzbl */
1406 {3, 4, 3}, /* cost of loading integer registers
1407 in QImode, HImode and SImode.
1408 Relative to reg-reg move (2). */
1409 {3, 4, 3}, /* cost of storing integer registers */
1410 4, /* cost of reg,reg fld/fst */
1411 {4, 4, 12}, /* cost of loading fp registers
1412 in SFmode, DFmode and XFmode */
1413 {6, 6, 8}, /* cost of storing fp registers
1414 in SFmode, DFmode and XFmode */
1415 2, /* cost of moving MMX register */
1416 {3, 3}, /* cost of loading MMX registers
1417 in SImode and DImode */
1418 {4, 4}, /* cost of storing MMX registers
1419 in SImode and DImode */
1420 2, /* cost of moving SSE register */
1421 {4, 4, 3}, /* cost of loading SSE registers
1422 in SImode, DImode and TImode */
1423 {4, 4, 5}, /* cost of storing SSE registers
1424 in SImode, DImode and TImode */
1425 3, /* MMX or SSE register to integer */
1427 MOVD reg64, xmmreg Double FSTORE 4
1428 MOVD reg32, xmmreg Double FSTORE 4
1430 MOVD reg64, xmmreg Double FADD 3
1432 MOVD reg32, xmmreg Double FADD 3
1434 32, /* size of l1 cache. */
1435 2048, /* size of l2 cache. */
1436 64, /* size of prefetch block */
1437 100, /* number of parallel prefetches */
1438 2, /* Branch cost */
1439 COSTS_N_INSNS (4), /* cost of FADD and FSUB insns. */
1440 COSTS_N_INSNS (4), /* cost of FMUL instruction. */
1441 COSTS_N_INSNS (19), /* cost of FDIV instruction. */
1442 COSTS_N_INSNS (2), /* cost of FABS instruction. */
1443 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
1444 COSTS_N_INSNS (35), /* cost of FSQRT instruction. */
1447 4, /* scalar_stmt_cost. */
1448 2, /* scalar load_cost. */
1449 2, /* scalar_store_cost. */
1450 6, /* vec_stmt_cost. */
1451 0, /* vec_to_scalar_cost. */
1452 2, /* scalar_to_vec_cost. */
1453 2, /* vec_align_load_cost. */
1454 2, /* vec_unalign_load_cost. */
1455 2, /* vec_store_cost. */
1456 2, /* cond_taken_branch_cost. */
1457 1, /* cond_not_taken_branch_cost. */
1460 static stringop_algs pentium4_memcpy[2] = {
1461 {libcall, {{12, loop_1_byte, false}, {-1, rep_prefix_4_byte, false}}},
1462 DUMMY_STRINGOP_ALGS};
1463 static stringop_algs pentium4_memset[2] = {
1464 {libcall, {{6, loop_1_byte, false}, {48, loop, false},
1465 {20480, rep_prefix_4_byte, false}, {-1, libcall, false}}},
1466 DUMMY_STRINGOP_ALGS};
1469 struct processor_costs pentium4_cost = {
1470 COSTS_N_INSNS (1), /* cost of an add instruction */
1471 COSTS_N_INSNS (3), /* cost of a lea instruction */
1472 COSTS_N_INSNS (4), /* variable shift costs */
1473 COSTS_N_INSNS (4), /* constant shift costs */
1474 {COSTS_N_INSNS (15), /* cost of starting multiply for QI */
1475 COSTS_N_INSNS (15), /* HI */
1476 COSTS_N_INSNS (15), /* SI */
1477 COSTS_N_INSNS (15), /* DI */
1478 COSTS_N_INSNS (15)}, /* other */
1479 0, /* cost of multiply per each bit set */
1480 {COSTS_N_INSNS (56), /* cost of a divide/mod for QI */
1481 COSTS_N_INSNS (56), /* HI */
1482 COSTS_N_INSNS (56), /* SI */
1483 COSTS_N_INSNS (56), /* DI */
1484 COSTS_N_INSNS (56)}, /* other */
1485 COSTS_N_INSNS (1), /* cost of movsx */
1486 COSTS_N_INSNS (1), /* cost of movzx */
1487 16, /* "large" insn */
1489 2, /* cost for loading QImode using movzbl */
1490 {4, 5, 4}, /* cost of loading integer registers
1491 in QImode, HImode and SImode.
1492 Relative to reg-reg move (2). */
1493 {2, 3, 2}, /* cost of storing integer registers */
1494 2, /* cost of reg,reg fld/fst */
1495 {2, 2, 6}, /* cost of loading fp registers
1496 in SFmode, DFmode and XFmode */
1497 {4, 4, 6}, /* cost of storing fp registers
1498 in SFmode, DFmode and XFmode */
1499 2, /* cost of moving MMX register */
1500 {2, 2}, /* cost of loading MMX registers
1501 in SImode and DImode */
1502 {2, 2}, /* cost of storing MMX registers
1503 in SImode and DImode */
1504 12, /* cost of moving SSE register */
1505 {12, 12, 12}, /* cost of loading SSE registers
1506 in SImode, DImode and TImode */
1507 {2, 2, 8}, /* cost of storing SSE registers
1508 in SImode, DImode and TImode */
1509 10, /* MMX or SSE register to integer */
1510 8, /* size of l1 cache. */
1511 256, /* size of l2 cache. */
1512 64, /* size of prefetch block */
1513 6, /* number of parallel prefetches */
1514 2, /* Branch cost */
1515 COSTS_N_INSNS (5), /* cost of FADD and FSUB insns. */
1516 COSTS_N_INSNS (7), /* cost of FMUL instruction. */
1517 COSTS_N_INSNS (43), /* cost of FDIV instruction. */
1518 COSTS_N_INSNS (2), /* cost of FABS instruction. */
1519 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
1520 COSTS_N_INSNS (43), /* cost of FSQRT instruction. */
1523 1, /* scalar_stmt_cost. */
1524 1, /* scalar load_cost. */
1525 1, /* scalar_store_cost. */
1526 1, /* vec_stmt_cost. */
1527 1, /* vec_to_scalar_cost. */
1528 1, /* scalar_to_vec_cost. */
1529 1, /* vec_align_load_cost. */
1530 2, /* vec_unalign_load_cost. */
1531 1, /* vec_store_cost. */
1532 3, /* cond_taken_branch_cost. */
1533 1, /* cond_not_taken_branch_cost. */
1536 static stringop_algs nocona_memcpy[2] = {
1537 {libcall, {{12, loop_1_byte, false}, {-1, rep_prefix_4_byte, false}}},
1538 {libcall, {{32, loop, false}, {20000, rep_prefix_8_byte, false},
1539 {100000, unrolled_loop, false}, {-1, libcall, false}}}};
1541 static stringop_algs nocona_memset[2] = {
1542 {libcall, {{6, loop_1_byte, false}, {48, loop, false},
1543 {20480, rep_prefix_4_byte, false}, {-1, libcall, false}}},
1544 {libcall, {{24, loop, false}, {64, unrolled_loop, false},
1545 {8192, rep_prefix_8_byte, false}, {-1, libcall, false}}}};
1548 struct processor_costs nocona_cost = {
1549 COSTS_N_INSNS (1), /* cost of an add instruction */
1550 COSTS_N_INSNS (1), /* cost of a lea instruction */
1551 COSTS_N_INSNS (1), /* variable shift costs */
1552 COSTS_N_INSNS (1), /* constant shift costs */
1553 {COSTS_N_INSNS (10), /* cost of starting multiply for QI */
1554 COSTS_N_INSNS (10), /* HI */
1555 COSTS_N_INSNS (10), /* SI */
1556 COSTS_N_INSNS (10), /* DI */
1557 COSTS_N_INSNS (10)}, /* other */
1558 0, /* cost of multiply per each bit set */
1559 {COSTS_N_INSNS (66), /* cost of a divide/mod for QI */
1560 COSTS_N_INSNS (66), /* HI */
1561 COSTS_N_INSNS (66), /* SI */
1562 COSTS_N_INSNS (66), /* DI */
1563 COSTS_N_INSNS (66)}, /* other */
1564 COSTS_N_INSNS (1), /* cost of movsx */
1565 COSTS_N_INSNS (1), /* cost of movzx */
1566 16, /* "large" insn */
1567 17, /* MOVE_RATIO */
1568 4, /* cost for loading QImode using movzbl */
1569 {4, 4, 4}, /* cost of loading integer registers
1570 in QImode, HImode and SImode.
1571 Relative to reg-reg move (2). */
1572 {4, 4, 4}, /* cost of storing integer registers */
1573 3, /* cost of reg,reg fld/fst */
1574 {12, 12, 12}, /* cost of loading fp registers
1575 in SFmode, DFmode and XFmode */
1576 {4, 4, 4}, /* cost of storing fp registers
1577 in SFmode, DFmode and XFmode */
1578 6, /* cost of moving MMX register */
1579 {12, 12}, /* cost of loading MMX registers
1580 in SImode and DImode */
1581 {12, 12}, /* cost of storing MMX registers
1582 in SImode and DImode */
1583 6, /* cost of moving SSE register */
1584 {12, 12, 12}, /* cost of loading SSE registers
1585 in SImode, DImode and TImode */
1586 {12, 12, 12}, /* cost of storing SSE registers
1587 in SImode, DImode and TImode */
1588 8, /* MMX or SSE register to integer */
1589 8, /* size of l1 cache. */
1590 1024, /* size of l2 cache. */
1591 64, /* size of prefetch block */
1592 8, /* number of parallel prefetches */
1593 1, /* Branch cost */
1594 COSTS_N_INSNS (6), /* cost of FADD and FSUB insns. */
1595 COSTS_N_INSNS (8), /* cost of FMUL instruction. */
1596 COSTS_N_INSNS (40), /* cost of FDIV instruction. */
1597 COSTS_N_INSNS (3), /* cost of FABS instruction. */
1598 COSTS_N_INSNS (3), /* cost of FCHS instruction. */
1599 COSTS_N_INSNS (44), /* cost of FSQRT instruction. */
1602 1, /* scalar_stmt_cost. */
1603 1, /* scalar load_cost. */
1604 1, /* scalar_store_cost. */
1605 1, /* vec_stmt_cost. */
1606 1, /* vec_to_scalar_cost. */
1607 1, /* scalar_to_vec_cost. */
1608 1, /* vec_align_load_cost. */
1609 2, /* vec_unalign_load_cost. */
1610 1, /* vec_store_cost. */
1611 3, /* cond_taken_branch_cost. */
1612 1, /* cond_not_taken_branch_cost. */
1615 static stringop_algs atom_memcpy[2] = {
1616 {libcall, {{11, loop, false}, {-1, rep_prefix_4_byte, false}}},
1617 {libcall, {{32, loop, false}, {64, rep_prefix_4_byte, false},
1618 {8192, rep_prefix_8_byte, false}, {-1, libcall, false}}}};
1619 static stringop_algs atom_memset[2] = {
1620 {libcall, {{8, loop, false}, {15, unrolled_loop, false},
1621 {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
1622 {libcall, {{24, loop, false}, {32, unrolled_loop, false},
1623 {8192, rep_prefix_8_byte, false}, {-1, libcall, false}}}};
1625 struct processor_costs atom_cost = {
1626 COSTS_N_INSNS (1), /* cost of an add instruction */
1627 COSTS_N_INSNS (1) + 1, /* cost of a lea instruction */
1628 COSTS_N_INSNS (1), /* variable shift costs */
1629 COSTS_N_INSNS (1), /* constant shift costs */
1630 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
1631 COSTS_N_INSNS (4), /* HI */
1632 COSTS_N_INSNS (3), /* SI */
1633 COSTS_N_INSNS (4), /* DI */
1634 COSTS_N_INSNS (2)}, /* other */
1635 0, /* cost of multiply per each bit set */
1636 {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */
1637 COSTS_N_INSNS (26), /* HI */
1638 COSTS_N_INSNS (42), /* SI */
1639 COSTS_N_INSNS (74), /* DI */
1640 COSTS_N_INSNS (74)}, /* other */
1641 COSTS_N_INSNS (1), /* cost of movsx */
1642 COSTS_N_INSNS (1), /* cost of movzx */
1643 8, /* "large" insn */
1644 17, /* MOVE_RATIO */
1645 4, /* cost for loading QImode using movzbl */
1646 {4, 4, 4}, /* cost of loading integer registers
1647 in QImode, HImode and SImode.
1648 Relative to reg-reg move (2). */
1649 {4, 4, 4}, /* cost of storing integer registers */
1650 4, /* cost of reg,reg fld/fst */
1651 {12, 12, 12}, /* cost of loading fp registers
1652 in SFmode, DFmode and XFmode */
1653 {6, 6, 8}, /* cost of storing fp registers
1654 in SFmode, DFmode and XFmode */
1655 2, /* cost of moving MMX register */
1656 {8, 8}, /* cost of loading MMX registers
1657 in SImode and DImode */
1658 {8, 8}, /* cost of storing MMX registers
1659 in SImode and DImode */
1660 2, /* cost of moving SSE register */
1661 {8, 8, 8}, /* cost of loading SSE registers
1662 in SImode, DImode and TImode */
1663 {8, 8, 8}, /* cost of storing SSE registers
1664 in SImode, DImode and TImode */
1665 5, /* MMX or SSE register to integer */
1666 32, /* size of l1 cache. */
1667 256, /* size of l2 cache. */
1668 64, /* size of prefetch block */
1669 6, /* number of parallel prefetches */
1670 3, /* Branch cost */
1671 COSTS_N_INSNS (8), /* cost of FADD and FSUB insns. */
1672 COSTS_N_INSNS (8), /* cost of FMUL instruction. */
1673 COSTS_N_INSNS (20), /* cost of FDIV instruction. */
1674 COSTS_N_INSNS (8), /* cost of FABS instruction. */
1675 COSTS_N_INSNS (8), /* cost of FCHS instruction. */
1676 COSTS_N_INSNS (40), /* cost of FSQRT instruction. */
1679 1, /* scalar_stmt_cost. */
1680 1, /* scalar load_cost. */
1681 1, /* scalar_store_cost. */
1682 1, /* vec_stmt_cost. */
1683 1, /* vec_to_scalar_cost. */
1684 1, /* scalar_to_vec_cost. */
1685 1, /* vec_align_load_cost. */
1686 2, /* vec_unalign_load_cost. */
1687 1, /* vec_store_cost. */
1688 3, /* cond_taken_branch_cost. */
1689 1, /* cond_not_taken_branch_cost. */
1692 static stringop_algs slm_memcpy[2] = {
1693 {libcall, {{11, loop, false}, {-1, rep_prefix_4_byte, false}}},
1694 {libcall, {{32, loop, false}, {64, rep_prefix_4_byte, false},
1695 {8192, rep_prefix_8_byte, false}, {-1, libcall, false}}}};
1696 static stringop_algs slm_memset[2] = {
1697 {libcall, {{8, loop, false}, {15, unrolled_loop, false},
1698 {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
1699 {libcall, {{24, loop, false}, {32, unrolled_loop, false},
1700 {8192, rep_prefix_8_byte, false}, {-1, libcall, false}}}};
1702 struct processor_costs slm_cost = {
1703 COSTS_N_INSNS (1), /* cost of an add instruction */
1704 COSTS_N_INSNS (1) + 1, /* cost of a lea instruction */
1705 COSTS_N_INSNS (1), /* variable shift costs */
1706 COSTS_N_INSNS (1), /* constant shift costs */
1707 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
1708 COSTS_N_INSNS (3), /* HI */
1709 COSTS_N_INSNS (3), /* SI */
1710 COSTS_N_INSNS (4), /* DI */
1711 COSTS_N_INSNS (2)}, /* other */
1712 0, /* cost of multiply per each bit set */
1713 {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */
1714 COSTS_N_INSNS (26), /* HI */
1715 COSTS_N_INSNS (42), /* SI */
1716 COSTS_N_INSNS (74), /* DI */
1717 COSTS_N_INSNS (74)}, /* other */
1718 COSTS_N_INSNS (1), /* cost of movsx */
1719 COSTS_N_INSNS (1), /* cost of movzx */
1720 8, /* "large" insn */
1721 17, /* MOVE_RATIO */
1722 4, /* cost for loading QImode using movzbl */
1723 {4, 4, 4}, /* cost of loading integer registers
1724 in QImode, HImode and SImode.
1725 Relative to reg-reg move (2). */
1726 {4, 4, 4}, /* cost of storing integer registers */
1727 4, /* cost of reg,reg fld/fst */
1728 {12, 12, 12}, /* cost of loading fp registers
1729 in SFmode, DFmode and XFmode */
1730 {6, 6, 8}, /* cost of storing fp registers
1731 in SFmode, DFmode and XFmode */
1732 2, /* cost of moving MMX register */
1733 {8, 8}, /* cost of loading MMX registers
1734 in SImode and DImode */
1735 {8, 8}, /* cost of storing MMX registers
1736 in SImode and DImode */
1737 2, /* cost of moving SSE register */
1738 {8, 8, 8}, /* cost of loading SSE registers
1739 in SImode, DImode and TImode */
1740 {8, 8, 8}, /* cost of storing SSE registers
1741 in SImode, DImode and TImode */
1742 5, /* MMX or SSE register to integer */
1743 32, /* size of l1 cache. */
1744 256, /* size of l2 cache. */
1745 64, /* size of prefetch block */
1746 6, /* number of parallel prefetches */
1747 3, /* Branch cost */
1748 COSTS_N_INSNS (8), /* cost of FADD and FSUB insns. */
1749 COSTS_N_INSNS (8), /* cost of FMUL instruction. */
1750 COSTS_N_INSNS (20), /* cost of FDIV instruction. */
1751 COSTS_N_INSNS (8), /* cost of FABS instruction. */
1752 COSTS_N_INSNS (8), /* cost of FCHS instruction. */
1753 COSTS_N_INSNS (40), /* cost of FSQRT instruction. */
1756 1, /* scalar_stmt_cost. */
1757 1, /* scalar load_cost. */
1758 1, /* scalar_store_cost. */
1759 1, /* vec_stmt_cost. */
1760 4, /* vec_to_scalar_cost. */
1761 1, /* scalar_to_vec_cost. */
1762 1, /* vec_align_load_cost. */
1763 2, /* vec_unalign_load_cost. */
1764 1, /* vec_store_cost. */
1765 3, /* cond_taken_branch_cost. */
1766 1, /* cond_not_taken_branch_cost. */
1769 static stringop_algs intel_memcpy[2] = {
1770 {libcall, {{11, loop, false}, {-1, rep_prefix_4_byte, false}}},
1771 {libcall, {{32, loop, false}, {64, rep_prefix_4_byte, false},
1772 {8192, rep_prefix_8_byte, false}, {-1, libcall, false}}}};
1773 static stringop_algs intel_memset[2] = {
1774 {libcall, {{8, loop, false}, {15, unrolled_loop, false},
1775 {2048, rep_prefix_4_byte, false}, {-1, libcall, false}}},
1776 {libcall, {{24, loop, false}, {32, unrolled_loop, false},
1777 {8192, rep_prefix_8_byte, false}, {-1, libcall, false}}}};
1779 struct processor_costs intel_cost = {
1780 COSTS_N_INSNS (1), /* cost of an add instruction */
1781 COSTS_N_INSNS (1) + 1, /* cost of a lea instruction */
1782 COSTS_N_INSNS (1), /* variable shift costs */
1783 COSTS_N_INSNS (1), /* constant shift costs */
1784 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
1785 COSTS_N_INSNS (3), /* HI */
1786 COSTS_N_INSNS (3), /* SI */
1787 COSTS_N_INSNS (4), /* DI */
1788 COSTS_N_INSNS (2)}, /* other */
1789 0, /* cost of multiply per each bit set */
1790 {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */
1791 COSTS_N_INSNS (26), /* HI */
1792 COSTS_N_INSNS (42), /* SI */
1793 COSTS_N_INSNS (74), /* DI */
1794 COSTS_N_INSNS (74)}, /* other */
1795 COSTS_N_INSNS (1), /* cost of movsx */
1796 COSTS_N_INSNS (1), /* cost of movzx */
1797 8, /* "large" insn */
1798 17, /* MOVE_RATIO */
1799 4, /* cost for loading QImode using movzbl */
1800 {4, 4, 4}, /* cost of loading integer registers
1801 in QImode, HImode and SImode.
1802 Relative to reg-reg move (2). */
1803 {4, 4, 4}, /* cost of storing integer registers */
1804 4, /* cost of reg,reg fld/fst */
1805 {12, 12, 12}, /* cost of loading fp registers
1806 in SFmode, DFmode and XFmode */
1807 {6, 6, 8}, /* cost of storing fp registers
1808 in SFmode, DFmode and XFmode */
1809 2, /* cost of moving MMX register */
1810 {8, 8}, /* cost of loading MMX registers
1811 in SImode and DImode */
1812 {8, 8}, /* cost of storing MMX registers
1813 in SImode and DImode */
1814 2, /* cost of moving SSE register */
1815 {8, 8, 8}, /* cost of loading SSE registers
1816 in SImode, DImode and TImode */
1817 {8, 8, 8}, /* cost of storing SSE registers
1818 in SImode, DImode and TImode */
1819 5, /* MMX or SSE register to integer */
1820 32, /* size of l1 cache. */
1821 256, /* size of l2 cache. */
1822 64, /* size of prefetch block */
1823 6, /* number of parallel prefetches */
1824 3, /* Branch cost */
1825 COSTS_N_INSNS (8), /* cost of FADD and FSUB insns. */
1826 COSTS_N_INSNS (8), /* cost of FMUL instruction. */
1827 COSTS_N_INSNS (20), /* cost of FDIV instruction. */
1828 COSTS_N_INSNS (8), /* cost of FABS instruction. */
1829 COSTS_N_INSNS (8), /* cost of FCHS instruction. */
1830 COSTS_N_INSNS (40), /* cost of FSQRT instruction. */
1833 1, /* scalar_stmt_cost. */
1834 1, /* scalar load_cost. */
1835 1, /* scalar_store_cost. */
1836 1, /* vec_stmt_cost. */
1837 4, /* vec_to_scalar_cost. */
1838 1, /* scalar_to_vec_cost. */
1839 1, /* vec_align_load_cost. */
1840 2, /* vec_unalign_load_cost. */
1841 1, /* vec_store_cost. */
1842 3, /* cond_taken_branch_cost. */
1843 1, /* cond_not_taken_branch_cost. */
1846 /* Generic should produce code tuned for Core-i7 (and newer chips)
1847 and btver1 (and newer chips). */
1849 static stringop_algs generic_memcpy[2] = {
1850 {libcall, {{32, loop, false}, {8192, rep_prefix_4_byte, false},
1851 {-1, libcall, false}}},
1852 {libcall, {{32, loop, false}, {8192, rep_prefix_8_byte, false},
1853 {-1, libcall, false}}}};
1854 static stringop_algs generic_memset[2] = {
1855 {libcall, {{32, loop, false}, {8192, rep_prefix_4_byte, false},
1856 {-1, libcall, false}}},
1857 {libcall, {{32, loop, false}, {8192, rep_prefix_8_byte, false},
1858 {-1, libcall, false}}}};
1860 struct processor_costs generic_cost = {
1861 COSTS_N_INSNS (1), /* cost of an add instruction */
1862 /* On all chips taken into consideration lea is 2 cycles and more. With
1863 this cost however our current implementation of synth_mult results in
1864 use of unnecessary temporary registers causing regression on several
1865 SPECfp benchmarks. */
1866 COSTS_N_INSNS (1) + 1, /* cost of a lea instruction */
1867 COSTS_N_INSNS (1), /* variable shift costs */
1868 COSTS_N_INSNS (1), /* constant shift costs */
1869 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
1870 COSTS_N_INSNS (4), /* HI */
1871 COSTS_N_INSNS (3), /* SI */
1872 COSTS_N_INSNS (4), /* DI */
1873 COSTS_N_INSNS (2)}, /* other */
1874 0, /* cost of multiply per each bit set */
1875 {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */
1876 COSTS_N_INSNS (26), /* HI */
1877 COSTS_N_INSNS (42), /* SI */
1878 COSTS_N_INSNS (74), /* DI */
1879 COSTS_N_INSNS (74)}, /* other */
1880 COSTS_N_INSNS (1), /* cost of movsx */
1881 COSTS_N_INSNS (1), /* cost of movzx */
1882 8, /* "large" insn */
1883 17, /* MOVE_RATIO */
1884 4, /* cost for loading QImode using movzbl */
1885 {4, 4, 4}, /* cost of loading integer registers
1886 in QImode, HImode and SImode.
1887 Relative to reg-reg move (2). */
1888 {4, 4, 4}, /* cost of storing integer registers */
1889 4, /* cost of reg,reg fld/fst */
1890 {12, 12, 12}, /* cost of loading fp registers
1891 in SFmode, DFmode and XFmode */
1892 {6, 6, 8}, /* cost of storing fp registers
1893 in SFmode, DFmode and XFmode */
1894 2, /* cost of moving MMX register */
1895 {8, 8}, /* cost of loading MMX registers
1896 in SImode and DImode */
1897 {8, 8}, /* cost of storing MMX registers
1898 in SImode and DImode */
1899 2, /* cost of moving SSE register */
1900 {8, 8, 8}, /* cost of loading SSE registers
1901 in SImode, DImode and TImode */
1902 {8, 8, 8}, /* cost of storing SSE registers
1903 in SImode, DImode and TImode */
1904 5, /* MMX or SSE register to integer */
1905 32, /* size of l1 cache. */
1906 512, /* size of l2 cache. */
1907 64, /* size of prefetch block */
1908 6, /* number of parallel prefetches */
1909 /* Benchmarks shows large regressions on K8 sixtrack benchmark when this
1910 value is increased to perhaps more appropriate value of 5. */
1911 3, /* Branch cost */
1912 COSTS_N_INSNS (8), /* cost of FADD and FSUB insns. */
1913 COSTS_N_INSNS (8), /* cost of FMUL instruction. */
1914 COSTS_N_INSNS (20), /* cost of FDIV instruction. */
1915 COSTS_N_INSNS (8), /* cost of FABS instruction. */
1916 COSTS_N_INSNS (8), /* cost of FCHS instruction. */
1917 COSTS_N_INSNS (40), /* cost of FSQRT instruction. */
1920 1, /* scalar_stmt_cost. */
1921 1, /* scalar load_cost. */
1922 1, /* scalar_store_cost. */
1923 1, /* vec_stmt_cost. */
1924 1, /* vec_to_scalar_cost. */
1925 1, /* scalar_to_vec_cost. */
1926 1, /* vec_align_load_cost. */
1927 2, /* vec_unalign_load_cost. */
1928 1, /* vec_store_cost. */
1929 3, /* cond_taken_branch_cost. */
1930 1, /* cond_not_taken_branch_cost. */
1933 /* core_cost should produce code tuned for Core familly of CPUs. */
1934 static stringop_algs core_memcpy[2] = {
1935 {libcall, {{1024, rep_prefix_4_byte, true}, {-1, libcall, false}}},
1936 {libcall, {{24, loop, true}, {128, rep_prefix_8_byte, true},
1937 {-1, libcall, false}}}};
1938 static stringop_algs core_memset[2] = {
1939 {libcall, {{6, loop_1_byte, true},
1941 {8192, rep_prefix_4_byte, true},
1942 {-1, libcall, false}}},
1943 {libcall, {{24, loop, true}, {512, rep_prefix_8_byte, true},
1944 {-1, libcall, false}}}};
1947 struct processor_costs core_cost = {
1948 COSTS_N_INSNS (1), /* cost of an add instruction */
1949 /* On all chips taken into consideration lea is 2 cycles and more. With
1950 this cost however our current implementation of synth_mult results in
1951 use of unnecessary temporary registers causing regression on several
1952 SPECfp benchmarks. */
1953 COSTS_N_INSNS (1) + 1, /* cost of a lea instruction */
1954 COSTS_N_INSNS (1), /* variable shift costs */
1955 COSTS_N_INSNS (1), /* constant shift costs */
1956 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
1957 COSTS_N_INSNS (4), /* HI */
1958 COSTS_N_INSNS (3), /* SI */
1959 COSTS_N_INSNS (4), /* DI */
1960 COSTS_N_INSNS (2)}, /* other */
1961 0, /* cost of multiply per each bit set */
1962 {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */
1963 COSTS_N_INSNS (26), /* HI */
1964 COSTS_N_INSNS (42), /* SI */
1965 COSTS_N_INSNS (74), /* DI */
1966 COSTS_N_INSNS (74)}, /* other */
1967 COSTS_N_INSNS (1), /* cost of movsx */
1968 COSTS_N_INSNS (1), /* cost of movzx */
1969 8, /* "large" insn */
1970 17, /* MOVE_RATIO */
1971 4, /* cost for loading QImode using movzbl */
1972 {4, 4, 4}, /* cost of loading integer registers
1973 in QImode, HImode and SImode.
1974 Relative to reg-reg move (2). */
1975 {4, 4, 4}, /* cost of storing integer registers */
1976 4, /* cost of reg,reg fld/fst */
1977 {12, 12, 12}, /* cost of loading fp registers
1978 in SFmode, DFmode and XFmode */
1979 {6, 6, 8}, /* cost of storing fp registers
1980 in SFmode, DFmode and XFmode */
1981 2, /* cost of moving MMX register */
1982 {8, 8}, /* cost of loading MMX registers
1983 in SImode and DImode */
1984 {8, 8}, /* cost of storing MMX registers
1985 in SImode and DImode */
1986 2, /* cost of moving SSE register */
1987 {8, 8, 8}, /* cost of loading SSE registers
1988 in SImode, DImode and TImode */
1989 {8, 8, 8}, /* cost of storing SSE registers
1990 in SImode, DImode and TImode */
1991 5, /* MMX or SSE register to integer */
1992 64, /* size of l1 cache. */
1993 512, /* size of l2 cache. */
1994 64, /* size of prefetch block */
1995 6, /* number of parallel prefetches */
1996 /* FIXME perhaps more appropriate value is 5. */
1997 3, /* Branch cost */
1998 COSTS_N_INSNS (8), /* cost of FADD and FSUB insns. */
1999 COSTS_N_INSNS (8), /* cost of FMUL instruction. */
2000 COSTS_N_INSNS (20), /* cost of FDIV instruction. */
2001 COSTS_N_INSNS (8), /* cost of FABS instruction. */
2002 COSTS_N_INSNS (8), /* cost of FCHS instruction. */
2003 COSTS_N_INSNS (40), /* cost of FSQRT instruction. */
2006 1, /* scalar_stmt_cost. */
2007 1, /* scalar load_cost. */
2008 1, /* scalar_store_cost. */
2009 1, /* vec_stmt_cost. */
2010 1, /* vec_to_scalar_cost. */
2011 1, /* scalar_to_vec_cost. */
2012 1, /* vec_align_load_cost. */
2013 2, /* vec_unalign_load_cost. */
2014 1, /* vec_store_cost. */
2015 3, /* cond_taken_branch_cost. */
2016 1, /* cond_not_taken_branch_cost. */
2020 /* Set by -mtune. */
2021 const struct processor_costs *ix86_tune_cost = &pentium_cost;
2023 /* Set by -mtune or -Os. */
2024 const struct processor_costs *ix86_cost = &pentium_cost;
2026 /* Processor feature/optimization bitmasks. */
2027 #define m_386 (1<<PROCESSOR_I386)
2028 #define m_486 (1<<PROCESSOR_I486)
2029 #define m_PENT (1<<PROCESSOR_PENTIUM)
2030 #define m_PPRO (1<<PROCESSOR_PENTIUMPRO)
2031 #define m_PENT4 (1<<PROCESSOR_PENTIUM4)
2032 #define m_NOCONA (1<<PROCESSOR_NOCONA)
2033 #define m_P4_NOCONA (m_PENT4 | m_NOCONA)
2034 #define m_CORE2 (1<<PROCESSOR_CORE2)
2035 #define m_NEHALEM (1<<PROCESSOR_NEHALEM)
2036 #define m_SANDYBRIDGE (1<<PROCESSOR_SANDYBRIDGE)
2037 #define m_HASWELL (1<<PROCESSOR_HASWELL)
2038 #define m_CORE_ALL (m_CORE2 | m_NEHALEM | m_SANDYBRIDGE | m_HASWELL)
2039 #define m_BONNELL (1<<PROCESSOR_BONNELL)
2040 #define m_SILVERMONT (1<<PROCESSOR_SILVERMONT)
2041 #define m_KNL (1<<PROCESSOR_KNL)
2042 #define m_INTEL (1<<PROCESSOR_INTEL)
2044 #define m_GEODE (1<<PROCESSOR_GEODE)
2045 #define m_K6 (1<<PROCESSOR_K6)
2046 #define m_K6_GEODE (m_K6 | m_GEODE)
2047 #define m_K8 (1<<PROCESSOR_K8)
2048 #define m_ATHLON (1<<PROCESSOR_ATHLON)
2049 #define m_ATHLON_K8 (m_K8 | m_ATHLON)
2050 #define m_AMDFAM10 (1<<PROCESSOR_AMDFAM10)
2051 #define m_BDVER1 (1<<PROCESSOR_BDVER1)
2052 #define m_BDVER2 (1<<PROCESSOR_BDVER2)
2053 #define m_BDVER3 (1<<PROCESSOR_BDVER3)
2054 #define m_BDVER4 (1<<PROCESSOR_BDVER4)
2055 #define m_BTVER1 (1<<PROCESSOR_BTVER1)
2056 #define m_BTVER2 (1<<PROCESSOR_BTVER2)
2057 #define m_BDVER (m_BDVER1 | m_BDVER2 | m_BDVER3 | m_BDVER4)
2058 #define m_BTVER (m_BTVER1 | m_BTVER2)
2059 #define m_AMD_MULTIPLE (m_ATHLON_K8 | m_AMDFAM10 | m_BDVER | m_BTVER)
2061 #define m_GENERIC (1<<PROCESSOR_GENERIC)
2063 const char* ix86_tune_feature_names[X86_TUNE_LAST] = {
2065 #define DEF_TUNE(tune, name, selector) name,
2066 #include "x86-tune.def"
2070 /* Feature tests against the various tunings. */
2071 unsigned char ix86_tune_features[X86_TUNE_LAST];
2073 /* Feature tests against the various tunings used to create ix86_tune_features
2074 based on the processor mask. */
2075 static unsigned int initial_ix86_tune_features[X86_TUNE_LAST] = {
2077 #define DEF_TUNE(tune, name, selector) selector,
2078 #include "x86-tune.def"
2082 /* Feature tests against the various architecture variations. */
2083 unsigned char ix86_arch_features[X86_ARCH_LAST];
2085 /* Feature tests against the various architecture variations, used to create
2086 ix86_arch_features based on the processor mask. */
2087 static unsigned int initial_ix86_arch_features[X86_ARCH_LAST] = {
2088 /* X86_ARCH_CMOV: Conditional move was added for pentiumpro. */
2089 ~(m_386 | m_486 | m_PENT | m_K6),
2091 /* X86_ARCH_CMPXCHG: Compare and exchange was added for 80486. */
2094 /* X86_ARCH_CMPXCHG8B: Compare and exchange 8 bytes was added for pentium. */
2097 /* X86_ARCH_XADD: Exchange and add was added for 80486. */
2100 /* X86_ARCH_BSWAP: Byteswap was added for 80486. */
2104 /* In case the average insn count for single function invocation is
2105 lower than this constant, emit fast (but longer) prologue and
2107 #define FAST_PROLOGUE_INSN_COUNT 20
2109 /* Names for 8 (low), 8 (high), and 16-bit registers, respectively. */
2110 static const char *const qi_reg_name[] = QI_REGISTER_NAMES;
2111 static const char *const qi_high_reg_name[] = QI_HIGH_REGISTER_NAMES;
2112 static const char *const hi_reg_name[] = HI_REGISTER_NAMES;
2114 /* Array of the smallest class containing reg number REGNO, indexed by
2115 REGNO. Used by REGNO_REG_CLASS in i386.h. */
2117 enum reg_class const regclass_map[FIRST_PSEUDO_REGISTER] =
2119 /* ax, dx, cx, bx */
2120 AREG, DREG, CREG, BREG,
2121 /* si, di, bp, sp */
2122 SIREG, DIREG, NON_Q_REGS, NON_Q_REGS,
2124 FP_TOP_REG, FP_SECOND_REG, FLOAT_REGS, FLOAT_REGS,
2125 FLOAT_REGS, FLOAT_REGS, FLOAT_REGS, FLOAT_REGS,
2128 /* flags, fpsr, fpcr, frame */
2129 NO_REGS, NO_REGS, NO_REGS, NON_Q_REGS,
2131 SSE_FIRST_REG, SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS,
2134 MMX_REGS, MMX_REGS, MMX_REGS, MMX_REGS, MMX_REGS, MMX_REGS,
2137 NON_Q_REGS, NON_Q_REGS, NON_Q_REGS, NON_Q_REGS,
2138 NON_Q_REGS, NON_Q_REGS, NON_Q_REGS, NON_Q_REGS,
2139 /* SSE REX registers */
2140 SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS,
2142 /* AVX-512 SSE registers */
2143 EVEX_SSE_REGS, EVEX_SSE_REGS, EVEX_SSE_REGS, EVEX_SSE_REGS,
2144 EVEX_SSE_REGS, EVEX_SSE_REGS, EVEX_SSE_REGS, EVEX_SSE_REGS,
2145 EVEX_SSE_REGS, EVEX_SSE_REGS, EVEX_SSE_REGS, EVEX_SSE_REGS,
2146 EVEX_SSE_REGS, EVEX_SSE_REGS, EVEX_SSE_REGS, EVEX_SSE_REGS,
2147 /* Mask registers. */
2148 MASK_REGS, MASK_EVEX_REGS, MASK_EVEX_REGS, MASK_EVEX_REGS,
2149 MASK_EVEX_REGS, MASK_EVEX_REGS, MASK_EVEX_REGS, MASK_EVEX_REGS,
2150 /* MPX bound registers */
2151 BND_REGS, BND_REGS, BND_REGS, BND_REGS,
2154 /* The "default" register map used in 32bit mode. */
2156 int const dbx_register_map[FIRST_PSEUDO_REGISTER] =
2158 0, 2, 1, 3, 6, 7, 4, 5, /* general regs */
2159 12, 13, 14, 15, 16, 17, 18, 19, /* fp regs */
2160 -1, -1, -1, -1, -1, /* arg, flags, fpsr, fpcr, frame */
2161 21, 22, 23, 24, 25, 26, 27, 28, /* SSE */
2162 29, 30, 31, 32, 33, 34, 35, 36, /* MMX */
2163 -1, -1, -1, -1, -1, -1, -1, -1, /* extended integer registers */
2164 -1, -1, -1, -1, -1, -1, -1, -1, /* extended SSE registers */
2165 -1, -1, -1, -1, -1, -1, -1, -1, /* AVX-512 registers 16-23*/
2166 -1, -1, -1, -1, -1, -1, -1, -1, /* AVX-512 registers 24-31*/
2167 93, 94, 95, 96, 97, 98, 99, 100, /* Mask registers */
2168 101, 102, 103, 104, /* bound registers */
2171 /* The "default" register map used in 64bit mode. */
2173 int const dbx64_register_map[FIRST_PSEUDO_REGISTER] =
2175 0, 1, 2, 3, 4, 5, 6, 7, /* general regs */
2176 33, 34, 35, 36, 37, 38, 39, 40, /* fp regs */
2177 -1, -1, -1, -1, -1, /* arg, flags, fpsr, fpcr, frame */
2178 17, 18, 19, 20, 21, 22, 23, 24, /* SSE */
2179 41, 42, 43, 44, 45, 46, 47, 48, /* MMX */
2180 8,9,10,11,12,13,14,15, /* extended integer registers */
2181 25, 26, 27, 28, 29, 30, 31, 32, /* extended SSE registers */
2182 67, 68, 69, 70, 71, 72, 73, 74, /* AVX-512 registers 16-23 */
2183 75, 76, 77, 78, 79, 80, 81, 82, /* AVX-512 registers 24-31 */
2184 118, 119, 120, 121, 122, 123, 124, 125, /* Mask registers */
2185 126, 127, 128, 129, /* bound registers */
2188 /* Define the register numbers to be used in Dwarf debugging information.
2189 The SVR4 reference port C compiler uses the following register numbers
2190 in its Dwarf output code:
2191 0 for %eax (gcc regno = 0)
2192 1 for %ecx (gcc regno = 2)
2193 2 for %edx (gcc regno = 1)
2194 3 for %ebx (gcc regno = 3)
2195 4 for %esp (gcc regno = 7)
2196 5 for %ebp (gcc regno = 6)
2197 6 for %esi (gcc regno = 4)
2198 7 for %edi (gcc regno = 5)
2199 The following three DWARF register numbers are never generated by
2200 the SVR4 C compiler or by the GNU compilers, but SDB on x86/svr4
2201 believes these numbers have these meanings.
2202 8 for %eip (no gcc equivalent)
2203 9 for %eflags (gcc regno = 17)
2204 10 for %trapno (no gcc equivalent)
2205 It is not at all clear how we should number the FP stack registers
2206 for the x86 architecture. If the version of SDB on x86/svr4 were
2207 a bit less brain dead with respect to floating-point then we would
2208 have a precedent to follow with respect to DWARF register numbers
2209 for x86 FP registers, but the SDB on x86/svr4 is so completely
2210 broken with respect to FP registers that it is hardly worth thinking
2211 of it as something to strive for compatibility with.
2212 The version of x86/svr4 SDB I have at the moment does (partially)
2213 seem to believe that DWARF register number 11 is associated with
2214 the x86 register %st(0), but that's about all. Higher DWARF
2215 register numbers don't seem to be associated with anything in
2216 particular, and even for DWARF regno 11, SDB only seems to under-
2217 stand that it should say that a variable lives in %st(0) (when
2218 asked via an `=' command) if we said it was in DWARF regno 11,
2219 but SDB still prints garbage when asked for the value of the
2220 variable in question (via a `/' command).
2221 (Also note that the labels SDB prints for various FP stack regs
2222 when doing an `x' command are all wrong.)
2223 Note that these problems generally don't affect the native SVR4
2224 C compiler because it doesn't allow the use of -O with -g and
2225 because when it is *not* optimizing, it allocates a memory
2226 location for each floating-point variable, and the memory
2227 location is what gets described in the DWARF AT_location
2228 attribute for the variable in question.
2229 Regardless of the severe mental illness of the x86/svr4 SDB, we
2230 do something sensible here and we use the following DWARF
2231 register numbers. Note that these are all stack-top-relative
2233 11 for %st(0) (gcc regno = 8)
2234 12 for %st(1) (gcc regno = 9)
2235 13 for %st(2) (gcc regno = 10)
2236 14 for %st(3) (gcc regno = 11)
2237 15 for %st(4) (gcc regno = 12)
2238 16 for %st(5) (gcc regno = 13)
2239 17 for %st(6) (gcc regno = 14)
2240 18 for %st(7) (gcc regno = 15)
2242 int const svr4_dbx_register_map[FIRST_PSEUDO_REGISTER] =
2244 0, 2, 1, 3, 6, 7, 5, 4, /* general regs */
2245 11, 12, 13, 14, 15, 16, 17, 18, /* fp regs */
2246 -1, 9, -1, -1, -1, /* arg, flags, fpsr, fpcr, frame */
2247 21, 22, 23, 24, 25, 26, 27, 28, /* SSE registers */
2248 29, 30, 31, 32, 33, 34, 35, 36, /* MMX registers */
2249 -1, -1, -1, -1, -1, -1, -1, -1, /* extended integer registers */
2250 -1, -1, -1, -1, -1, -1, -1, -1, /* extended SSE registers */
2251 -1, -1, -1, -1, -1, -1, -1, -1, /* AVX-512 registers 16-23*/
2252 -1, -1, -1, -1, -1, -1, -1, -1, /* AVX-512 registers 24-31*/
2253 93, 94, 95, 96, 97, 98, 99, 100, /* Mask registers */
2254 101, 102, 103, 104, /* bound registers */
2257 /* Define parameter passing and return registers. */
2259 static int const x86_64_int_parameter_registers[6] =
2261 DI_REG, SI_REG, DX_REG, CX_REG, R8_REG, R9_REG
2264 static int const x86_64_ms_abi_int_parameter_registers[4] =
2266 CX_REG, DX_REG, R8_REG, R9_REG
2269 static int const x86_64_int_return_registers[4] =
2271 AX_REG, DX_REG, DI_REG, SI_REG
2274 /* Additional registers that are clobbered by SYSV calls. */
2276 int const x86_64_ms_sysv_extra_clobbered_registers[12] =
2280 XMM8_REG, XMM9_REG, XMM10_REG, XMM11_REG,
2281 XMM12_REG, XMM13_REG, XMM14_REG, XMM15_REG
2284 /* Define the structure for the machine field in struct function. */
2286 struct GTY(()) stack_local_entry {
2287 unsigned short mode;
2290 struct stack_local_entry *next;
2293 /* Structure describing stack frame layout.
2294 Stack grows downward:
2300 saved static chain if ix86_static_chain_on_stack
2302 saved frame pointer if frame_pointer_needed
2303 <- HARD_FRAME_POINTER
2309 <- sse_regs_save_offset
2312 [va_arg registers] |
2316 [padding2] | = to_allocate
2325 int outgoing_arguments_size;
2327 /* The offsets relative to ARG_POINTER. */
2328 HOST_WIDE_INT frame_pointer_offset;
2329 HOST_WIDE_INT hard_frame_pointer_offset;
2330 HOST_WIDE_INT stack_pointer_offset;
2331 HOST_WIDE_INT hfp_save_offset;
2332 HOST_WIDE_INT reg_save_offset;
2333 HOST_WIDE_INT sse_reg_save_offset;
2335 /* When save_regs_using_mov is set, emit prologue using
2336 move instead of push instructions. */
2337 bool save_regs_using_mov;
2340 /* Which cpu are we scheduling for. */
2341 enum attr_cpu ix86_schedule;
2343 /* Which cpu are we optimizing for. */
2344 enum processor_type ix86_tune;
2346 /* Which instruction set architecture to use. */
2347 enum processor_type ix86_arch;
2349 /* True if processor has SSE prefetch instruction. */
2350 unsigned char x86_prefetch_sse;
2352 /* -mstackrealign option */
2353 static const char ix86_force_align_arg_pointer_string[]
2354 = "force_align_arg_pointer";
2356 static rtx (*ix86_gen_leave) (void);
2357 static rtx (*ix86_gen_add3) (rtx, rtx, rtx);
2358 static rtx (*ix86_gen_sub3) (rtx, rtx, rtx);
2359 static rtx (*ix86_gen_sub3_carry) (rtx, rtx, rtx, rtx, rtx);
2360 static rtx (*ix86_gen_one_cmpl2) (rtx, rtx);
2361 static rtx (*ix86_gen_monitor) (rtx, rtx, rtx);
2362 static rtx (*ix86_gen_monitorx) (rtx, rtx, rtx);
2363 static rtx (*ix86_gen_andsp) (rtx, rtx, rtx);
2364 static rtx (*ix86_gen_allocate_stack_worker) (rtx, rtx);
2365 static rtx (*ix86_gen_adjust_stack_and_probe) (rtx, rtx, rtx);
2366 static rtx (*ix86_gen_probe_stack_range) (rtx, rtx, rtx);
2367 static rtx (*ix86_gen_tls_global_dynamic_64) (rtx, rtx, rtx);
2368 static rtx (*ix86_gen_tls_local_dynamic_base_64) (rtx, rtx);
2370 /* Preferred alignment for stack boundary in bits. */
2371 unsigned int ix86_preferred_stack_boundary;
2373 /* Alignment for incoming stack boundary in bits specified at
2375 static unsigned int ix86_user_incoming_stack_boundary;
2377 /* Default alignment for incoming stack boundary in bits. */
2378 static unsigned int ix86_default_incoming_stack_boundary;
2380 /* Alignment for incoming stack boundary in bits. */
2381 unsigned int ix86_incoming_stack_boundary;
2383 /* Calling abi specific va_list type nodes. */
2384 static GTY(()) tree sysv_va_list_type_node;
2385 static GTY(()) tree ms_va_list_type_node;
2387 /* Prefix built by ASM_GENERATE_INTERNAL_LABEL. */
2388 char internal_label_prefix[16];
2389 int internal_label_prefix_len;
2391 /* Fence to use after loop using movnt. */
2394 /* Register class used for passing given 64bit part of the argument.
2395 These represent classes as documented by the PS ABI, with the exception
2396 of SSESF, SSEDF classes, that are basically SSE class, just gcc will
2397 use SF or DFmode move instead of DImode to avoid reformatting penalties.
2399 Similarly we play games with INTEGERSI_CLASS to use cheaper SImode moves
2400 whenever possible (upper half does contain padding). */
2401 enum x86_64_reg_class
2404 X86_64_INTEGER_CLASS,
2405 X86_64_INTEGERSI_CLASS,
2412 X86_64_COMPLEX_X87_CLASS,
2416 #define MAX_CLASSES 8
2418 /* Table of constants used by fldpi, fldln2, etc.... */
2419 static REAL_VALUE_TYPE ext_80387_constants_table [5];
2420 static bool ext_80387_constants_init = 0;
2423 static struct machine_function * ix86_init_machine_status (void);
2424 static rtx ix86_function_value (const_tree, const_tree, bool);
2425 static bool ix86_function_value_regno_p (const unsigned int);
2426 static unsigned int ix86_function_arg_boundary (machine_mode,
2428 static rtx ix86_static_chain (const_tree, bool);
2429 static int ix86_function_regparm (const_tree, const_tree);
2430 static void ix86_compute_frame_layout (struct ix86_frame *);
2431 static bool ix86_expand_vector_init_one_nonzero (bool, machine_mode,
2433 static void ix86_add_new_builtins (HOST_WIDE_INT);
2434 static tree ix86_canonical_va_list_type (tree);
2435 static void predict_jump (int);
2436 static unsigned int split_stack_prologue_scratch_regno (void);
2437 static bool i386_asm_output_addr_const_extra (FILE *, rtx);
2439 enum ix86_function_specific_strings
2441 IX86_FUNCTION_SPECIFIC_ARCH,
2442 IX86_FUNCTION_SPECIFIC_TUNE,
2443 IX86_FUNCTION_SPECIFIC_MAX
2446 static char *ix86_target_string (HOST_WIDE_INT, int, const char *,
2447 const char *, enum fpmath_unit, bool);
2448 static void ix86_function_specific_save (struct cl_target_option *,
2449 struct gcc_options *opts);
2450 static void ix86_function_specific_restore (struct gcc_options *opts,
2451 struct cl_target_option *);
2452 static void ix86_function_specific_post_stream_in (struct cl_target_option *);
2453 static void ix86_function_specific_print (FILE *, int,
2454 struct cl_target_option *);
2455 static bool ix86_valid_target_attribute_p (tree, tree, tree, int);
2456 static bool ix86_valid_target_attribute_inner_p (tree, char *[],
2457 struct gcc_options *,
2458 struct gcc_options *,
2459 struct gcc_options *);
2460 static bool ix86_can_inline_p (tree, tree);
2461 static void ix86_set_current_function (tree);
2462 static unsigned int ix86_minimum_incoming_stack_boundary (bool);
2464 static enum calling_abi ix86_function_abi (const_tree);
2467 #ifndef SUBTARGET32_DEFAULT_CPU
2468 #define SUBTARGET32_DEFAULT_CPU "i386"
2471 /* Whether -mtune= or -march= were specified */
2472 static int ix86_tune_defaulted;
2473 static int ix86_arch_specified;
2475 /* Vectorization library interface and handlers. */
2476 static tree (*ix86_veclib_handler) (enum built_in_function, tree, tree);
2478 static tree ix86_veclibabi_svml (enum built_in_function, tree, tree);
2479 static tree ix86_veclibabi_acml (enum built_in_function, tree, tree);
2481 /* Processor target table, indexed by processor number */
2484 const char *const name; /* processor name */
2485 const struct processor_costs *cost; /* Processor costs */
2486 const int align_loop; /* Default alignments. */
2487 const int align_loop_max_skip;
2488 const int align_jump;
2489 const int align_jump_max_skip;
2490 const int align_func;
2493 /* This table must be in sync with enum processor_type in i386.h. */
2494 static const struct ptt processor_target_table[PROCESSOR_max] =
2496 {"generic", &generic_cost, 16, 10, 16, 10, 16},
2497 {"i386", &i386_cost, 4, 3, 4, 3, 4},
2498 {"i486", &i486_cost, 16, 15, 16, 15, 16},
2499 {"pentium", &pentium_cost, 16, 7, 16, 7, 16},
2500 {"pentiumpro", &pentiumpro_cost, 16, 15, 16, 10, 16},
2501 {"pentium4", &pentium4_cost, 0, 0, 0, 0, 0},
2502 {"nocona", &nocona_cost, 0, 0, 0, 0, 0},
2503 {"core2", &core_cost, 16, 10, 16, 10, 16},
2504 {"nehalem", &core_cost, 16, 10, 16, 10, 16},
2505 {"sandybridge", &core_cost, 16, 10, 16, 10, 16},
2506 {"haswell", &core_cost, 16, 10, 16, 10, 16},
2507 {"bonnell", &atom_cost, 16, 15, 16, 7, 16},
2508 {"silvermont", &slm_cost, 16, 15, 16, 7, 16},
2509 {"knl", &slm_cost, 16, 15, 16, 7, 16},
2510 {"intel", &intel_cost, 16, 15, 16, 7, 16},
2511 {"geode", &geode_cost, 0, 0, 0, 0, 0},
2512 {"k6", &k6_cost, 32, 7, 32, 7, 32},
2513 {"athlon", &athlon_cost, 16, 7, 16, 7, 16},
2514 {"k8", &k8_cost, 16, 7, 16, 7, 16},
2515 {"amdfam10", &amdfam10_cost, 32, 24, 32, 7, 32},
2516 {"bdver1", &bdver1_cost, 16, 10, 16, 7, 11},
2517 {"bdver2", &bdver2_cost, 16, 10, 16, 7, 11},
2518 {"bdver3", &bdver3_cost, 16, 10, 16, 7, 11},
2519 {"bdver4", &bdver4_cost, 16, 10, 16, 7, 11},
2520 {"btver1", &btver1_cost, 16, 10, 16, 7, 11},
2521 {"btver2", &btver2_cost, 16, 10, 16, 7, 11}
2525 rest_of_handle_insert_vzeroupper (void)
2529 /* vzeroupper instructions are inserted immediately after reload to
2530 account for possible spills from 256bit registers. The pass
2531 reuses mode switching infrastructure by re-running mode insertion
2532 pass, so disable entities that have already been processed. */
2533 for (i = 0; i < MAX_386_ENTITIES; i++)
2534 ix86_optimize_mode_switching[i] = 0;
2536 ix86_optimize_mode_switching[AVX_U128] = 1;
2538 /* Call optimize_mode_switching. */
2539 g->get_passes ()->execute_pass_mode_switching ();
2545 const pass_data pass_data_insert_vzeroupper =
2547 RTL_PASS, /* type */
2548 "vzeroupper", /* name */
2549 OPTGROUP_NONE, /* optinfo_flags */
2550 TV_NONE, /* tv_id */
2551 0, /* properties_required */
2552 0, /* properties_provided */
2553 0, /* properties_destroyed */
2554 0, /* todo_flags_start */
2555 TODO_df_finish, /* todo_flags_finish */
2558 class pass_insert_vzeroupper : public rtl_opt_pass
2561 pass_insert_vzeroupper(gcc::context *ctxt)
2562 : rtl_opt_pass(pass_data_insert_vzeroupper, ctxt)
2565 /* opt_pass methods: */
2566 virtual bool gate (function *)
2568 return TARGET_AVX && !TARGET_AVX512F
2569 && TARGET_VZEROUPPER && flag_expensive_optimizations
2573 virtual unsigned int execute (function *)
2575 return rest_of_handle_insert_vzeroupper ();
2578 }; // class pass_insert_vzeroupper
2583 make_pass_insert_vzeroupper (gcc::context *ctxt)
2585 return new pass_insert_vzeroupper (ctxt);
2588 /* Return true if a red-zone is in use. */
2591 ix86_using_red_zone (void)
2593 return TARGET_RED_ZONE && !TARGET_64BIT_MS_ABI;
2596 /* Return a string that documents the current -m options. The caller is
2597 responsible for freeing the string. */
2600 ix86_target_string (HOST_WIDE_INT isa, int flags, const char *arch,
2601 const char *tune, enum fpmath_unit fpmath,
2604 struct ix86_target_opts
2606 const char *option; /* option string */
2607 HOST_WIDE_INT mask; /* isa mask options */
2610 /* This table is ordered so that options like -msse4.2 that imply
2611 preceding options while match those first. */
2612 static struct ix86_target_opts isa_opts[] =
2614 { "-mfma4", OPTION_MASK_ISA_FMA4 },
2615 { "-mfma", OPTION_MASK_ISA_FMA },
2616 { "-mxop", OPTION_MASK_ISA_XOP },
2617 { "-mlwp", OPTION_MASK_ISA_LWP },
2618 { "-mavx512f", OPTION_MASK_ISA_AVX512F },
2619 { "-mavx512er", OPTION_MASK_ISA_AVX512ER },
2620 { "-mavx512cd", OPTION_MASK_ISA_AVX512CD },
2621 { "-mavx512pf", OPTION_MASK_ISA_AVX512PF },
2622 { "-mavx512dq", OPTION_MASK_ISA_AVX512DQ },
2623 { "-mavx512bw", OPTION_MASK_ISA_AVX512BW },
2624 { "-mavx512vl", OPTION_MASK_ISA_AVX512VL },
2625 { "-mavx512ifma", OPTION_MASK_ISA_AVX512IFMA },
2626 { "-mavx512vbmi", OPTION_MASK_ISA_AVX512VBMI },
2627 { "-msse4a", OPTION_MASK_ISA_SSE4A },
2628 { "-msse4.2", OPTION_MASK_ISA_SSE4_2 },
2629 { "-msse4.1", OPTION_MASK_ISA_SSE4_1 },
2630 { "-mssse3", OPTION_MASK_ISA_SSSE3 },
2631 { "-msse3", OPTION_MASK_ISA_SSE3 },
2632 { "-msse2", OPTION_MASK_ISA_SSE2 },
2633 { "-msse", OPTION_MASK_ISA_SSE },
2634 { "-m3dnow", OPTION_MASK_ISA_3DNOW },
2635 { "-m3dnowa", OPTION_MASK_ISA_3DNOW_A },
2636 { "-mmmx", OPTION_MASK_ISA_MMX },
2637 { "-mabm", OPTION_MASK_ISA_ABM },
2638 { "-mbmi", OPTION_MASK_ISA_BMI },
2639 { "-mbmi2", OPTION_MASK_ISA_BMI2 },
2640 { "-mlzcnt", OPTION_MASK_ISA_LZCNT },
2641 { "-mhle", OPTION_MASK_ISA_HLE },
2642 { "-mfxsr", OPTION_MASK_ISA_FXSR },
2643 { "-mrdseed", OPTION_MASK_ISA_RDSEED },
2644 { "-mprfchw", OPTION_MASK_ISA_PRFCHW },
2645 { "-madx", OPTION_MASK_ISA_ADX },
2646 { "-mtbm", OPTION_MASK_ISA_TBM },
2647 { "-mpopcnt", OPTION_MASK_ISA_POPCNT },
2648 { "-mmovbe", OPTION_MASK_ISA_MOVBE },
2649 { "-mcrc32", OPTION_MASK_ISA_CRC32 },
2650 { "-maes", OPTION_MASK_ISA_AES },
2651 { "-msha", OPTION_MASK_ISA_SHA },
2652 { "-mpclmul", OPTION_MASK_ISA_PCLMUL },
2653 { "-mfsgsbase", OPTION_MASK_ISA_FSGSBASE },
2654 { "-mrdrnd", OPTION_MASK_ISA_RDRND },
2655 { "-mf16c", OPTION_MASK_ISA_F16C },
2656 { "-mrtm", OPTION_MASK_ISA_RTM },
2657 { "-mxsave", OPTION_MASK_ISA_XSAVE },
2658 { "-mxsaveopt", OPTION_MASK_ISA_XSAVEOPT },
2659 { "-mprefetchwt1", OPTION_MASK_ISA_PREFETCHWT1 },
2660 { "-mclflushopt", OPTION_MASK_ISA_CLFLUSHOPT },
2661 { "-mxsavec", OPTION_MASK_ISA_XSAVEC },
2662 { "-mxsaves", OPTION_MASK_ISA_XSAVES },
2663 { "-mmpx", OPTION_MASK_ISA_MPX },
2664 { "-mclwb", OPTION_MASK_ISA_CLWB },
2665 { "-mpcommit", OPTION_MASK_ISA_PCOMMIT },
2666 { "-mmwaitx", OPTION_MASK_ISA_MWAITX },
2670 static struct ix86_target_opts flag_opts[] =
2672 { "-m128bit-long-double", MASK_128BIT_LONG_DOUBLE },
2673 { "-mlong-double-128", MASK_LONG_DOUBLE_128 },
2674 { "-mlong-double-64", MASK_LONG_DOUBLE_64 },
2675 { "-m80387", MASK_80387 },
2676 { "-maccumulate-outgoing-args", MASK_ACCUMULATE_OUTGOING_ARGS },
2677 { "-malign-double", MASK_ALIGN_DOUBLE },
2678 { "-mcld", MASK_CLD },
2679 { "-mfp-ret-in-387", MASK_FLOAT_RETURNS },
2680 { "-mieee-fp", MASK_IEEE_FP },
2681 { "-minline-all-stringops", MASK_INLINE_ALL_STRINGOPS },
2682 { "-minline-stringops-dynamically", MASK_INLINE_STRINGOPS_DYNAMICALLY },
2683 { "-mms-bitfields", MASK_MS_BITFIELD_LAYOUT },
2684 { "-mno-align-stringops", MASK_NO_ALIGN_STRINGOPS },
2685 { "-mno-fancy-math-387", MASK_NO_FANCY_MATH_387 },
2686 { "-mno-push-args", MASK_NO_PUSH_ARGS },
2687 { "-mno-red-zone", MASK_NO_RED_ZONE },
2688 { "-momit-leaf-frame-pointer", MASK_OMIT_LEAF_FRAME_POINTER },
2689 { "-mrecip", MASK_RECIP },
2690 { "-mrtd", MASK_RTD },
2691 { "-msseregparm", MASK_SSEREGPARM },
2692 { "-mstack-arg-probe", MASK_STACK_PROBE },
2693 { "-mtls-direct-seg-refs", MASK_TLS_DIRECT_SEG_REFS },
2694 { "-mvect8-ret-in-mem", MASK_VECT8_RETURNS },
2695 { "-m8bit-idiv", MASK_USE_8BIT_IDIV },
2696 { "-mvzeroupper", MASK_VZEROUPPER },
2697 { "-mavx256-split-unaligned-load", MASK_AVX256_SPLIT_UNALIGNED_LOAD},
2698 { "-mavx256-split-unaligned-store", MASK_AVX256_SPLIT_UNALIGNED_STORE},
2699 { "-mprefer-avx128", MASK_PREFER_AVX128},
2702 const char *opts[ARRAY_SIZE (isa_opts) + ARRAY_SIZE (flag_opts) + 6][2];
2705 char target_other[40];
2715 memset (opts, '\0', sizeof (opts));
2717 /* Add -march= option. */
2720 opts[num][0] = "-march=";
2721 opts[num++][1] = arch;
2724 /* Add -mtune= option. */
2727 opts[num][0] = "-mtune=";
2728 opts[num++][1] = tune;
2731 /* Add -m32/-m64/-mx32. */
2732 if ((isa & OPTION_MASK_ISA_64BIT) != 0)
2734 if ((isa & OPTION_MASK_ABI_64) != 0)
2738 isa &= ~ (OPTION_MASK_ISA_64BIT
2739 | OPTION_MASK_ABI_64
2740 | OPTION_MASK_ABI_X32);
2744 opts[num++][0] = abi;
2746 /* Pick out the options in isa options. */
2747 for (i = 0; i < ARRAY_SIZE (isa_opts); i++)
2749 if ((isa & isa_opts[i].mask) != 0)
2751 opts[num++][0] = isa_opts[i].option;
2752 isa &= ~ isa_opts[i].mask;
2756 if (isa && add_nl_p)
2758 opts[num++][0] = isa_other;
2759 sprintf (isa_other, "(other isa: %#" HOST_WIDE_INT_PRINT "x)",
2763 /* Add flag options. */
2764 for (i = 0; i < ARRAY_SIZE (flag_opts); i++)
2766 if ((flags & flag_opts[i].mask) != 0)
2768 opts[num++][0] = flag_opts[i].option;
2769 flags &= ~ flag_opts[i].mask;
2773 if (flags && add_nl_p)
2775 opts[num++][0] = target_other;
2776 sprintf (target_other, "(other flags: %#x)", flags);
2779 /* Add -fpmath= option. */
2782 opts[num][0] = "-mfpmath=";
2783 switch ((int) fpmath)
2786 opts[num++][1] = "387";
2790 opts[num++][1] = "sse";
2793 case FPMATH_387 | FPMATH_SSE:
2794 opts[num++][1] = "sse+387";
2806 gcc_assert (num < ARRAY_SIZE (opts));
2808 /* Size the string. */
2810 sep_len = (add_nl_p) ? 3 : 1;
2811 for (i = 0; i < num; i++)
2814 for (j = 0; j < 2; j++)
2816 len += strlen (opts[i][j]);
2819 /* Build the string. */
2820 ret = ptr = (char *) xmalloc (len);
2823 for (i = 0; i < num; i++)
2827 for (j = 0; j < 2; j++)
2828 len2[j] = (opts[i][j]) ? strlen (opts[i][j]) : 0;
2835 if (add_nl_p && line_len + len2[0] + len2[1] > 70)
2843 for (j = 0; j < 2; j++)
2846 memcpy (ptr, opts[i][j], len2[j]);
2848 line_len += len2[j];
2853 gcc_assert (ret + len >= ptr);
2858 /* Return true, if profiling code should be emitted before
2859 prologue. Otherwise it returns false.
2860 Note: For x86 with "hotfix" it is sorried. */
2862 ix86_profile_before_prologue (void)
2864 return flag_fentry != 0;
2867 /* Function that is callable from the debugger to print the current
2869 void ATTRIBUTE_UNUSED
2870 ix86_debug_options (void)
2872 char *opts = ix86_target_string (ix86_isa_flags, target_flags,
2873 ix86_arch_string, ix86_tune_string,
2878 fprintf (stderr, "%s\n\n", opts);
2882 fputs ("<no options>\n\n", stderr);
2887 static const char *stringop_alg_names[] = {
2889 #define DEF_ALG(alg, name) #name,
2890 #include "stringop.def"
2895 /* Parse parameter string passed to -mmemcpy-strategy= or -mmemset-strategy=.
2896 The string is of the following form (or comma separated list of it):
2898 strategy_alg:max_size:[align|noalign]
2900 where the full size range for the strategy is either [0, max_size] or
2901 [min_size, max_size], in which min_size is the max_size + 1 of the
2902 preceding range. The last size range must have max_size == -1.
2907 -mmemcpy-strategy=libcall:-1:noalign
2909 this is equivalent to (for known size memcpy) -mstringop-strategy=libcall
2913 -mmemset-strategy=rep_8byte:16:noalign,vector_loop:2048:align,libcall:-1:noalign
2915 This is to tell the compiler to use the following strategy for memset
2916 1) when the expected size is between [1, 16], use rep_8byte strategy;
2917 2) when the size is between [17, 2048], use vector_loop;
2918 3) when the size is > 2048, use libcall. */
2920 struct stringop_size_range
2928 ix86_parse_stringop_strategy_string (char *strategy_str, bool is_memset)
2930 const struct stringop_algs *default_algs;
2931 stringop_size_range input_ranges[MAX_STRINGOP_ALGS];
2932 char *curr_range_str, *next_range_str;
2936 default_algs = &ix86_cost->memset[TARGET_64BIT != 0];
2938 default_algs = &ix86_cost->memcpy[TARGET_64BIT != 0];
2940 curr_range_str = strategy_str;
2947 next_range_str = strchr (curr_range_str, ',');
2949 *next_range_str++ = '\0';
2951 if (3 != sscanf (curr_range_str, "%20[^:]:%d:%10s",
2952 alg_name, &maxs, align))
2954 error ("wrong arg %s to option %s", curr_range_str,
2955 is_memset ? "-mmemset_strategy=" : "-mmemcpy_strategy=");
2959 if (n > 0 && (maxs < (input_ranges[n - 1].max + 1) && maxs != -1))
2961 error ("size ranges of option %s should be increasing",
2962 is_memset ? "-mmemset_strategy=" : "-mmemcpy_strategy=");
2966 for (i = 0; i < last_alg; i++)
2967 if (!strcmp (alg_name, stringop_alg_names[i]))
2972 error ("wrong stringop strategy name %s specified for option %s",
2974 is_memset ? "-mmemset_strategy=" : "-mmemcpy_strategy=");
2978 if ((stringop_alg) i == rep_prefix_8_byte
2981 /* rep; movq isn't available in 32-bit code. */
2982 error ("stringop strategy name %s specified for option %s "
2983 "not supported for 32-bit code",
2985 is_memset ? "-mmemset_strategy=" : "-mmemcpy_strategy=");
2989 input_ranges[n].max = maxs;
2990 input_ranges[n].alg = (stringop_alg) i;
2991 if (!strcmp (align, "align"))
2992 input_ranges[n].noalign = false;
2993 else if (!strcmp (align, "noalign"))
2994 input_ranges[n].noalign = true;
2997 error ("unknown alignment %s specified for option %s",
2998 align, is_memset ? "-mmemset_strategy=" : "-mmemcpy_strategy=");
3002 curr_range_str = next_range_str;
3004 while (curr_range_str);
3006 if (input_ranges[n - 1].max != -1)
3008 error ("the max value for the last size range should be -1"
3010 is_memset ? "-mmemset_strategy=" : "-mmemcpy_strategy=");
3014 if (n > MAX_STRINGOP_ALGS)
3016 error ("too many size ranges specified in option %s",
3017 is_memset ? "-mmemset_strategy=" : "-mmemcpy_strategy=");
3021 /* Now override the default algs array. */
3022 for (i = 0; i < n; i++)
3024 *const_cast<int *>(&default_algs->size[i].max) = input_ranges[i].max;
3025 *const_cast<stringop_alg *>(&default_algs->size[i].alg)
3026 = input_ranges[i].alg;
3027 *const_cast<int *>(&default_algs->size[i].noalign)
3028 = input_ranges[i].noalign;
3033 /* parse -mtune-ctrl= option. When DUMP is true,
3034 print the features that are explicitly set. */
3037 parse_mtune_ctrl_str (bool dump)
3039 if (!ix86_tune_ctrl_string)
3042 char *next_feature_string = NULL;
3043 char *curr_feature_string = xstrdup (ix86_tune_ctrl_string);
3044 char *orig = curr_feature_string;
3050 next_feature_string = strchr (curr_feature_string, ',');
3051 if (next_feature_string)
3052 *next_feature_string++ = '\0';
3053 if (*curr_feature_string == '^')
3055 curr_feature_string++;
3058 for (i = 0; i < X86_TUNE_LAST; i++)
3060 if (!strcmp (curr_feature_string, ix86_tune_feature_names[i]))
3062 ix86_tune_features[i] = !clear;
3064 fprintf (stderr, "Explicitly %s feature %s\n",
3065 clear ? "clear" : "set", ix86_tune_feature_names[i]);
3069 if (i == X86_TUNE_LAST)
3070 error ("Unknown parameter to option -mtune-ctrl: %s",
3071 clear ? curr_feature_string - 1 : curr_feature_string);
3072 curr_feature_string = next_feature_string;
3074 while (curr_feature_string);
3078 /* Helper function to set ix86_tune_features. IX86_TUNE is the
3082 set_ix86_tune_features (enum processor_type ix86_tune, bool dump)
3084 unsigned int ix86_tune_mask = 1u << ix86_tune;
3087 for (i = 0; i < X86_TUNE_LAST; ++i)
3089 if (ix86_tune_no_default)
3090 ix86_tune_features[i] = 0;
3092 ix86_tune_features[i] = !!(initial_ix86_tune_features[i] & ix86_tune_mask);
3097 fprintf (stderr, "List of x86 specific tuning parameter names:\n");
3098 for (i = 0; i < X86_TUNE_LAST; i++)
3099 fprintf (stderr, "%s : %s\n", ix86_tune_feature_names[i],
3100 ix86_tune_features[i] ? "on" : "off");
3103 parse_mtune_ctrl_str (dump);
3107 /* Default align_* from the processor table. */
3110 ix86_default_align (struct gcc_options *opts)
3112 if (opts->x_align_loops == 0)
3114 opts->x_align_loops = processor_target_table[ix86_tune].align_loop;
3115 align_loops_max_skip = processor_target_table[ix86_tune].align_loop_max_skip;
3117 if (opts->x_align_jumps == 0)
3119 opts->x_align_jumps = processor_target_table[ix86_tune].align_jump;
3120 align_jumps_max_skip = processor_target_table[ix86_tune].align_jump_max_skip;
3122 if (opts->x_align_functions == 0)
3124 opts->x_align_functions = processor_target_table[ix86_tune].align_func;
3128 /* Implement TARGET_OVERRIDE_OPTIONS_AFTER_CHANGE hook. */
3131 ix86_override_options_after_change (void)
3133 ix86_default_align (&global_options);
3136 /* Override various settings based on options. If MAIN_ARGS_P, the
3137 options are from the command line, otherwise they are from
3141 ix86_option_override_internal (bool main_args_p,
3142 struct gcc_options *opts,
3143 struct gcc_options *opts_set)
3146 unsigned int ix86_arch_mask;
3147 const bool ix86_tune_specified = (opts->x_ix86_tune_string != NULL);
3152 #define PTA_3DNOW (HOST_WIDE_INT_1 << 0)
3153 #define PTA_3DNOW_A (HOST_WIDE_INT_1 << 1)
3154 #define PTA_64BIT (HOST_WIDE_INT_1 << 2)
3155 #define PTA_ABM (HOST_WIDE_INT_1 << 3)
3156 #define PTA_AES (HOST_WIDE_INT_1 << 4)
3157 #define PTA_AVX (HOST_WIDE_INT_1 << 5)
3158 #define PTA_BMI (HOST_WIDE_INT_1 << 6)
3159 #define PTA_CX16 (HOST_WIDE_INT_1 << 7)
3160 #define PTA_F16C (HOST_WIDE_INT_1 << 8)
3161 #define PTA_FMA (HOST_WIDE_INT_1 << 9)
3162 #define PTA_FMA4 (HOST_WIDE_INT_1 << 10)
3163 #define PTA_FSGSBASE (HOST_WIDE_INT_1 << 11)
3164 #define PTA_LWP (HOST_WIDE_INT_1 << 12)
3165 #define PTA_LZCNT (HOST_WIDE_INT_1 << 13)
3166 #define PTA_MMX (HOST_WIDE_INT_1 << 14)
3167 #define PTA_MOVBE (HOST_WIDE_INT_1 << 15)
3168 #define PTA_NO_SAHF (HOST_WIDE_INT_1 << 16)
3169 #define PTA_PCLMUL (HOST_WIDE_INT_1 << 17)
3170 #define PTA_POPCNT (HOST_WIDE_INT_1 << 18)
3171 #define PTA_PREFETCH_SSE (HOST_WIDE_INT_1 << 19)
3172 #define PTA_RDRND (HOST_WIDE_INT_1 << 20)
3173 #define PTA_SSE (HOST_WIDE_INT_1 << 21)
3174 #define PTA_SSE2 (HOST_WIDE_INT_1 << 22)
3175 #define PTA_SSE3 (HOST_WIDE_INT_1 << 23)
3176 #define PTA_SSE4_1 (HOST_WIDE_INT_1 << 24)
3177 #define PTA_SSE4_2 (HOST_WIDE_INT_1 << 25)
3178 #define PTA_SSE4A (HOST_WIDE_INT_1 << 26)
3179 #define PTA_SSSE3 (HOST_WIDE_INT_1 << 27)
3180 #define PTA_TBM (HOST_WIDE_INT_1 << 28)
3181 #define PTA_XOP (HOST_WIDE_INT_1 << 29)
3182 #define PTA_AVX2 (HOST_WIDE_INT_1 << 30)
3183 #define PTA_BMI2 (HOST_WIDE_INT_1 << 31)
3184 #define PTA_RTM (HOST_WIDE_INT_1 << 32)
3185 #define PTA_HLE (HOST_WIDE_INT_1 << 33)
3186 #define PTA_PRFCHW (HOST_WIDE_INT_1 << 34)
3187 #define PTA_RDSEED (HOST_WIDE_INT_1 << 35)
3188 #define PTA_ADX (HOST_WIDE_INT_1 << 36)
3189 #define PTA_FXSR (HOST_WIDE_INT_1 << 37)
3190 #define PTA_XSAVE (HOST_WIDE_INT_1 << 38)
3191 #define PTA_XSAVEOPT (HOST_WIDE_INT_1 << 39)
3192 #define PTA_AVX512F (HOST_WIDE_INT_1 << 40)
3193 #define PTA_AVX512ER (HOST_WIDE_INT_1 << 41)
3194 #define PTA_AVX512PF (HOST_WIDE_INT_1 << 42)
3195 #define PTA_AVX512CD (HOST_WIDE_INT_1 << 43)
3196 #define PTA_MPX (HOST_WIDE_INT_1 << 44)
3197 #define PTA_SHA (HOST_WIDE_INT_1 << 45)
3198 #define PTA_PREFETCHWT1 (HOST_WIDE_INT_1 << 46)
3199 #define PTA_CLFLUSHOPT (HOST_WIDE_INT_1 << 47)
3200 #define PTA_XSAVEC (HOST_WIDE_INT_1 << 48)
3201 #define PTA_XSAVES (HOST_WIDE_INT_1 << 49)
3202 #define PTA_AVX512DQ (HOST_WIDE_INT_1 << 50)
3203 #define PTA_AVX512BW (HOST_WIDE_INT_1 << 51)
3204 #define PTA_AVX512VL (HOST_WIDE_INT_1 << 52)
3205 #define PTA_AVX512IFMA (HOST_WIDE_INT_1 << 53)
3206 #define PTA_AVX512VBMI (HOST_WIDE_INT_1 << 54)
3207 #define PTA_CLWB (HOST_WIDE_INT_1 << 55)
3208 #define PTA_PCOMMIT (HOST_WIDE_INT_1 << 56)
3209 #define PTA_MWAITX (HOST_WIDE_INT_1 << 57)
3212 (PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 | PTA_SSSE3 \
3213 | PTA_CX16 | PTA_FXSR)
3214 #define PTA_NEHALEM \
3215 (PTA_CORE2 | PTA_SSE4_1 | PTA_SSE4_2 | PTA_POPCNT)
3216 #define PTA_WESTMERE \
3217 (PTA_NEHALEM | PTA_AES | PTA_PCLMUL)
3218 #define PTA_SANDYBRIDGE \
3219 (PTA_WESTMERE | PTA_AVX | PTA_XSAVE | PTA_XSAVEOPT)
3220 #define PTA_IVYBRIDGE \
3221 (PTA_SANDYBRIDGE | PTA_FSGSBASE | PTA_RDRND | PTA_F16C)
3222 #define PTA_HASWELL \
3223 (PTA_IVYBRIDGE | PTA_AVX2 | PTA_BMI | PTA_BMI2 | PTA_LZCNT \
3224 | PTA_FMA | PTA_MOVBE | PTA_HLE)
3225 #define PTA_BROADWELL \
3226 (PTA_HASWELL | PTA_ADX | PTA_PRFCHW | PTA_RDSEED)
3228 (PTA_BROADWELL | PTA_AVX512PF | PTA_AVX512ER | PTA_AVX512F | PTA_AVX512CD)
3229 #define PTA_BONNELL \
3230 (PTA_CORE2 | PTA_MOVBE)
3231 #define PTA_SILVERMONT \
3232 (PTA_WESTMERE | PTA_MOVBE)
3234 /* if this reaches 64, need to widen struct pta flags below */
3238 const char *const name; /* processor name or nickname. */
3239 const enum processor_type processor;
3240 const enum attr_cpu schedule;
3241 const unsigned HOST_WIDE_INT flags;
3243 const processor_alias_table[] =
3245 {"i386", PROCESSOR_I386, CPU_NONE, 0},
3246 {"i486", PROCESSOR_I486, CPU_NONE, 0},
3247 {"i586", PROCESSOR_PENTIUM, CPU_PENTIUM, 0},
3248 {"pentium", PROCESSOR_PENTIUM, CPU_PENTIUM, 0},
3249 {"pentium-mmx", PROCESSOR_PENTIUM, CPU_PENTIUM, PTA_MMX},
3250 {"winchip-c6", PROCESSOR_I486, CPU_NONE, PTA_MMX},
3251 {"winchip2", PROCESSOR_I486, CPU_NONE, PTA_MMX | PTA_3DNOW | PTA_PRFCHW},
3252 {"c3", PROCESSOR_I486, CPU_NONE, PTA_MMX | PTA_3DNOW | PTA_PRFCHW},
3253 {"c3-2", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO,
3254 PTA_MMX | PTA_SSE | PTA_FXSR},
3255 {"i686", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO, 0},
3256 {"pentiumpro", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO, 0},
3257 {"pentium2", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO, PTA_MMX | PTA_FXSR},
3258 {"pentium3", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO,
3259 PTA_MMX | PTA_SSE | PTA_FXSR},
3260 {"pentium3m", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO,
3261 PTA_MMX | PTA_SSE | PTA_FXSR},
3262 {"pentium-m", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO,
3263 PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_FXSR},
3264 {"pentium4", PROCESSOR_PENTIUM4, CPU_NONE,
3265 PTA_MMX |PTA_SSE | PTA_SSE2 | PTA_FXSR},
3266 {"pentium4m", PROCESSOR_PENTIUM4, CPU_NONE,
3267 PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_FXSR},
3268 {"prescott", PROCESSOR_NOCONA, CPU_NONE,
3269 PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 | PTA_FXSR},
3270 {"nocona", PROCESSOR_NOCONA, CPU_NONE,
3271 PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
3272 | PTA_CX16 | PTA_NO_SAHF | PTA_FXSR},
3273 {"core2", PROCESSOR_CORE2, CPU_CORE2, PTA_CORE2},
3274 {"nehalem", PROCESSOR_NEHALEM, CPU_NEHALEM, PTA_NEHALEM},
3275 {"corei7", PROCESSOR_NEHALEM, CPU_NEHALEM, PTA_NEHALEM},
3276 {"westmere", PROCESSOR_NEHALEM, CPU_NEHALEM, PTA_WESTMERE},
3277 {"sandybridge", PROCESSOR_SANDYBRIDGE, CPU_NEHALEM,
3279 {"corei7-avx", PROCESSOR_SANDYBRIDGE, CPU_NEHALEM,
3281 {"ivybridge", PROCESSOR_SANDYBRIDGE, CPU_NEHALEM,
3283 {"core-avx-i", PROCESSOR_SANDYBRIDGE, CPU_NEHALEM,
3285 {"haswell", PROCESSOR_HASWELL, CPU_NEHALEM, PTA_HASWELL},
3286 {"core-avx2", PROCESSOR_HASWELL, CPU_NEHALEM, PTA_HASWELL},
3287 {"broadwell", PROCESSOR_HASWELL, CPU_NEHALEM, PTA_BROADWELL},
3288 {"bonnell", PROCESSOR_BONNELL, CPU_ATOM, PTA_BONNELL},
3289 {"atom", PROCESSOR_BONNELL, CPU_ATOM, PTA_BONNELL},
3290 {"silvermont", PROCESSOR_SILVERMONT, CPU_SLM, PTA_SILVERMONT},
3291 {"slm", PROCESSOR_SILVERMONT, CPU_SLM, PTA_SILVERMONT},
3292 {"knl", PROCESSOR_KNL, CPU_KNL, PTA_KNL},
3293 {"intel", PROCESSOR_INTEL, CPU_SLM, PTA_NEHALEM},
3294 {"geode", PROCESSOR_GEODE, CPU_GEODE,
3295 PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_PREFETCH_SSE | PTA_PRFCHW},
3296 {"k6", PROCESSOR_K6, CPU_K6, PTA_MMX},
3297 {"k6-2", PROCESSOR_K6, CPU_K6, PTA_MMX | PTA_3DNOW | PTA_PRFCHW},
3298 {"k6-3", PROCESSOR_K6, CPU_K6, PTA_MMX | PTA_3DNOW | PTA_PRFCHW},
3299 {"athlon", PROCESSOR_ATHLON, CPU_ATHLON,
3300 PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_PREFETCH_SSE | PTA_PRFCHW},
3301 {"athlon-tbird", PROCESSOR_ATHLON, CPU_ATHLON,
3302 PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_PREFETCH_SSE | PTA_PRFCHW},
3303 {"athlon-4", PROCESSOR_ATHLON, CPU_ATHLON,
3304 PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE | PTA_PRFCHW | PTA_FXSR},
3305 {"athlon-xp", PROCESSOR_ATHLON, CPU_ATHLON,
3306 PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE | PTA_PRFCHW | PTA_FXSR},
3307 {"athlon-mp", PROCESSOR_ATHLON, CPU_ATHLON,
3308 PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE | PTA_PRFCHW | PTA_FXSR},
3309 {"x86-64", PROCESSOR_K8, CPU_K8,
3310 PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_NO_SAHF | PTA_FXSR},
3311 {"k8", PROCESSOR_K8, CPU_K8,
3312 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
3313 | PTA_SSE2 | PTA_NO_SAHF | PTA_PRFCHW | PTA_FXSR},
3314 {"k8-sse3", PROCESSOR_K8, CPU_K8,
3315 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
3316 | PTA_SSE2 | PTA_SSE3 | PTA_NO_SAHF | PTA_PRFCHW | PTA_FXSR},
3317 {"opteron", PROCESSOR_K8, CPU_K8,
3318 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
3319 | PTA_SSE2 | PTA_NO_SAHF | PTA_PRFCHW | PTA_FXSR},
3320 {"opteron-sse3", PROCESSOR_K8, CPU_K8,
3321 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
3322 | PTA_SSE2 | PTA_SSE3 | PTA_NO_SAHF | PTA_PRFCHW | PTA_FXSR},
3323 {"athlon64", PROCESSOR_K8, CPU_K8,
3324 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
3325 | PTA_SSE2 | PTA_NO_SAHF | PTA_PRFCHW | PTA_FXSR},
3326 {"athlon64-sse3", PROCESSOR_K8, CPU_K8,
3327 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
3328 | PTA_SSE2 | PTA_SSE3 | PTA_NO_SAHF | PTA_PRFCHW | PTA_FXSR},
3329 {"athlon-fx", PROCESSOR_K8, CPU_K8,
3330 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
3331 | PTA_SSE2 | PTA_NO_SAHF | PTA_PRFCHW | PTA_FXSR},
3332 {"amdfam10", PROCESSOR_AMDFAM10, CPU_AMDFAM10,
3333 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE | PTA_SSE2
3334 | PTA_SSE3 | PTA_SSE4A | PTA_CX16 | PTA_ABM | PTA_PRFCHW | PTA_FXSR},
3335 {"barcelona", PROCESSOR_AMDFAM10, CPU_AMDFAM10,
3336 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE | PTA_SSE2
3337 | PTA_SSE3 | PTA_SSE4A | PTA_CX16 | PTA_ABM | PTA_PRFCHW | PTA_FXSR},
3338 {"bdver1", PROCESSOR_BDVER1, CPU_BDVER1,
3339 PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
3340 | PTA_SSE4A | PTA_CX16 | PTA_ABM | PTA_SSSE3 | PTA_SSE4_1
3341 | PTA_SSE4_2 | PTA_AES | PTA_PCLMUL | PTA_AVX | PTA_FMA4
3342 | PTA_XOP | PTA_LWP | PTA_PRFCHW | PTA_FXSR | PTA_XSAVE},
3343 {"bdver2", PROCESSOR_BDVER2, CPU_BDVER2,
3344 PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
3345 | PTA_SSE4A | PTA_CX16 | PTA_ABM | PTA_SSSE3 | PTA_SSE4_1
3346 | PTA_SSE4_2 | PTA_AES | PTA_PCLMUL | PTA_AVX | PTA_FMA4
3347 | PTA_XOP | PTA_LWP | PTA_BMI | PTA_TBM | PTA_F16C
3348 | PTA_FMA | PTA_PRFCHW | PTA_FXSR | PTA_XSAVE},
3349 {"bdver3", PROCESSOR_BDVER3, CPU_BDVER3,
3350 PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
3351 | PTA_SSE4A | PTA_CX16 | PTA_ABM | PTA_SSSE3 | PTA_SSE4_1
3352 | PTA_SSE4_2 | PTA_AES | PTA_PCLMUL | PTA_AVX | PTA_FMA4
3353 | PTA_XOP | PTA_LWP | PTA_BMI | PTA_TBM | PTA_F16C
3354 | PTA_FMA | PTA_PRFCHW | PTA_FXSR | PTA_XSAVE
3355 | PTA_XSAVEOPT | PTA_FSGSBASE},
3356 {"bdver4", PROCESSOR_BDVER4, CPU_BDVER4,
3357 PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
3358 | PTA_SSE4A | PTA_CX16 | PTA_ABM | PTA_SSSE3 | PTA_SSE4_1
3359 | PTA_SSE4_2 | PTA_AES | PTA_PCLMUL | PTA_AVX | PTA_AVX2
3360 | PTA_FMA4 | PTA_XOP | PTA_LWP | PTA_BMI | PTA_BMI2
3361 | PTA_TBM | PTA_F16C | PTA_FMA | PTA_PRFCHW | PTA_FXSR
3362 | PTA_XSAVE | PTA_XSAVEOPT | PTA_FSGSBASE | PTA_RDRND
3363 | PTA_MOVBE | PTA_MWAITX},
3364 {"btver1", PROCESSOR_BTVER1, CPU_GENERIC,
3365 PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
3366 | PTA_SSSE3 | PTA_SSE4A |PTA_ABM | PTA_CX16 | PTA_PRFCHW
3367 | PTA_FXSR | PTA_XSAVE},
3368 {"btver2", PROCESSOR_BTVER2, CPU_BTVER2,
3369 PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
3370 | PTA_SSSE3 | PTA_SSE4A |PTA_ABM | PTA_CX16 | PTA_SSE4_1
3371 | PTA_SSE4_2 | PTA_AES | PTA_PCLMUL | PTA_AVX
3372 | PTA_BMI | PTA_F16C | PTA_MOVBE | PTA_PRFCHW
3373 | PTA_FXSR | PTA_XSAVE | PTA_XSAVEOPT},
3375 {"generic", PROCESSOR_GENERIC, CPU_GENERIC,
3377 | PTA_HLE /* flags are only used for -march switch. */ },
3380 /* -mrecip options. */
3383 const char *string; /* option name */
3384 unsigned int mask; /* mask bits to set */
3386 const recip_options[] =
3388 { "all", RECIP_MASK_ALL },
3389 { "none", RECIP_MASK_NONE },
3390 { "div", RECIP_MASK_DIV },
3391 { "sqrt", RECIP_MASK_SQRT },
3392 { "vec-div", RECIP_MASK_VEC_DIV },
3393 { "vec-sqrt", RECIP_MASK_VEC_SQRT },
3396 int const pta_size = ARRAY_SIZE (processor_alias_table);
3398 /* Set up prefix/suffix so the error messages refer to either the command
3399 line argument, or the attribute(target). */
3408 prefix = "option(\"";
3413 /* Turn off both OPTION_MASK_ABI_64 and OPTION_MASK_ABI_X32 if
3414 TARGET_64BIT_DEFAULT is true and TARGET_64BIT is false. */
3415 if (TARGET_64BIT_DEFAULT && !TARGET_64BIT_P (opts->x_ix86_isa_flags))
3416 opts->x_ix86_isa_flags &= ~(OPTION_MASK_ABI_64 | OPTION_MASK_ABI_X32);
3417 #ifdef TARGET_BI_ARCH
3420 #if TARGET_BI_ARCH == 1
3421 /* When TARGET_BI_ARCH == 1, by default, OPTION_MASK_ABI_64
3422 is on and OPTION_MASK_ABI_X32 is off. We turn off
3423 OPTION_MASK_ABI_64 if OPTION_MASK_ABI_X32 is turned on by
3425 if (TARGET_X32_P (opts->x_ix86_isa_flags))
3426 opts->x_ix86_isa_flags &= ~OPTION_MASK_ABI_64;
3428 /* When TARGET_BI_ARCH == 2, by default, OPTION_MASK_ABI_X32 is
3429 on and OPTION_MASK_ABI_64 is off. We turn off
3430 OPTION_MASK_ABI_X32 if OPTION_MASK_ABI_64 is turned on by
3431 -m64 or OPTION_MASK_CODE16 is turned on by -m16. */
3432 if (TARGET_LP64_P (opts->x_ix86_isa_flags)
3433 || TARGET_16BIT_P (opts->x_ix86_isa_flags))
3434 opts->x_ix86_isa_flags &= ~OPTION_MASK_ABI_X32;
3439 if (TARGET_X32_P (opts->x_ix86_isa_flags))
3441 /* Always turn on OPTION_MASK_ISA_64BIT and turn off
3442 OPTION_MASK_ABI_64 for TARGET_X32. */
3443 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_64BIT;
3444 opts->x_ix86_isa_flags &= ~OPTION_MASK_ABI_64;
3446 else if (TARGET_16BIT_P (opts->x_ix86_isa_flags))
3447 opts->x_ix86_isa_flags &= ~(OPTION_MASK_ISA_64BIT
3448 | OPTION_MASK_ABI_X32
3449 | OPTION_MASK_ABI_64);
3450 else if (TARGET_LP64_P (opts->x_ix86_isa_flags))
3452 /* Always turn on OPTION_MASK_ISA_64BIT and turn off
3453 OPTION_MASK_ABI_X32 for TARGET_LP64. */
3454 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_64BIT;
3455 opts->x_ix86_isa_flags &= ~OPTION_MASK_ABI_X32;
3458 #ifdef SUBTARGET_OVERRIDE_OPTIONS
3459 SUBTARGET_OVERRIDE_OPTIONS;
3462 #ifdef SUBSUBTARGET_OVERRIDE_OPTIONS
3463 SUBSUBTARGET_OVERRIDE_OPTIONS;
3466 /* -fPIC is the default for x86_64. */
3467 if (TARGET_MACHO && TARGET_64BIT_P (opts->x_ix86_isa_flags))
3468 opts->x_flag_pic = 2;
3470 /* Need to check -mtune=generic first. */
3471 if (opts->x_ix86_tune_string)
3473 /* As special support for cross compilers we read -mtune=native
3474 as -mtune=generic. With native compilers we won't see the
3475 -mtune=native, as it was changed by the driver. */
3476 if (!strcmp (opts->x_ix86_tune_string, "native"))
3478 opts->x_ix86_tune_string = "generic";
3480 else if (!strcmp (opts->x_ix86_tune_string, "x86-64"))
3481 warning (OPT_Wdeprecated, "%stune=x86-64%s is deprecated; use "
3482 "%stune=k8%s or %stune=generic%s instead as appropriate",
3483 prefix, suffix, prefix, suffix, prefix, suffix);
3487 if (opts->x_ix86_arch_string)
3488 opts->x_ix86_tune_string = opts->x_ix86_arch_string;
3489 if (!opts->x_ix86_tune_string)
3491 opts->x_ix86_tune_string
3492 = processor_target_table[TARGET_CPU_DEFAULT].name;
3493 ix86_tune_defaulted = 1;
3496 /* opts->x_ix86_tune_string is set to opts->x_ix86_arch_string
3497 or defaulted. We need to use a sensible tune option. */
3498 if (!strcmp (opts->x_ix86_tune_string, "x86-64"))
3500 opts->x_ix86_tune_string = "generic";
3504 if (opts->x_ix86_stringop_alg == rep_prefix_8_byte
3505 && !TARGET_64BIT_P (opts->x_ix86_isa_flags))
3507 /* rep; movq isn't available in 32-bit code. */
3508 error ("-mstringop-strategy=rep_8byte not supported for 32-bit code");
3509 opts->x_ix86_stringop_alg = no_stringop;
3512 if (!opts->x_ix86_arch_string)
3513 opts->x_ix86_arch_string
3514 = TARGET_64BIT_P (opts->x_ix86_isa_flags)
3515 ? "x86-64" : SUBTARGET32_DEFAULT_CPU;
3517 ix86_arch_specified = 1;
3519 if (opts_set->x_ix86_pmode)
3521 if ((TARGET_LP64_P (opts->x_ix86_isa_flags)
3522 && opts->x_ix86_pmode == PMODE_SI)
3523 || (!TARGET_64BIT_P (opts->x_ix86_isa_flags)
3524 && opts->x_ix86_pmode == PMODE_DI))
3525 error ("address mode %qs not supported in the %s bit mode",
3526 TARGET_64BIT_P (opts->x_ix86_isa_flags) ? "short" : "long",
3527 TARGET_64BIT_P (opts->x_ix86_isa_flags) ? "64" : "32");
3530 opts->x_ix86_pmode = TARGET_LP64_P (opts->x_ix86_isa_flags)
3531 ? PMODE_DI : PMODE_SI;
3533 if (!opts_set->x_ix86_abi)
3534 opts->x_ix86_abi = DEFAULT_ABI;
3536 /* For targets using ms ABI enable ms-extensions, if not
3537 explicit turned off. For non-ms ABI we turn off this
3539 if (!opts_set->x_flag_ms_extensions)
3540 opts->x_flag_ms_extensions = (MS_ABI == DEFAULT_ABI);
3542 if (opts_set->x_ix86_cmodel)
3544 switch (opts->x_ix86_cmodel)
3548 if (opts->x_flag_pic)
3549 opts->x_ix86_cmodel = CM_SMALL_PIC;
3550 if (!TARGET_64BIT_P (opts->x_ix86_isa_flags))
3551 error ("code model %qs not supported in the %s bit mode",
3557 if (opts->x_flag_pic)
3558 opts->x_ix86_cmodel = CM_MEDIUM_PIC;
3559 if (!TARGET_64BIT_P (opts->x_ix86_isa_flags))
3560 error ("code model %qs not supported in the %s bit mode",
3562 else if (TARGET_X32_P (opts->x_ix86_isa_flags))
3563 error ("code model %qs not supported in x32 mode",
3569 if (opts->x_flag_pic)
3570 opts->x_ix86_cmodel = CM_LARGE_PIC;
3571 if (!TARGET_64BIT_P (opts->x_ix86_isa_flags))
3572 error ("code model %qs not supported in the %s bit mode",
3574 else if (TARGET_X32_P (opts->x_ix86_isa_flags))
3575 error ("code model %qs not supported in x32 mode",
3580 if (opts->x_flag_pic)
3581 error ("code model %s does not support PIC mode", "32");
3582 if (TARGET_64BIT_P (opts->x_ix86_isa_flags))
3583 error ("code model %qs not supported in the %s bit mode",
3588 if (opts->x_flag_pic)
3590 error ("code model %s does not support PIC mode", "kernel");
3591 opts->x_ix86_cmodel = CM_32;
3593 if (!TARGET_64BIT_P (opts->x_ix86_isa_flags))
3594 error ("code model %qs not supported in the %s bit mode",
3604 /* For TARGET_64BIT and MS_ABI, force pic on, in order to enable the
3605 use of rip-relative addressing. This eliminates fixups that
3606 would otherwise be needed if this object is to be placed in a
3607 DLL, and is essentially just as efficient as direct addressing. */
3608 if (TARGET_64BIT_P (opts->x_ix86_isa_flags)
3609 && (TARGET_RDOS || TARGET_PECOFF))
3610 opts->x_ix86_cmodel = CM_MEDIUM_PIC, opts->x_flag_pic = 1;
3611 else if (TARGET_64BIT_P (opts->x_ix86_isa_flags))
3612 opts->x_ix86_cmodel = opts->x_flag_pic ? CM_SMALL_PIC : CM_SMALL;
3614 opts->x_ix86_cmodel = CM_32;
3616 if (TARGET_MACHO && opts->x_ix86_asm_dialect == ASM_INTEL)
3618 error ("-masm=intel not supported in this configuration");
3619 opts->x_ix86_asm_dialect = ASM_ATT;
3621 if ((TARGET_64BIT_P (opts->x_ix86_isa_flags) != 0)
3622 != ((opts->x_ix86_isa_flags & OPTION_MASK_ISA_64BIT) != 0))
3623 sorry ("%i-bit mode not compiled in",
3624 (opts->x_ix86_isa_flags & OPTION_MASK_ISA_64BIT) ? 64 : 32);
3626 for (i = 0; i < pta_size; i++)
3627 if (! strcmp (opts->x_ix86_arch_string, processor_alias_table[i].name))
3629 ix86_schedule = processor_alias_table[i].schedule;
3630 ix86_arch = processor_alias_table[i].processor;
3631 /* Default cpu tuning to the architecture. */
3632 ix86_tune = ix86_arch;
3634 if (TARGET_64BIT_P (opts->x_ix86_isa_flags)
3635 && !(processor_alias_table[i].flags & PTA_64BIT))
3636 error ("CPU you selected does not support x86-64 "
3639 if (processor_alias_table[i].flags & PTA_MMX
3640 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_MMX))
3641 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_MMX;
3642 if (processor_alias_table[i].flags & PTA_3DNOW
3643 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_3DNOW))
3644 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_3DNOW;
3645 if (processor_alias_table[i].flags & PTA_3DNOW_A
3646 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_3DNOW_A))
3647 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_3DNOW_A;
3648 if (processor_alias_table[i].flags & PTA_SSE
3649 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE))
3650 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSE;
3651 if (processor_alias_table[i].flags & PTA_SSE2
3652 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE2))
3653 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSE2;
3654 if (processor_alias_table[i].flags & PTA_SSE3
3655 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE3))
3656 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSE3;
3657 if (processor_alias_table[i].flags & PTA_SSSE3
3658 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSSE3))
3659 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSSE3;
3660 if (processor_alias_table[i].flags & PTA_SSE4_1
3661 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE4_1))
3662 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSE4_1;
3663 if (processor_alias_table[i].flags & PTA_SSE4_2
3664 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE4_2))
3665 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSE4_2;
3666 if (processor_alias_table[i].flags & PTA_AVX
3667 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX))
3668 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX;
3669 if (processor_alias_table[i].flags & PTA_AVX2
3670 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX2))
3671 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX2;
3672 if (processor_alias_table[i].flags & PTA_FMA
3673 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_FMA))
3674 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_FMA;
3675 if (processor_alias_table[i].flags & PTA_SSE4A
3676 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE4A))
3677 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSE4A;
3678 if (processor_alias_table[i].flags & PTA_FMA4
3679 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_FMA4))
3680 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_FMA4;
3681 if (processor_alias_table[i].flags & PTA_XOP
3682 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_XOP))
3683 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_XOP;
3684 if (processor_alias_table[i].flags & PTA_LWP
3685 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_LWP))
3686 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_LWP;
3687 if (processor_alias_table[i].flags & PTA_ABM
3688 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_ABM))
3689 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_ABM;
3690 if (processor_alias_table[i].flags & PTA_BMI
3691 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_BMI))
3692 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_BMI;
3693 if (processor_alias_table[i].flags & (PTA_LZCNT | PTA_ABM)
3694 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_LZCNT))
3695 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_LZCNT;
3696 if (processor_alias_table[i].flags & PTA_TBM
3697 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_TBM))
3698 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_TBM;
3699 if (processor_alias_table[i].flags & PTA_BMI2
3700 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_BMI2))
3701 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_BMI2;
3702 if (processor_alias_table[i].flags & PTA_CX16
3703 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_CX16))
3704 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_CX16;
3705 if (processor_alias_table[i].flags & (PTA_POPCNT | PTA_ABM)
3706 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_POPCNT))
3707 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_POPCNT;
3708 if (!(TARGET_64BIT_P (opts->x_ix86_isa_flags)
3709 && (processor_alias_table[i].flags & PTA_NO_SAHF))
3710 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SAHF))
3711 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SAHF;
3712 if (processor_alias_table[i].flags & PTA_MOVBE
3713 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_MOVBE))
3714 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_MOVBE;
3715 if (processor_alias_table[i].flags & PTA_AES
3716 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_AES))
3717 ix86_isa_flags |= OPTION_MASK_ISA_AES;
3718 if (processor_alias_table[i].flags & PTA_SHA
3719 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_SHA))
3720 ix86_isa_flags |= OPTION_MASK_ISA_SHA;
3721 if (processor_alias_table[i].flags & PTA_PCLMUL
3722 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_PCLMUL))
3723 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_PCLMUL;
3724 if (processor_alias_table[i].flags & PTA_FSGSBASE
3725 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_FSGSBASE))
3726 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_FSGSBASE;
3727 if (processor_alias_table[i].flags & PTA_RDRND
3728 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_RDRND))
3729 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_RDRND;
3730 if (processor_alias_table[i].flags & PTA_F16C
3731 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_F16C))
3732 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_F16C;
3733 if (processor_alias_table[i].flags & PTA_RTM
3734 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_RTM))
3735 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_RTM;
3736 if (processor_alias_table[i].flags & PTA_HLE
3737 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_HLE))
3738 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_HLE;
3739 if (processor_alias_table[i].flags & PTA_PRFCHW
3740 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_PRFCHW))
3741 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_PRFCHW;
3742 if (processor_alias_table[i].flags & PTA_RDSEED
3743 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_RDSEED))
3744 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_RDSEED;
3745 if (processor_alias_table[i].flags & PTA_ADX
3746 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_ADX))
3747 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_ADX;
3748 if (processor_alias_table[i].flags & PTA_FXSR
3749 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_FXSR))
3750 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_FXSR;
3751 if (processor_alias_table[i].flags & PTA_XSAVE
3752 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_XSAVE))
3753 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_XSAVE;
3754 if (processor_alias_table[i].flags & PTA_XSAVEOPT
3755 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_XSAVEOPT))
3756 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_XSAVEOPT;
3757 if (processor_alias_table[i].flags & PTA_AVX512F
3758 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512F))
3759 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512F;
3760 if (processor_alias_table[i].flags & PTA_AVX512ER
3761 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512ER))
3762 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512ER;
3763 if (processor_alias_table[i].flags & PTA_AVX512PF
3764 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512PF))
3765 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512PF;
3766 if (processor_alias_table[i].flags & PTA_AVX512CD
3767 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512CD))
3768 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512CD;
3769 if (processor_alias_table[i].flags & PTA_PREFETCHWT1
3770 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_PREFETCHWT1))
3771 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_PREFETCHWT1;
3772 if (processor_alias_table[i].flags & PTA_PCOMMIT
3773 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_PCOMMIT))
3774 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_PCOMMIT;
3775 if (processor_alias_table[i].flags & PTA_CLWB
3776 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_CLWB))
3777 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_CLWB;
3778 if (processor_alias_table[i].flags & PTA_CLFLUSHOPT
3779 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_CLFLUSHOPT))
3780 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_CLFLUSHOPT;
3781 if (processor_alias_table[i].flags & PTA_XSAVEC
3782 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_XSAVEC))
3783 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_XSAVEC;
3784 if (processor_alias_table[i].flags & PTA_XSAVES
3785 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_XSAVES))
3786 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_XSAVES;
3787 if (processor_alias_table[i].flags & PTA_AVX512DQ
3788 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512DQ))
3789 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512DQ;
3790 if (processor_alias_table[i].flags & PTA_AVX512BW
3791 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512BW))
3792 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512BW;
3793 if (processor_alias_table[i].flags & PTA_AVX512VL
3794 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512VL))
3795 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512VL;
3796 if (processor_alias_table[i].flags & PTA_MPX
3797 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_MPX))
3798 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_MPX;
3799 if (processor_alias_table[i].flags & PTA_AVX512VBMI
3800 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512VBMI))
3801 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512VBMI;
3802 if (processor_alias_table[i].flags & PTA_AVX512IFMA
3803 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512IFMA))
3804 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512IFMA;
3805 if (processor_alias_table[i].flags & (PTA_PREFETCH_SSE | PTA_SSE))
3806 x86_prefetch_sse = true;
3807 if (processor_alias_table[i].flags & PTA_MWAITX
3808 && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_MWAITX))
3809 opts->x_ix86_isa_flags |= OPTION_MASK_ISA_MWAITX;
3814 if (TARGET_X32 && (opts->x_ix86_isa_flags & OPTION_MASK_ISA_MPX))
3815 error ("Intel MPX does not support x32");
3817 if (TARGET_X32 && (ix86_isa_flags & OPTION_MASK_ISA_MPX))
3818 error ("Intel MPX does not support x32");
3820 if (!strcmp (opts->x_ix86_arch_string, "generic"))
3821 error ("generic CPU can be used only for %stune=%s %s",
3822 prefix, suffix, sw);
3823 else if (!strcmp (opts->x_ix86_arch_string, "intel"))
3824 error ("intel CPU can be used only for %stune=%s %s",
3825 prefix, suffix, sw);
3826 else if (i == pta_size)
3827 error ("bad value (%s) for %sarch=%s %s",
3828 opts->x_ix86_arch_string, prefix, suffix, sw);
3830 ix86_arch_mask = 1u << ix86_arch;
3831 for (i = 0; i < X86_ARCH_LAST; ++i)
3832 ix86_arch_features[i] = !!(initial_ix86_arch_features[i] & ix86_arch_mask);
3834 for (i = 0; i < pta_size; i++)
3835 if (! strcmp (opts->x_ix86_tune_string, processor_alias_table[i].name))
3837 ix86_schedule = processor_alias_table[i].schedule;
3838 ix86_tune = processor_alias_table[i].processor;
3839 if (TARGET_64BIT_P (opts->x_ix86_isa_flags))
3841 if (!(processor_alias_table[i].flags & PTA_64BIT))
3843 if (ix86_tune_defaulted)
3845 opts->x_ix86_tune_string = "x86-64";
3846 for (i = 0; i < pta_size; i++)
3847 if (! strcmp (opts->x_ix86_tune_string,
3848 processor_alias_table[i].name))
3850 ix86_schedule = processor_alias_table[i].schedule;
3851 ix86_tune = processor_alias_table[i].processor;
3854 error ("CPU you selected does not support x86-64 "
3858 /* Intel CPUs have always interpreted SSE prefetch instructions as
3859 NOPs; so, we can enable SSE prefetch instructions even when
3860 -mtune (rather than -march) points us to a processor that has them.
3861 However, the VIA C3 gives a SIGILL, so we only do that for i686 and
3862 higher processors. */
3864 && (processor_alias_table[i].flags & (PTA_PREFETCH_SSE | PTA_SSE)))
3865 x86_prefetch_sse = true;
3869 if (ix86_tune_specified && i == pta_size)
3870 error ("bad value (%s) for %stune=%s %s",
3871 opts->x_ix86_tune_string, prefix, suffix, sw);
3873 set_ix86_tune_features (ix86_tune, opts->x_ix86_dump_tunes);
3875 #ifndef USE_IX86_FRAME_POINTER
3876 #define USE_IX86_FRAME_POINTER 0
3879 #ifndef USE_X86_64_FRAME_POINTER
3880 #define USE_X86_64_FRAME_POINTER 0
3883 /* Set the default values for switches whose default depends on TARGET_64BIT
3884 in case they weren't overwritten by command line options. */
3885 if (TARGET_64BIT_P (opts->x_ix86_isa_flags))
3887 if (opts->x_optimize >= 1 && !opts_set->x_flag_omit_frame_pointer)
3888 opts->x_flag_omit_frame_pointer = !USE_X86_64_FRAME_POINTER;
3889 if (opts->x_flag_asynchronous_unwind_tables
3890 && !opts_set->x_flag_unwind_tables
3891 && TARGET_64BIT_MS_ABI)
3892 opts->x_flag_unwind_tables = 1;
3893 if (opts->x_flag_asynchronous_unwind_tables == 2)
3894 opts->x_flag_unwind_tables
3895 = opts->x_flag_asynchronous_unwind_tables = 1;
3896 if (opts->x_flag_pcc_struct_return == 2)
3897 opts->x_flag_pcc_struct_return = 0;
3901 if (opts->x_optimize >= 1 && !opts_set->x_flag_omit_frame_pointer)
3902 opts->x_flag_omit_frame_pointer
3903 = !(USE_IX86_FRAME_POINTER || opts->x_optimize_size);
3904 if (opts->x_flag_asynchronous_unwind_tables == 2)
3905 opts->x_flag_asynchronous_unwind_tables = !USE_IX86_FRAME_POINTER;
3906 if (opts->x_flag_pcc_struct_return == 2)
3907 opts->x_flag_pcc_struct_return = DEFAULT_PCC_STRUCT_RETURN;
3910 ix86_tune_cost = processor_target_table[ix86_tune].cost;
3911 /* TODO: ix86_cost should be chosen at instruction or function granuality
3912 so for cold code we use size_cost even in !optimize_size compilation. */
3913 if (opts->x_optimize_size)
3914 ix86_cost = &ix86_size_cost;
3916 ix86_cost = ix86_tune_cost;
3918 /* Arrange to set up i386_stack_locals for all functions. */
3919 init_machine_status = ix86_init_machine_status;
3921 /* Validate -mregparm= value. */
3922 if (opts_set->x_ix86_regparm)
3924 if (TARGET_64BIT_P (opts->x_ix86_isa_flags))
3925 warning (0, "-mregparm is ignored in 64-bit mode");
3926 if (opts->x_ix86_regparm > REGPARM_MAX)
3928 error ("-mregparm=%d is not between 0 and %d",
3929 opts->x_ix86_regparm, REGPARM_MAX);
3930 opts->x_ix86_regparm = 0;
3933 if (TARGET_64BIT_P (opts->x_ix86_isa_flags))
3934 opts->x_ix86_regparm = REGPARM_MAX;
3936 /* Default align_* from the processor table. */
3937 ix86_default_align (opts);
3939 /* Provide default for -mbranch-cost= value. */
3940 if (!opts_set->x_ix86_branch_cost)
3941 opts->x_ix86_branch_cost = ix86_tune_cost->branch_cost;
3943 if (TARGET_64BIT_P (opts->x_ix86_isa_flags))
3945 opts->x_target_flags
3946 |= TARGET_SUBTARGET64_DEFAULT & ~opts_set->x_target_flags;
3948 /* Enable by default the SSE and MMX builtins. Do allow the user to
3949 explicitly disable any of these. In particular, disabling SSE and
3950 MMX for kernel code is extremely useful. */
3951 if (!ix86_arch_specified)
3952 opts->x_ix86_isa_flags
3953 |= ((OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_MMX
3954 | TARGET_SUBTARGET64_ISA_DEFAULT)
3955 & ~opts->x_ix86_isa_flags_explicit);
3957 if (TARGET_RTD_P (opts->x_target_flags))
3958 warning (0, "%srtd%s is ignored in 64bit mode", prefix, suffix);
3962 opts->x_target_flags
3963 |= TARGET_SUBTARGET32_DEFAULT & ~opts_set->x_target_flags;
3965 if (!ix86_arch_specified)
3966 opts->x_ix86_isa_flags
3967 |= TARGET_SUBTARGET32_ISA_DEFAULT & ~opts->x_ix86_isa_flags_explicit;
3969 /* i386 ABI does not specify red zone. It still makes sense to use it
3970 when programmer takes care to stack from being destroyed. */
3971 if (!(opts_set->x_target_flags & MASK_NO_RED_ZONE))
3972 opts->x_target_flags |= MASK_NO_RED_ZONE;
3975 /* Keep nonleaf frame pointers. */
3976 if (opts->x_flag_omit_frame_pointer)
3977 opts->x_target_flags &= ~MASK_OMIT_LEAF_FRAME_POINTER;
3978 else if (TARGET_OMIT_LEAF_FRAME_POINTER_P (opts->x_target_flags))
3979 opts->x_flag_omit_frame_pointer = 1;
3981 /* If we're doing fast math, we don't care about comparison order
3982 wrt NaNs. This lets us use a shorter comparison sequence. */
3983 if (opts->x_flag_finite_math_only)
3984 opts->x_target_flags &= ~MASK_IEEE_FP;
3986 /* If the architecture always has an FPU, turn off NO_FANCY_MATH_387,
3987 since the insns won't need emulation. */
3988 if (ix86_tune_features [X86_TUNE_ALWAYS_FANCY_MATH_387])
3989 opts->x_target_flags &= ~MASK_NO_FANCY_MATH_387;
3991 /* Likewise, if the target doesn't have a 387, or we've specified
3992 software floating point, don't use 387 inline intrinsics. */
3993 if (!TARGET_80387_P (opts->x_target_flags))
3994 opts->x_target_flags |= MASK_NO_FANCY_MATH_387;
3996 /* Turn on MMX builtins for -msse. */
3997 if (TARGET_SSE_P (opts->x_ix86_isa_flags))
3998 opts->x_ix86_isa_flags
3999 |= OPTION_MASK_ISA_MMX & ~opts->x_ix86_isa_flags_explicit;
4001 /* Enable SSE prefetch. */
4002 if (TARGET_SSE_P (opts->x_ix86_isa_flags)
4003 || (TARGET_PRFCHW && !TARGET_3DNOW_P (opts->x_ix86_isa_flags)))
4004 x86_prefetch_sse = true;
4006 /* Enable prefetch{,w} instructions for -m3dnow and -mprefetchwt1. */
4007 if (TARGET_3DNOW_P (opts->x_ix86_isa_flags)
4008 || TARGET_PREFETCHWT1_P (opts->x_ix86_isa_flags))
4009 opts->x_ix86_isa_flags
4010 |= OPTION_MASK_ISA_PRFCHW & ~opts->x_ix86_isa_flags_explicit;
4012 /* Enable popcnt instruction for -msse4.2 or -mabm. */
4013 if (TARGET_SSE4_2_P (opts->x_ix86_isa_flags)
4014 || TARGET_ABM_P (opts->x_ix86_isa_flags))
4015 opts->x_ix86_isa_flags
4016 |= OPTION_MASK_ISA_POPCNT & ~opts->x_ix86_isa_flags_explicit;
4018 /* Enable lzcnt instruction for -mabm. */
4019 if (TARGET_ABM_P(opts->x_ix86_isa_flags))
4020 opts->x_ix86_isa_flags
4021 |= OPTION_MASK_ISA_LZCNT & ~opts->x_ix86_isa_flags_explicit;
4023 /* Validate -mpreferred-stack-boundary= value or default it to
4024 PREFERRED_STACK_BOUNDARY_DEFAULT. */
4025 ix86_preferred_stack_boundary = PREFERRED_STACK_BOUNDARY_DEFAULT;
4026 if (opts_set->x_ix86_preferred_stack_boundary_arg)
4028 int min = (TARGET_64BIT_P (opts->x_ix86_isa_flags)
4029 ? (TARGET_SSE_P (opts->x_ix86_isa_flags) ? 4 : 3) : 2);
4030 int max = (TARGET_SEH ? 4 : 12);
4032 if (opts->x_ix86_preferred_stack_boundary_arg < min
4033 || opts->x_ix86_preferred_stack_boundary_arg > max)
4036 error ("-mpreferred-stack-boundary is not supported "
4039 error ("-mpreferred-stack-boundary=%d is not between %d and %d",
4040 opts->x_ix86_preferred_stack_boundary_arg, min, max);
4043 ix86_preferred_stack_boundary
4044 = (1 << opts->x_ix86_preferred_stack_boundary_arg) * BITS_PER_UNIT;
4047 /* Set the default value for -mstackrealign. */
4048 if (opts->x_ix86_force_align_arg_pointer == -1)
4049 opts->x_ix86_force_align_arg_pointer = STACK_REALIGN_DEFAULT;
4051 ix86_default_incoming_stack_boundary = PREFERRED_STACK_BOUNDARY;
4053 /* Validate -mincoming-stack-boundary= value or default it to
4054 MIN_STACK_BOUNDARY/PREFERRED_STACK_BOUNDARY. */
4055 ix86_incoming_stack_boundary = ix86_default_incoming_stack_boundary;
4056 if (opts_set->x_ix86_incoming_stack_boundary_arg)
4058 if (opts->x_ix86_incoming_stack_boundary_arg
4059 < (TARGET_64BIT_P (opts->x_ix86_isa_flags) ? 4 : 2)
4060 || opts->x_ix86_incoming_stack_boundary_arg > 12)
4061 error ("-mincoming-stack-boundary=%d is not between %d and 12",
4062 opts->x_ix86_incoming_stack_boundary_arg,
4063 TARGET_64BIT_P (opts->x_ix86_isa_flags) ? 4 : 2);
4066 ix86_user_incoming_stack_boundary
4067 = (1 << opts->x_ix86_incoming_stack_boundary_arg) * BITS_PER_UNIT;
4068 ix86_incoming_stack_boundary
4069 = ix86_user_incoming_stack_boundary;
4073 #ifndef NO_PROFILE_COUNTERS
4074 if (flag_nop_mcount)
4075 error ("-mnop-mcount is not compatible with this target");
4077 if (flag_nop_mcount && flag_pic)
4078 error ("-mnop-mcount is not implemented for -fPIC");
4080 /* Accept -msseregparm only if at least SSE support is enabled. */
4081 if (TARGET_SSEREGPARM_P (opts->x_target_flags)
4082 && ! TARGET_SSE_P (opts->x_ix86_isa_flags))
4083 error ("%ssseregparm%s used without SSE enabled", prefix, suffix);
4085 if (opts_set->x_ix86_fpmath)
4087 if (opts->x_ix86_fpmath & FPMATH_SSE)
4089 if (!TARGET_SSE_P (opts->x_ix86_isa_flags))
4091 warning (0, "SSE instruction set disabled, using 387 arithmetics");
4092 opts->x_ix86_fpmath = FPMATH_387;
4094 else if ((opts->x_ix86_fpmath & FPMATH_387)
4095 && !TARGET_80387_P (opts->x_target_flags))
4097 warning (0, "387 instruction set disabled, using SSE arithmetics");
4098 opts->x_ix86_fpmath = FPMATH_SSE;
4102 /* For all chips supporting SSE2, -mfpmath=sse performs better than
4103 fpmath=387. The second is however default at many targets since the
4104 extra 80bit precision of temporaries is considered to be part of ABI.
4105 Overwrite the default at least for -ffast-math.
4106 TODO: -mfpmath=both seems to produce same performing code with bit
4107 smaller binaries. It is however not clear if register allocation is
4108 ready for this setting.
4109 Also -mfpmath=387 is overall a lot more compact (bout 4-5%) than SSE
4110 codegen. We may switch to 387 with -ffast-math for size optimized
4112 else if (fast_math_flags_set_p (&global_options)
4113 && TARGET_SSE2_P (opts->x_ix86_isa_flags))
4114 opts->x_ix86_fpmath = FPMATH_SSE;
4116 opts->x_ix86_fpmath = TARGET_FPMATH_DEFAULT_P (opts->x_ix86_isa_flags);
4118 /* If the i387 is disabled, then do not return values in it. */
4119 if (!TARGET_80387_P (opts->x_target_flags))
4120 opts->x_target_flags &= ~MASK_FLOAT_RETURNS;
4122 /* Use external vectorized library in vectorizing intrinsics. */
4123 if (opts_set->x_ix86_veclibabi_type)
4124 switch (opts->x_ix86_veclibabi_type)
4126 case ix86_veclibabi_type_svml:
4127 ix86_veclib_handler = ix86_veclibabi_svml;
4130 case ix86_veclibabi_type_acml:
4131 ix86_veclib_handler = ix86_veclibabi_acml;
4138 if (ix86_tune_features [X86_TUNE_ACCUMULATE_OUTGOING_ARGS]
4139 && !(opts_set->x_target_flags & MASK_ACCUMULATE_OUTGOING_ARGS))
4140 opts->x_target_flags |= MASK_ACCUMULATE_OUTGOING_ARGS;
4142 /* If stack probes are required, the space used for large function
4143 arguments on the stack must also be probed, so enable
4144 -maccumulate-outgoing-args so this happens in the prologue. */
4145 if (TARGET_STACK_PROBE_P (opts->x_target_flags)
4146 && !(opts->x_target_flags & MASK_ACCUMULATE_OUTGOING_ARGS))
4148 if (opts_set->x_target_flags & MASK_ACCUMULATE_OUTGOING_ARGS)
4149 warning (0, "stack probing requires %saccumulate-outgoing-args%s "
4150 "for correctness", prefix, suffix);
4151 opts->x_target_flags |= MASK_ACCUMULATE_OUTGOING_ARGS;
4154 /* Figure out what ASM_GENERATE_INTERNAL_LABEL builds as a prefix. */
4157 ASM_GENERATE_INTERNAL_LABEL (internal_label_prefix, "LX", 0);
4158 p = strchr (internal_label_prefix, 'X');
4159 internal_label_prefix_len = p - internal_label_prefix;
4163 /* When scheduling description is not available, disable scheduler pass
4164 so it won't slow down the compilation and make x87 code slower. */
4165 if (!TARGET_SCHEDULE)
4166 opts->x_flag_schedule_insns_after_reload = opts->x_flag_schedule_insns = 0;
4168 maybe_set_param_value (PARAM_SIMULTANEOUS_PREFETCHES,
4169 ix86_tune_cost->simultaneous_prefetches,
4170 opts->x_param_values,
4171 opts_set->x_param_values);
4172 maybe_set_param_value (PARAM_L1_CACHE_LINE_SIZE,
4173 ix86_tune_cost->prefetch_block,
4174 opts->x_param_values,
4175 opts_set->x_param_values);
4176 maybe_set_param_value (PARAM_L1_CACHE_SIZE,
4177 ix86_tune_cost->l1_cache_size,
4178 opts->x_param_values,
4179 opts_set->x_param_values);
4180 maybe_set_param_value (PARAM_L2_CACHE_SIZE,
4181 ix86_tune_cost->l2_cache_size,
4182 opts->x_param_values,
4183 opts_set->x_param_values);
4185 /* Enable sw prefetching at -O3 for CPUS that prefetching is helpful. */
4186 if (opts->x_flag_prefetch_loop_arrays < 0
4188 && (opts->x_optimize >= 3 || opts->x_flag_profile_use)
4189 && !opts->x_optimize_size
4190 && TARGET_SOFTWARE_PREFETCHING_BENEFICIAL)
4191 opts->x_flag_prefetch_loop_arrays = 1;
4193 /* If using typedef char *va_list, signal that __builtin_va_start (&ap, 0)
4194 can be opts->x_optimized to ap = __builtin_next_arg (0). */
4195 if (!TARGET_64BIT_P (opts->x_ix86_isa_flags) && !opts->x_flag_split_stack)
4196 targetm.expand_builtin_va_start = NULL;
4198 if (TARGET_64BIT_P (opts->x_ix86_isa_flags))
4200 ix86_gen_leave = gen_leave_rex64;
4201 if (Pmode == DImode)
4203 ix86_gen_tls_global_dynamic_64 = gen_tls_global_dynamic_64_di;
4204 ix86_gen_tls_local_dynamic_base_64
4205 = gen_tls_local_dynamic_base_64_di;
4209 ix86_gen_tls_global_dynamic_64 = gen_tls_global_dynamic_64_si;
4210 ix86_gen_tls_local_dynamic_base_64
4211 = gen_tls_local_dynamic_base_64_si;
4215 ix86_gen_leave = gen_leave;
4217 if (Pmode == DImode)
4219 ix86_gen_add3 = gen_adddi3;
4220 ix86_gen_sub3 = gen_subdi3;
4221 ix86_gen_sub3_carry = gen_subdi3_carry;
4222 ix86_gen_one_cmpl2 = gen_one_cmpldi2;
4223 ix86_gen_andsp = gen_anddi3;
4224 ix86_gen_allocate_stack_worker = gen_allocate_stack_worker_probe_di;
4225 ix86_gen_adjust_stack_and_probe = gen_adjust_stack_and_probedi;
4226 ix86_gen_probe_stack_range = gen_probe_stack_rangedi;
4227 ix86_gen_monitor = gen_sse3_monitor_di;
4228 ix86_gen_monitorx = gen_monitorx_di;
4232 ix86_gen_add3 = gen_addsi3;
4233 ix86_gen_sub3 = gen_subsi3;
4234 ix86_gen_sub3_carry = gen_subsi3_carry;
4235 ix86_gen_one_cmpl2 = gen_one_cmplsi2;
4236 ix86_gen_andsp = gen_andsi3;
4237 ix86_gen_allocate_stack_worker = gen_allocate_stack_worker_probe_si;
4238 ix86_gen_adjust_stack_and_probe = gen_adjust_stack_and_probesi;
4239 ix86_gen_probe_stack_range = gen_probe_stack_rangesi;
4240 ix86_gen_monitor = gen_sse3_monitor_si;
4241 ix86_gen_monitorx = gen_monitorx_si;
4245 /* Use -mcld by default for 32-bit code if configured with --enable-cld. */
4246 if (!TARGET_64BIT_P (opts->x_ix86_isa_flags))
4247 opts->x_target_flags |= MASK_CLD & ~opts_set->x_target_flags;
4250 if (!TARGET_64BIT_P (opts->x_ix86_isa_flags) && opts->x_flag_pic)
4252 if (opts->x_flag_fentry > 0)
4253 sorry ("-mfentry isn%'t supported for 32-bit in combination "
4255 opts->x_flag_fentry = 0;
4257 else if (TARGET_SEH)
4259 if (opts->x_flag_fentry == 0)
4260 sorry ("-mno-fentry isn%'t compatible with SEH");
4261 opts->x_flag_fentry = 1;
4263 else if (opts->x_flag_fentry < 0)
4265 #if defined(PROFILE_BEFORE_PROLOGUE)
4266 opts->x_flag_fentry = 1;
4268 opts->x_flag_fentry = 0;
4272 if (!(opts_set->x_target_flags & MASK_VZEROUPPER))
4273 opts->x_target_flags |= MASK_VZEROUPPER;
4274 if (!ix86_tune_features[X86_TUNE_AVX256_UNALIGNED_LOAD_OPTIMAL]
4275 && !(opts_set->x_target_flags & MASK_AVX256_SPLIT_UNALIGNED_LOAD))
4276 opts->x_target_flags |= MASK_AVX256_SPLIT_UNALIGNED_LOAD;
4277 if (!ix86_tune_features[X86_TUNE_AVX256_UNALIGNED_STORE_OPTIMAL]
4278 && !(opts_set->x_target_flags & MASK_AVX256_SPLIT_UNALIGNED_STORE))
4279 opts->x_target_flags |= MASK_AVX256_SPLIT_UNALIGNED_STORE;
4280 /* Enable 128-bit AVX instruction generation
4281 for the auto-vectorizer. */
4282 if (TARGET_AVX128_OPTIMAL
4283 && !(opts_set->x_target_flags & MASK_PREFER_AVX128))
4284 opts->x_target_flags |= MASK_PREFER_AVX128;
4286 if (opts->x_ix86_recip_name)
4288 char *p = ASTRDUP (opts->x_ix86_recip_name);
4290 unsigned int mask, i;
4293 while ((q = strtok (p, ",")) != NULL)
4304 if (!strcmp (q, "default"))
4305 mask = RECIP_MASK_ALL;
4308 for (i = 0; i < ARRAY_SIZE (recip_options); i++)
4309 if (!strcmp (q, recip_options[i].string))
4311 mask = recip_options[i].mask;
4315 if (i == ARRAY_SIZE (recip_options))
4317 error ("unknown option for -mrecip=%s", q);
4319 mask = RECIP_MASK_NONE;
4323 opts->x_recip_mask_explicit |= mask;
4325 opts->x_recip_mask &= ~mask;
4327 opts->x_recip_mask |= mask;
4331 if (TARGET_RECIP_P (opts->x_target_flags))
4332 opts->x_recip_mask |= RECIP_MASK_ALL & ~opts->x_recip_mask_explicit;
4333 else if (opts_set->x_target_flags & MASK_RECIP)
4334 opts->x_recip_mask &= ~(RECIP_MASK_ALL & ~opts->x_recip_mask_explicit);
4336 /* Default long double to 64-bit for 32-bit Bionic and to __float128
4337 for 64-bit Bionic. */
4338 if (TARGET_HAS_BIONIC
4339 && !(opts_set->x_target_flags
4340 & (MASK_LONG_DOUBLE_64 | MASK_LONG_DOUBLE_128)))
4341 opts->x_target_flags |= (TARGET_64BIT
4342 ? MASK_LONG_DOUBLE_128
4343 : MASK_LONG_DOUBLE_64);
4345 /* Only one of them can be active. */
4346 gcc_assert ((opts->x_target_flags & MASK_LONG_DOUBLE_64) == 0
4347 || (opts->x_target_flags & MASK_LONG_DOUBLE_128) == 0);
4349 /* Save the initial options in case the user does function specific
4352 target_option_default_node = target_option_current_node
4353 = build_target_option_node (opts);
4355 /* Handle stack protector */
4356 if (!opts_set->x_ix86_stack_protector_guard)
4357 opts->x_ix86_stack_protector_guard
4358 = TARGET_HAS_BIONIC ? SSP_GLOBAL : SSP_TLS;
4360 /* Handle -mmemcpy-strategy= and -mmemset-strategy= */
4361 if (opts->x_ix86_tune_memcpy_strategy)
4363 char *str = xstrdup (opts->x_ix86_tune_memcpy_strategy);
4364 ix86_parse_stringop_strategy_string (str, false);
4368 if (opts->x_ix86_tune_memset_strategy)
4370 char *str = xstrdup (opts->x_ix86_tune_memset_strategy);
4371 ix86_parse_stringop_strategy_string (str, true);
4376 /* Implement the TARGET_OPTION_OVERRIDE hook. */
4379 ix86_option_override (void)
4381 opt_pass *pass_insert_vzeroupper = make_pass_insert_vzeroupper (g);
4382 struct register_pass_info insert_vzeroupper_info
4383 = { pass_insert_vzeroupper, "reload",
4384 1, PASS_POS_INSERT_AFTER
4387 ix86_option_override_internal (true, &global_options, &global_options_set);
4390 /* This needs to be done at start up. It's convenient to do it here. */
4391 register_pass (&insert_vzeroupper_info);
4394 /* Implement the TARGET_OFFLOAD_OPTIONS hook. */
4396 ix86_offload_options (void)
4399 return xstrdup ("-foffload-abi=lp64");
4400 return xstrdup ("-foffload-abi=ilp32");
4403 /* Update register usage after having seen the compiler flags. */
4406 ix86_conditional_register_usage (void)
4410 /* For 32-bit targets, squash the REX registers. */
4413 for (i = FIRST_REX_INT_REG; i <= LAST_REX_INT_REG; i++)
4414 fixed_regs[i] = call_used_regs[i] = 1, reg_names[i] = "";
4415 for (i = FIRST_REX_SSE_REG; i <= LAST_REX_SSE_REG; i++)
4416 fixed_regs[i] = call_used_regs[i] = 1, reg_names[i] = "";
4417 for (i = FIRST_EXT_REX_SSE_REG; i <= LAST_EXT_REX_SSE_REG; i++)
4418 fixed_regs[i] = call_used_regs[i] = 1, reg_names[i] = "";
4421 /* See the definition of CALL_USED_REGISTERS in i386.h. */
4422 c_mask = (TARGET_64BIT_MS_ABI ? (1 << 3)
4423 : TARGET_64BIT ? (1 << 2)
4426 CLEAR_HARD_REG_SET (reg_class_contents[(int)CLOBBERED_REGS]);
4428 for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
4430 /* Set/reset conditionally defined registers from
4431 CALL_USED_REGISTERS initializer. */
4432 if (call_used_regs[i] > 1)
4433 call_used_regs[i] = !!(call_used_regs[i] & c_mask);
4435 /* Calculate registers of CLOBBERED_REGS register set
4436 as call used registers from GENERAL_REGS register set. */
4437 if (TEST_HARD_REG_BIT (reg_class_contents[(int)GENERAL_REGS], i)
4438 && call_used_regs[i])
4439 SET_HARD_REG_BIT (reg_class_contents[(int)CLOBBERED_REGS], i);
4442 /* If MMX is disabled, squash the registers. */
4444 for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
4445 if (TEST_HARD_REG_BIT (reg_class_contents[(int)MMX_REGS], i))
4446 fixed_regs[i] = call_used_regs[i] = 1, reg_names[i] = "";
4448 /* If SSE is disabled, squash the registers. */
4450 for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
4451 if (TEST_HARD_REG_BIT (reg_class_contents[(int)SSE_REGS], i))
4452 fixed_regs[i] = call_used_regs[i] = 1, reg_names[i] = "";
4454 /* If the FPU is disabled, squash the registers. */
4455 if (! (TARGET_80387 || TARGET_FLOAT_RETURNS_IN_80387))
4456 for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
4457 if (TEST_HARD_REG_BIT (reg_class_contents[(int)FLOAT_REGS], i))
4458 fixed_regs[i] = call_used_regs[i] = 1, reg_names[i] = "";
4460 /* If AVX512F is disabled, squash the registers. */
4461 if (! TARGET_AVX512F)
4463 for (i = FIRST_EXT_REX_SSE_REG; i <= LAST_EXT_REX_SSE_REG; i++)
4464 fixed_regs[i] = call_used_regs[i] = 1, reg_names[i] = "";
4466 for (i = FIRST_MASK_REG; i <= LAST_MASK_REG; i++)
4467 fixed_regs[i] = call_used_regs[i] = 1, reg_names[i] = "";
4470 /* If MPX is disabled, squash the registers. */
4472 for (i = FIRST_BND_REG; i <= LAST_BND_REG; i++)
4473 fixed_regs[i] = call_used_regs[i] = 1, reg_names[i] = "";
4477 /* Save the current options */
4480 ix86_function_specific_save (struct cl_target_option *ptr,
4481 struct gcc_options *opts)
4483 ptr->arch = ix86_arch;
4484 ptr->schedule = ix86_schedule;
4485 ptr->prefetch_sse = x86_prefetch_sse;
4486 ptr->tune = ix86_tune;
4487 ptr->branch_cost = ix86_branch_cost;
4488 ptr->tune_defaulted = ix86_tune_defaulted;
4489 ptr->arch_specified = ix86_arch_specified;
4490 ptr->x_ix86_isa_flags_explicit = opts->x_ix86_isa_flags_explicit;
4491 ptr->x_ix86_target_flags_explicit = opts->x_ix86_target_flags_explicit;
4492 ptr->x_recip_mask_explicit = opts->x_recip_mask_explicit;
4493 ptr->x_ix86_arch_string = opts->x_ix86_arch_string;
4494 ptr->x_ix86_tune_string = opts->x_ix86_tune_string;
4495 ptr->x_ix86_cmodel = opts->x_ix86_cmodel;
4496 ptr->x_ix86_abi = opts->x_ix86_abi;
4497 ptr->x_ix86_asm_dialect = opts->x_ix86_asm_dialect;
4498 ptr->x_ix86_branch_cost = opts->x_ix86_branch_cost;
4499 ptr->x_ix86_dump_tunes = opts->x_ix86_dump_tunes;
4500 ptr->x_ix86_force_align_arg_pointer = opts->x_ix86_force_align_arg_pointer;
4501 ptr->x_ix86_force_drap = opts->x_ix86_force_drap;
4502 ptr->x_ix86_incoming_stack_boundary_arg = opts->x_ix86_incoming_stack_boundary_arg;
4503 ptr->x_ix86_pmode = opts->x_ix86_pmode;
4504 ptr->x_ix86_preferred_stack_boundary_arg = opts->x_ix86_preferred_stack_boundary_arg;
4505 ptr->x_ix86_recip_name = opts->x_ix86_recip_name;
4506 ptr->x_ix86_regparm = opts->x_ix86_regparm;
4507 ptr->x_ix86_section_threshold = opts->x_ix86_section_threshold;
4508 ptr->x_ix86_sse2avx = opts->x_ix86_sse2avx;
4509 ptr->x_ix86_stack_protector_guard = opts->x_ix86_stack_protector_guard;
4510 ptr->x_ix86_stringop_alg = opts->x_ix86_stringop_alg;
4511 ptr->x_ix86_tls_dialect = opts->x_ix86_tls_dialect;
4512 ptr->x_ix86_tune_ctrl_string = opts->x_ix86_tune_ctrl_string;
4513 ptr->x_ix86_tune_memcpy_strategy = opts->x_ix86_tune_memcpy_strategy;
4514 ptr->x_ix86_tune_memset_strategy = opts->x_ix86_tune_memset_strategy;
4515 ptr->x_ix86_tune_no_default = opts->x_ix86_tune_no_default;
4516 ptr->x_ix86_veclibabi_type = opts->x_ix86_veclibabi_type;
4518 /* The fields are char but the variables are not; make sure the
4519 values fit in the fields. */
4520 gcc_assert (ptr->arch == ix86_arch);
4521 gcc_assert (ptr->schedule == ix86_schedule);
4522 gcc_assert (ptr->tune == ix86_tune);
4523 gcc_assert (ptr->branch_cost == ix86_branch_cost);
4526 /* Restore the current options */
4529 ix86_function_specific_restore (struct gcc_options *opts,
4530 struct cl_target_option *ptr)
4532 enum processor_type old_tune = ix86_tune;
4533 enum processor_type old_arch = ix86_arch;
4534 unsigned int ix86_arch_mask;
4537 /* We don't change -fPIC. */
4538 opts->x_flag_pic = flag_pic;
4540 ix86_arch = (enum processor_type) ptr->arch;
4541 ix86_schedule = (enum attr_cpu) ptr->schedule;
4542 ix86_tune = (enum processor_type) ptr->tune;
4543 x86_prefetch_sse = ptr->prefetch_sse;
4544 opts->x_ix86_branch_cost = ptr->branch_cost;
4545 ix86_tune_defaulted = ptr->tune_defaulted;
4546 ix86_arch_specified = ptr->arch_specified;
4547 opts->x_ix86_isa_flags_explicit = ptr->x_ix86_isa_flags_explicit;
4548 opts->x_ix86_target_flags_explicit = ptr->x_ix86_target_flags_explicit;
4549 opts->x_recip_mask_explicit = ptr->x_recip_mask_explicit;
4550 opts->x_ix86_arch_string = ptr->x_ix86_arch_string;
4551 opts->x_ix86_tune_string = ptr->x_ix86_tune_string;
4552 opts->x_ix86_cmodel = ptr->x_ix86_cmodel;
4553 opts->x_ix86_abi = ptr->x_ix86_abi;
4554 opts->x_ix86_asm_dialect = ptr->x_ix86_asm_dialect;
4555 opts->x_ix86_branch_cost = ptr->x_ix86_branch_cost;
4556 opts->x_ix86_dump_tunes = ptr->x_ix86_dump_tunes;
4557 opts->x_ix86_force_align_arg_pointer = ptr->x_ix86_force_align_arg_pointer;
4558 opts->x_ix86_force_drap = ptr->x_ix86_force_drap;
4559 opts->x_ix86_incoming_stack_boundary_arg = ptr->x_ix86_incoming_stack_boundary_arg;
4560 opts->x_ix86_pmode = ptr->x_ix86_pmode;
4561 opts->x_ix86_preferred_stack_boundary_arg = ptr->x_ix86_preferred_stack_boundary_arg;
4562 opts->x_ix86_recip_name = ptr->x_ix86_recip_name;
4563 opts->x_ix86_regparm = ptr->x_ix86_regparm;
4564 opts->x_ix86_section_threshold = ptr->x_ix86_section_threshold;
4565 opts->x_ix86_sse2avx = ptr->x_ix86_sse2avx;
4566 opts->x_ix86_stack_protector_guard = ptr->x_ix86_stack_protector_guard;
4567 opts->x_ix86_stringop_alg = ptr->x_ix86_stringop_alg;
4568 opts->x_ix86_tls_dialect = ptr->x_ix86_tls_dialect;
4569 opts->x_ix86_tune_ctrl_string = ptr->x_ix86_tune_ctrl_string;
4570 opts->x_ix86_tune_memcpy_strategy = ptr->x_ix86_tune_memcpy_strategy;
4571 opts->x_ix86_tune_memset_strategy = ptr->x_ix86_tune_memset_strategy;
4572 opts->x_ix86_tune_no_default = ptr->x_ix86_tune_no_default;
4573 opts->x_ix86_veclibabi_type = ptr->x_ix86_veclibabi_type;
4574 ix86_tune_cost = processor_target_table[ix86_tune].cost;
4575 /* TODO: ix86_cost should be chosen at instruction or function granuality
4576 so for cold code we use size_cost even in !optimize_size compilation. */
4577 if (opts->x_optimize_size)
4578 ix86_cost = &ix86_size_cost;
4580 ix86_cost = ix86_tune_cost;
4582 /* Recreate the arch feature tests if the arch changed */
4583 if (old_arch != ix86_arch)
4585 ix86_arch_mask = 1u << ix86_arch;
4586 for (i = 0; i < X86_ARCH_LAST; ++i)
4587 ix86_arch_features[i]
4588 = !!(initial_ix86_arch_features[i] & ix86_arch_mask);
4591 /* Recreate the tune optimization tests */
4592 if (old_tune != ix86_tune)
4593 set_ix86_tune_features (ix86_tune, false);
4596 /* Adjust target options after streaming them in. This is mainly about
4597 reconciling them with global options. */
4600 ix86_function_specific_post_stream_in (struct cl_target_option *ptr)
4602 /* flag_pic is a global option, but ix86_cmodel is target saved option
4603 partly computed from flag_pic. If flag_pic is on, adjust x_ix86_cmodel
4604 for PIC, or error out. */
4606 switch (ptr->x_ix86_cmodel)
4609 ptr->x_ix86_cmodel = CM_SMALL_PIC;
4613 ptr->x_ix86_cmodel = CM_MEDIUM_PIC;
4617 ptr->x_ix86_cmodel = CM_LARGE_PIC;
4621 error ("code model %s does not support PIC mode", "kernel");
4628 switch (ptr->x_ix86_cmodel)
4631 ptr->x_ix86_cmodel = CM_SMALL;
4635 ptr->x_ix86_cmodel = CM_MEDIUM;
4639 ptr->x_ix86_cmodel = CM_LARGE;
4647 /* Print the current options */
4650 ix86_function_specific_print (FILE *file, int indent,
4651 struct cl_target_option *ptr)
4654 = ix86_target_string (ptr->x_ix86_isa_flags, ptr->x_target_flags,
4655 NULL, NULL, ptr->x_ix86_fpmath, false);
4657 gcc_assert (ptr->arch < PROCESSOR_max);
4658 fprintf (file, "%*sarch = %d (%s)\n",
4660 ptr->arch, processor_target_table[ptr->arch].name);
4662 gcc_assert (ptr->tune < PROCESSOR_max);
4663 fprintf (file, "%*stune = %d (%s)\n",
4665 ptr->tune, processor_target_table[ptr->tune].name);
4667 fprintf (file, "%*sbranch_cost = %d\n", indent, "", ptr->branch_cost);
4671 fprintf (file, "%*s%s\n", indent, "", target_string);
4672 free (target_string);
4677 /* Inner function to process the attribute((target(...))), take an argument and
4678 set the current options from the argument. If we have a list, recursively go
4682 ix86_valid_target_attribute_inner_p (tree args, char *p_strings[],
4683 struct gcc_options *opts,
4684 struct gcc_options *opts_set,
4685 struct gcc_options *enum_opts_set)
4690 #define IX86_ATTR_ISA(S,O) { S, sizeof (S)-1, ix86_opt_isa, O, 0 }
4691 #define IX86_ATTR_STR(S,O) { S, sizeof (S)-1, ix86_opt_str, O, 0 }
4692 #define IX86_ATTR_ENUM(S,O) { S, sizeof (S)-1, ix86_opt_enum, O, 0 }
4693 #define IX86_ATTR_YES(S,O,M) { S, sizeof (S)-1, ix86_opt_yes, O, M }
4694 #define IX86_ATTR_NO(S,O,M) { S, sizeof (S)-1, ix86_opt_no, O, M }
4710 enum ix86_opt_type type;
4715 IX86_ATTR_ISA ("3dnow", OPT_m3dnow),
4716 IX86_ATTR_ISA ("abm", OPT_mabm),
4717 IX86_ATTR_ISA ("bmi", OPT_mbmi),
4718 IX86_ATTR_ISA ("bmi2", OPT_mbmi2),
4719 IX86_ATTR_ISA ("lzcnt", OPT_mlzcnt),
4720 IX86_ATTR_ISA ("tbm", OPT_mtbm),
4721 IX86_ATTR_ISA ("aes", OPT_maes),
4722 IX86_ATTR_ISA ("sha", OPT_msha),
4723 IX86_ATTR_ISA ("avx", OPT_mavx),
4724 IX86_ATTR_ISA ("avx2", OPT_mavx2),
4725 IX86_ATTR_ISA ("avx512f", OPT_mavx512f),
4726 IX86_ATTR_ISA ("avx512pf", OPT_mavx512pf),
4727 IX86_ATTR_ISA ("avx512er", OPT_mavx512er),
4728 IX86_ATTR_ISA ("avx512cd", OPT_mavx512cd),
4729 IX86_ATTR_ISA ("avx512dq", OPT_mavx512dq),
4730 IX86_ATTR_ISA ("avx512bw", OPT_mavx512bw),
4731 IX86_ATTR_ISA ("avx512vl", OPT_mavx512vl),
4732 IX86_ATTR_ISA ("mmx", OPT_mmmx),
4733 IX86_ATTR_ISA ("pclmul", OPT_mpclmul),
4734 IX86_ATTR_ISA ("popcnt", OPT_mpopcnt),
4735 IX86_ATTR_ISA ("sse", OPT_msse),
4736 IX86_ATTR_ISA ("sse2", OPT_msse2),
4737 IX86_ATTR_ISA ("sse3", OPT_msse3),
4738 IX86_ATTR_ISA ("sse4", OPT_msse4),
4739 IX86_ATTR_ISA ("sse4.1", OPT_msse4_1),
4740 IX86_ATTR_ISA ("sse4.2", OPT_msse4_2),
4741 IX86_ATTR_ISA ("sse4a", OPT_msse4a),
4742 IX86_ATTR_ISA ("ssse3", OPT_mssse3),
4743 IX86_ATTR_ISA ("fma4", OPT_mfma4),
4744 IX86_ATTR_ISA ("fma", OPT_mfma),
4745 IX86_ATTR_ISA ("xop", OPT_mxop),
4746 IX86_ATTR_ISA ("lwp", OPT_mlwp),
4747 IX86_ATTR_ISA ("fsgsbase", OPT_mfsgsbase),
4748 IX86_ATTR_ISA ("rdrnd", OPT_mrdrnd),
4749 IX86_ATTR_ISA ("f16c", OPT_mf16c),
4750 IX86_ATTR_ISA ("rtm", OPT_mrtm),
4751 IX86_ATTR_ISA ("hle", OPT_mhle),
4752 IX86_ATTR_ISA ("prfchw", OPT_mprfchw),
4753 IX86_ATTR_ISA ("rdseed", OPT_mrdseed),
4754 IX86_ATTR_ISA ("adx", OPT_madx),
4755 IX86_ATTR_ISA ("fxsr", OPT_mfxsr),
4756 IX86_ATTR_ISA ("xsave", OPT_mxsave),
4757 IX86_ATTR_ISA ("xsaveopt", OPT_mxsaveopt),
4758 IX86_ATTR_ISA ("prefetchwt1", OPT_mprefetchwt1),
4759 IX86_ATTR_ISA ("clflushopt", OPT_mclflushopt),
4760 IX86_ATTR_ISA ("xsavec", OPT_mxsavec),
4761 IX86_ATTR_ISA ("xsaves", OPT_mxsaves),
4762 IX86_ATTR_ISA ("avx512vbmi", OPT_mavx512vbmi),
4763 IX86_ATTR_ISA ("avx512ifma", OPT_mavx512ifma),
4764 IX86_ATTR_ISA ("clwb", OPT_mclwb),
4765 IX86_ATTR_ISA ("pcommit", OPT_mpcommit),
4766 IX86_ATTR_ISA ("mwaitx", OPT_mmwaitx),
4769 IX86_ATTR_ENUM ("fpmath=", OPT_mfpmath_),
4771 /* string options */
4772 IX86_ATTR_STR ("arch=", IX86_FUNCTION_SPECIFIC_ARCH),
4773 IX86_ATTR_STR ("tune=", IX86_FUNCTION_SPECIFIC_TUNE),
4776 IX86_ATTR_YES ("cld",
4780 IX86_ATTR_NO ("fancy-math-387",
4781 OPT_mfancy_math_387,
4782 MASK_NO_FANCY_MATH_387),
4784 IX86_ATTR_YES ("ieee-fp",
4788 IX86_ATTR_YES ("inline-all-stringops",
4789 OPT_minline_all_stringops,
4790 MASK_INLINE_ALL_STRINGOPS),
4792 IX86_ATTR_YES ("inline-stringops-dynamically",
4793 OPT_minline_stringops_dynamically,
4794 MASK_INLINE_STRINGOPS_DYNAMICALLY),
4796 IX86_ATTR_NO ("align-stringops",
4797 OPT_mno_align_stringops,
4798 MASK_NO_ALIGN_STRINGOPS),
4800 IX86_ATTR_YES ("recip",
4806 /* If this is a list, recurse to get the options. */
4807 if (TREE_CODE (args) == TREE_LIST)
4811 for (; args; args = TREE_CHAIN (args))
4812 if (TREE_VALUE (args)
4813 && !ix86_valid_target_attribute_inner_p (TREE_VALUE (args),
4814 p_strings, opts, opts_set,
4821 else if (TREE_CODE (args) != STRING_CST)
4823 error ("attribute %<target%> argument not a string");
4827 /* Handle multiple arguments separated by commas. */
4828 next_optstr = ASTRDUP (TREE_STRING_POINTER (args));
4830 while (next_optstr && *next_optstr != '\0')
4832 char *p = next_optstr;
4834 char *comma = strchr (next_optstr, ',');
4835 const char *opt_string;
4836 size_t len, opt_len;
4841 enum ix86_opt_type type = ix86_opt_unknown;
4847 len = comma - next_optstr;
4848 next_optstr = comma + 1;
4856 /* Recognize no-xxx. */
4857 if (len > 3 && p[0] == 'n' && p[1] == 'o' && p[2] == '-')
4866 /* Find the option. */
4869 for (i = 0; i < ARRAY_SIZE (attrs); i++)
4871 type = attrs[i].type;
4872 opt_len = attrs[i].len;
4873 if (ch == attrs[i].string[0]
4874 && ((type != ix86_opt_str && type != ix86_opt_enum)
4877 && memcmp (p, attrs[i].string, opt_len) == 0)
4880 mask = attrs[i].mask;
4881 opt_string = attrs[i].string;
4886 /* Process the option. */
4889 error ("attribute(target(\"%s\")) is unknown", orig_p);
4893 else if (type == ix86_opt_isa)
4895 struct cl_decoded_option decoded;
4897 generate_option (opt, NULL, opt_set_p, CL_TARGET, &decoded);
4898 ix86_handle_option (opts, opts_set,
4899 &decoded, input_location);
4902 else if (type == ix86_opt_yes || type == ix86_opt_no)
4904 if (type == ix86_opt_no)
4905 opt_set_p = !opt_set_p;
4908 opts->x_target_flags |= mask;
4910 opts->x_target_flags &= ~mask;
4913 else if (type == ix86_opt_str)
4917 error ("option(\"%s\") was already specified", opt_string);
4921 p_strings[opt] = xstrdup (p + opt_len);
4924 else if (type == ix86_opt_enum)
4929 arg_ok = opt_enum_arg_to_value (opt, p + opt_len, &value, CL_TARGET);
4931 set_option (opts, enum_opts_set, opt, value,
4932 p + opt_len, DK_UNSPECIFIED, input_location,
4936 error ("attribute(target(\"%s\")) is unknown", orig_p);
4948 /* Return a TARGET_OPTION_NODE tree of the target options listed or NULL. */
4951 ix86_valid_target_attribute_tree (tree args,
4952 struct gcc_options *opts,
4953 struct gcc_options *opts_set)
4955 const char *orig_arch_string = opts->x_ix86_arch_string;
4956 const char *orig_tune_string = opts->x_ix86_tune_string;
4957 enum fpmath_unit orig_fpmath_set = opts_set->x_ix86_fpmath;
4958 int orig_tune_defaulted = ix86_tune_defaulted;
4959 int orig_arch_specified = ix86_arch_specified;
4960 char *option_strings[IX86_FUNCTION_SPECIFIC_MAX] = { NULL, NULL };
4963 struct cl_target_option *def
4964 = TREE_TARGET_OPTION (target_option_default_node);
4965 struct gcc_options enum_opts_set;
4967 memset (&enum_opts_set, 0, sizeof (enum_opts_set));
4969 /* Process each of the options on the chain. */
4970 if (! ix86_valid_target_attribute_inner_p (args, option_strings, opts,
4971 opts_set, &enum_opts_set))
4972 return error_mark_node;
4974 /* If the changed options are different from the default, rerun
4975 ix86_option_override_internal, and then save the options away.
4976 The string options are are attribute options, and will be undone
4977 when we copy the save structure. */
4978 if (opts->x_ix86_isa_flags != def->x_ix86_isa_flags
4979 || opts->x_target_flags != def->x_target_flags
4980 || option_strings[IX86_FUNCTION_SPECIFIC_ARCH]
4981 || option_strings[IX86_FUNCTION_SPECIFIC_TUNE]
4982 || enum_opts_set.x_ix86_fpmath)
4984 /* If we are using the default tune= or arch=, undo the string assigned,
4985 and use the default. */
4986 if (option_strings[IX86_FUNCTION_SPECIFIC_ARCH])
4987 opts->x_ix86_arch_string = option_strings[IX86_FUNCTION_SPECIFIC_ARCH];
4988 else if (!orig_arch_specified)
4989 opts->x_ix86_arch_string = NULL;
4991 if (option_strings[IX86_FUNCTION_SPECIFIC_TUNE])
4992 opts->x_ix86_tune_string = option_strings[IX86_FUNCTION_SPECIFIC_TUNE];
4993 else if (orig_tune_defaulted)
4994 opts->x_ix86_tune_string = NULL;
4996 /* If fpmath= is not set, and we now have sse2 on 32-bit, use it. */
4997 if (enum_opts_set.x_ix86_fpmath)
4998 opts_set->x_ix86_fpmath = (enum fpmath_unit) 1;
4999 else if (!TARGET_64BIT_P (opts->x_ix86_isa_flags)
5000 && TARGET_SSE_P (opts->x_ix86_isa_flags))
5002 opts->x_ix86_fpmath = (enum fpmath_unit) (FPMATH_SSE | FPMATH_387);
5003 opts_set->x_ix86_fpmath = (enum fpmath_unit) 1;
5006 /* Do any overrides, such as arch=xxx, or tune=xxx support. */
5007 ix86_option_override_internal (false, opts, opts_set);
5009 /* Add any builtin functions with the new isa if any. */
5010 ix86_add_new_builtins (opts->x_ix86_isa_flags);
5012 /* Save the current options unless we are validating options for
5014 t = build_target_option_node (opts);
5016 opts->x_ix86_arch_string = orig_arch_string;
5017 opts->x_ix86_tune_string = orig_tune_string;
5018 opts_set->x_ix86_fpmath = orig_fpmath_set;
5020 /* Free up memory allocated to hold the strings */
5021 for (i = 0; i < IX86_FUNCTION_SPECIFIC_MAX; i++)
5022 free (option_strings[i]);
5028 /* Hook to validate attribute((target("string"))). */
5031 ix86_valid_target_attribute_p (tree fndecl,
5032 tree ARG_UNUSED (name),
5034 int ARG_UNUSED (flags))
5036 struct gcc_options func_options;
5037 tree new_target, new_optimize;
5040 /* attribute((target("default"))) does nothing, beyond
5041 affecting multi-versioning. */
5042 if (TREE_VALUE (args)
5043 && TREE_CODE (TREE_VALUE (args)) == STRING_CST
5044 && TREE_CHAIN (args) == NULL_TREE
5045 && strcmp (TREE_STRING_POINTER (TREE_VALUE (args)), "default") == 0)
5048 tree old_optimize = build_optimization_node (&global_options);
5050 /* Get the optimization options of the current function. */
5051 tree func_optimize = DECL_FUNCTION_SPECIFIC_OPTIMIZATION (fndecl);
5054 func_optimize = old_optimize;
5056 /* Init func_options. */
5057 memset (&func_options, 0, sizeof (func_options));
5058 init_options_struct (&func_options, NULL);
5059 lang_hooks.init_options_struct (&func_options);
5061 cl_optimization_restore (&func_options,
5062 TREE_OPTIMIZATION (func_optimize));
5064 /* Initialize func_options to the default before its target options can
5066 cl_target_option_restore (&func_options,
5067 TREE_TARGET_OPTION (target_option_default_node));
5069 new_target = ix86_valid_target_attribute_tree (args, &func_options,
5070 &global_options_set);
5072 new_optimize = build_optimization_node (&func_options);
5074 if (new_target == error_mark_node)
5077 else if (fndecl && new_target)
5079 DECL_FUNCTION_SPECIFIC_TARGET (fndecl) = new_target;
5081 if (old_optimize != new_optimize)
5082 DECL_FUNCTION_SPECIFIC_OPTIMIZATION (fndecl) = new_optimize;
5089 /* Hook to determine if one function can safely inline another. */
5092 ix86_can_inline_p (tree caller, tree callee)
5095 tree caller_tree = DECL_FUNCTION_SPECIFIC_TARGET (caller);
5096 tree callee_tree = DECL_FUNCTION_SPECIFIC_TARGET (callee);
5098 /* If callee has no option attributes, then it is ok to inline. */
5102 /* If caller has no option attributes, but callee does then it is not ok to
5104 else if (!caller_tree)
5109 struct cl_target_option *caller_opts = TREE_TARGET_OPTION (caller_tree);
5110 struct cl_target_option *callee_opts = TREE_TARGET_OPTION (callee_tree);
5112 /* Callee's isa options should a subset of the caller's, i.e. a SSE4 function
5113 can inline a SSE2 function but a SSE2 function can't inline a SSE4
5115 if ((caller_opts->x_ix86_isa_flags & callee_opts->x_ix86_isa_flags)
5116 != callee_opts->x_ix86_isa_flags)
5119 /* See if we have the same non-isa options. */
5120 else if (caller_opts->x_target_flags != callee_opts->x_target_flags)
5123 /* See if arch, tune, etc. are the same. */
5124 else if (caller_opts->arch != callee_opts->arch)
5127 else if (caller_opts->tune != callee_opts->tune)
5130 else if (caller_opts->x_ix86_fpmath != callee_opts->x_ix86_fpmath)
5133 else if (caller_opts->branch_cost != callee_opts->branch_cost)
5144 /* Remember the last target of ix86_set_current_function. */
5145 static GTY(()) tree ix86_previous_fndecl;
5147 /* Set targets globals to the default (or current #pragma GCC target
5148 if active). Invalidate ix86_previous_fndecl cache. */
5151 ix86_reset_previous_fndecl (void)
5153 tree new_tree = target_option_current_node;
5154 cl_target_option_restore (&global_options, TREE_TARGET_OPTION (new_tree));
5155 if (TREE_TARGET_GLOBALS (new_tree))
5156 restore_target_globals (TREE_TARGET_GLOBALS (new_tree));
5157 else if (new_tree == target_option_default_node)
5158 restore_target_globals (&default_target_globals);
5160 TREE_TARGET_GLOBALS (new_tree) = save_target_globals_default_opts ();
5161 ix86_previous_fndecl = NULL_TREE;
5164 /* Establish appropriate back-end context for processing the function
5165 FNDECL. The argument might be NULL to indicate processing at top
5166 level, outside of any function scope. */
5168 ix86_set_current_function (tree fndecl)
5170 /* Only change the context if the function changes. This hook is called
5171 several times in the course of compiling a function, and we don't want to
5172 slow things down too much or call target_reinit when it isn't safe. */
5173 if (fndecl == ix86_previous_fndecl)
5177 if (ix86_previous_fndecl == NULL_TREE)
5178 old_tree = target_option_current_node;
5179 else if (DECL_FUNCTION_SPECIFIC_TARGET (ix86_previous_fndecl))
5180 old_tree = DECL_FUNCTION_SPECIFIC_TARGET (ix86_previous_fndecl);
5182 old_tree = target_option_default_node;
5184 if (fndecl == NULL_TREE)
5186 if (old_tree != target_option_current_node)
5187 ix86_reset_previous_fndecl ();
5191 tree new_tree = DECL_FUNCTION_SPECIFIC_TARGET (fndecl);
5192 if (new_tree == NULL_TREE)
5193 new_tree = target_option_default_node;
5195 if (old_tree != new_tree)
5197 cl_target_option_restore (&global_options, TREE_TARGET_OPTION (new_tree));
5198 if (TREE_TARGET_GLOBALS (new_tree))
5199 restore_target_globals (TREE_TARGET_GLOBALS (new_tree));
5200 else if (new_tree == target_option_default_node)
5201 restore_target_globals (&default_target_globals);
5203 TREE_TARGET_GLOBALS (new_tree) = save_target_globals_default_opts ();
5205 ix86_previous_fndecl = fndecl;
5209 /* Return true if this goes in large data/bss. */
5212 ix86_in_large_data_p (tree exp)
5214 if (ix86_cmodel != CM_MEDIUM && ix86_cmodel != CM_MEDIUM_PIC)
5217 /* Functions are never large data. */
5218 if (TREE_CODE (exp) == FUNCTION_DECL)
5221 /* Automatic variables are never large data. */
5222 if (TREE_CODE (exp) == VAR_DECL && !is_global_var (exp))
5225 if (TREE_CODE (exp) == VAR_DECL && DECL_SECTION_NAME (exp))
5227 const char *section = DECL_SECTION_NAME (exp);
5228 if (strcmp (section, ".ldata") == 0
5229 || strcmp (section, ".lbss") == 0)
5235 HOST_WIDE_INT size = int_size_in_bytes (TREE_TYPE (exp));
5237 /* If this is an incomplete type with size 0, then we can't put it
5238 in data because it might be too big when completed. Also,
5239 int_size_in_bytes returns -1 if size can vary or is larger than
5240 an integer in which case also it is safer to assume that it goes in
5242 if (size <= 0 || size > ix86_section_threshold)
5249 /* Switch to the appropriate section for output of DECL.
5250 DECL is either a `VAR_DECL' node or a constant of some sort.
5251 RELOC indicates whether forming the initial value of DECL requires
5252 link-time relocations. */
5254 ATTRIBUTE_UNUSED static section *
5255 x86_64_elf_select_section (tree decl, int reloc,
5256 unsigned HOST_WIDE_INT align)
5258 if (ix86_in_large_data_p (decl))
5260 const char *sname = NULL;
5261 unsigned int flags = SECTION_WRITE;
5262 switch (categorize_decl_for_section (decl, reloc))
5267 case SECCAT_DATA_REL:
5268 sname = ".ldata.rel";
5270 case SECCAT_DATA_REL_LOCAL:
5271 sname = ".ldata.rel.local";
5273 case SECCAT_DATA_REL_RO:
5274 sname = ".ldata.rel.ro";
5276 case SECCAT_DATA_REL_RO_LOCAL:
5277 sname = ".ldata.rel.ro.local";
5281 flags |= SECTION_BSS;
5284 case SECCAT_RODATA_MERGE_STR:
5285 case SECCAT_RODATA_MERGE_STR_INIT:
5286 case SECCAT_RODATA_MERGE_CONST:
5290 case SECCAT_SRODATA:
5297 /* We don't split these for medium model. Place them into
5298 default sections and hope for best. */
5303 /* We might get called with string constants, but get_named_section
5304 doesn't like them as they are not DECLs. Also, we need to set
5305 flags in that case. */
5307 return get_section (sname, flags, NULL);
5308 return get_named_section (decl, sname, reloc);
5311 return default_elf_select_section (decl, reloc, align);
5314 /* Select a set of attributes for section NAME based on the properties
5315 of DECL and whether or not RELOC indicates that DECL's initializer
5316 might contain runtime relocations. */
5318 static unsigned int ATTRIBUTE_UNUSED
5319 x86_64_elf_section_type_flags (tree decl, const char *name, int reloc)
5321 unsigned int flags = default_section_type_flags (decl, name, reloc);
5323 if (decl == NULL_TREE
5324 && (strcmp (name, ".ldata.rel.ro") == 0
5325 || strcmp (name, ".ldata.rel.ro.local") == 0))
5326 flags |= SECTION_RELRO;
5328 if (strcmp (name, ".lbss") == 0
5329 || strncmp (name, ".lbss.", 5) == 0
5330 || strncmp (name, ".gnu.linkonce.lb.", 16) == 0)
5331 flags |= SECTION_BSS;
5336 /* Build up a unique section name, expressed as a
5337 STRING_CST node, and assign it to DECL_SECTION_NAME (decl).
5338 RELOC indicates whether the initial value of EXP requires
5339 link-time relocations. */
5341 static void ATTRIBUTE_UNUSED
5342 x86_64_elf_unique_section (tree decl, int reloc)
5344 if (ix86_in_large_data_p (decl))
5346 const char *prefix = NULL;
5347 /* We only need to use .gnu.linkonce if we don't have COMDAT groups. */
5348 bool one_only = DECL_COMDAT_GROUP (decl) && !HAVE_COMDAT_GROUP;
5350 switch (categorize_decl_for_section (decl, reloc))
5353 case SECCAT_DATA_REL:
5354 case SECCAT_DATA_REL_LOCAL:
5355 case SECCAT_DATA_REL_RO:
5356 case SECCAT_DATA_REL_RO_LOCAL:
5357 prefix = one_only ? ".ld" : ".ldata";
5360 prefix = one_only ? ".lb" : ".lbss";
5363 case SECCAT_RODATA_MERGE_STR:
5364 case SECCAT_RODATA_MERGE_STR_INIT:
5365 case SECCAT_RODATA_MERGE_CONST:
5366 prefix = one_only ? ".lr" : ".lrodata";
5368 case SECCAT_SRODATA:
5375 /* We don't split these for medium model. Place them into
5376 default sections and hope for best. */
5381 const char *name, *linkonce;
5384 name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl));
5385 name = targetm.strip_name_encoding (name);
5387 /* If we're using one_only, then there needs to be a .gnu.linkonce
5388 prefix to the section name. */
5389 linkonce = one_only ? ".gnu.linkonce" : "";
5391 string = ACONCAT ((linkonce, prefix, ".", name, NULL));
5393 set_decl_section_name (decl, string);
5397 default_unique_section (decl, reloc);
5400 #ifdef COMMON_ASM_OP
5401 /* This says how to output assembler code to declare an
5402 uninitialized external linkage data object.
5404 For medium model x86-64 we need to use .largecomm opcode for
5407 x86_elf_aligned_common (FILE *file,
5408 const char *name, unsigned HOST_WIDE_INT size,
5411 if ((ix86_cmodel == CM_MEDIUM || ix86_cmodel == CM_MEDIUM_PIC)
5412 && size > (unsigned int)ix86_section_threshold)
5413 fputs ("\t.largecomm\t", file);
5415 fputs (COMMON_ASM_OP, file);
5416 assemble_name (file, name);
5417 fprintf (file, "," HOST_WIDE_INT_PRINT_UNSIGNED ",%u\n",
5418 size, align / BITS_PER_UNIT);
5422 /* Utility function for targets to use in implementing
5423 ASM_OUTPUT_ALIGNED_BSS. */
5426 x86_output_aligned_bss (FILE *file, tree decl, const char *name,
5427 unsigned HOST_WIDE_INT size, int align)
5429 if ((ix86_cmodel == CM_MEDIUM || ix86_cmodel == CM_MEDIUM_PIC)
5430 && size > (unsigned int)ix86_section_threshold)
5431 switch_to_section (get_named_section (decl, ".lbss", 0));
5433 switch_to_section (bss_section);
5434 ASM_OUTPUT_ALIGN (file, floor_log2 (align / BITS_PER_UNIT));
5435 #ifdef ASM_DECLARE_OBJECT_NAME
5436 last_assemble_variable_decl = decl;
5437 ASM_DECLARE_OBJECT_NAME (file, name, decl);
5439 /* Standard thing is just output label for the object. */
5440 ASM_OUTPUT_LABEL (file, name);
5441 #endif /* ASM_DECLARE_OBJECT_NAME */
5442 ASM_OUTPUT_SKIP (file, size ? size : 1);
5445 /* Decide whether we must probe the stack before any space allocation
5446 on this target. It's essentially TARGET_STACK_PROBE except when
5447 -fstack-check causes the stack to be already probed differently. */
5450 ix86_target_stack_probe (void)
5452 /* Do not probe the stack twice if static stack checking is enabled. */
5453 if (flag_stack_check == STATIC_BUILTIN_STACK_CHECK)
5456 return TARGET_STACK_PROBE;
5459 /* Decide whether we can make a sibling call to a function. DECL is the
5460 declaration of the function being targeted by the call and EXP is the
5461 CALL_EXPR representing the call. */
5464 ix86_function_ok_for_sibcall (tree decl, tree exp)
5466 tree type, decl_or_type;
5469 /* If we are generating position-independent code, we cannot sibcall
5470 optimize direct calls to global functions, as the PLT requires
5471 %ebx be live. (Darwin does not have a PLT.) */
5476 && decl && !targetm.binds_local_p (decl))
5479 /* If we need to align the outgoing stack, then sibcalling would
5480 unalign the stack, which may break the called function. */
5481 if (ix86_minimum_incoming_stack_boundary (true)
5482 < PREFERRED_STACK_BOUNDARY)
5487 decl_or_type = decl;
5488 type = TREE_TYPE (decl);
5492 /* We're looking at the CALL_EXPR, we need the type of the function. */
5493 type = CALL_EXPR_FN (exp); /* pointer expression */
5494 type = TREE_TYPE (type); /* pointer type */
5495 type = TREE_TYPE (type); /* function type */
5496 decl_or_type = type;
5499 /* Check that the return value locations are the same. Like
5500 if we are returning floats on the 80387 register stack, we cannot
5501 make a sibcall from a function that doesn't return a float to a
5502 function that does or, conversely, from a function that does return
5503 a float to a function that doesn't; the necessary stack adjustment
5504 would not be executed. This is also the place we notice
5505 differences in the return value ABI. Note that it is ok for one
5506 of the functions to have void return type as long as the return
5507 value of the other is passed in a register. */
5508 a = ix86_function_value (TREE_TYPE (exp), decl_or_type, false);
5509 b = ix86_function_value (TREE_TYPE (DECL_RESULT (cfun->decl)),
5511 if (STACK_REG_P (a) || STACK_REG_P (b))
5513 if (!rtx_equal_p (a, b))
5516 else if (VOID_TYPE_P (TREE_TYPE (DECL_RESULT (cfun->decl))))
5518 else if (!rtx_equal_p (a, b))
5523 /* The SYSV ABI has more call-clobbered registers;
5524 disallow sibcalls from MS to SYSV. */
5525 if (cfun->machine->call_abi == MS_ABI
5526 && ix86_function_type_abi (type) == SYSV_ABI)
5531 /* If this call is indirect, we'll need to be able to use a
5532 call-clobbered register for the address of the target function.
5533 Make sure that all such registers are not used for passing
5534 parameters. Note that DLLIMPORT functions are indirect. */
5536 || (TARGET_DLLIMPORT_DECL_ATTRIBUTES && DECL_DLLIMPORT_P (decl)))
5538 if (ix86_function_regparm (type, NULL) >= 3)
5540 /* ??? Need to count the actual number of registers to be used,
5541 not the possible number of registers. Fix later. */
5547 /* Otherwise okay. That also includes certain types of indirect calls. */
5551 /* Handle "cdecl", "stdcall", "fastcall", "regparm", "thiscall",
5552 and "sseregparm" calling convention attributes;
5553 arguments as in struct attribute_spec.handler. */
5556 ix86_handle_cconv_attribute (tree *node, tree name,
5561 if (TREE_CODE (*node) != FUNCTION_TYPE
5562 && TREE_CODE (*node) != METHOD_TYPE
5563 && TREE_CODE (*node) != FIELD_DECL
5564 && TREE_CODE (*node) != TYPE_DECL)
5566 warning (OPT_Wattributes, "%qE attribute only applies to functions",
5568 *no_add_attrs = true;
5572 /* Can combine regparm with all attributes but fastcall, and thiscall. */
5573 if (is_attribute_p ("regparm", name))
5577 if (lookup_attribute ("fastcall", TYPE_ATTRIBUTES (*node)))
5579 error ("fastcall and regparm attributes are not compatible");
5582 if (lookup_attribute ("thiscall", TYPE_ATTRIBUTES (*node)))
5584 error ("regparam and thiscall attributes are not compatible");
5587 cst = TREE_VALUE (args);
5588 if (TREE_CODE (cst) != INTEGER_CST)
5590 warning (OPT_Wattributes,
5591 "%qE attribute requires an integer constant argument",
5593 *no_add_attrs = true;
5595 else if (compare_tree_int (cst, REGPARM_MAX) > 0)
5597 warning (OPT_Wattributes, "argument to %qE attribute larger than %d",
5599 *no_add_attrs = true;
5607 /* Do not warn when emulating the MS ABI. */
5608 if ((TREE_CODE (*node) != FUNCTION_TYPE
5609 && TREE_CODE (*node) != METHOD_TYPE)
5610 || ix86_function_type_abi (*node) != MS_ABI)
5611 warning (OPT_Wattributes, "%qE attribute ignored",
5613 *no_add_attrs = true;
5617 /* Can combine fastcall with stdcall (redundant) and sseregparm. */
5618 if (is_attribute_p ("fastcall", name))
5620 if (lookup_attribute ("cdecl", TYPE_ATTRIBUTES (*node)))
5622 error ("fastcall and cdecl attributes are not compatible");
5624 if (lookup_attribute ("stdcall", TYPE_ATTRIBUTES (*node)))
5626 error ("fastcall and stdcall attributes are not compatible");
5628 if (lookup_attribute ("regparm", TYPE_ATTRIBUTES (*node)))
5630 error ("fastcall and regparm attributes are not compatible");
5632 if (lookup_attribute ("thiscall", TYPE_ATTRIBUTES (*node)))
5634 error ("fastcall and thiscall attributes are not compatible");
5638 /* Can combine stdcall with fastcall (redundant), regparm and
5640 else if (is_attribute_p ("stdcall", name))
5642 if (lookup_attribute ("cdecl", TYPE_ATTRIBUTES (*node)))
5644 error ("stdcall and cdecl attributes are not compatible");
5646 if (lookup_attribute ("fastcall", TYPE_ATTRIBUTES (*node)))
5648 error ("stdcall and fastcall attributes are not compatible");
5650 if (lookup_attribute ("thiscall", TYPE_ATTRIBUTES (*node)))
5652 error ("stdcall and thiscall attributes are not compatible");
5656 /* Can combine cdecl with regparm and sseregparm. */
5657 else if (is_attribute_p ("cdecl", name))
5659 if (lookup_attribute ("stdcall", TYPE_ATTRIBUTES (*node)))
5661 error ("stdcall and cdecl attributes are not compatible");
5663 if (lookup_attribute ("fastcall", TYPE_ATTRIBUTES (*node)))
5665 error ("fastcall and cdecl attributes are not compatible");
5667 if (lookup_attribute ("thiscall", TYPE_ATTRIBUTES (*node)))
5669 error ("cdecl and thiscall attributes are not compatible");
5672 else if (is_attribute_p ("thiscall", name))
5674 if (TREE_CODE (*node) != METHOD_TYPE && pedantic)
5675 warning (OPT_Wattributes, "%qE attribute is used for non-class method",
5677 if (lookup_attribute ("stdcall", TYPE_ATTRIBUTES (*node)))
5679 error ("stdcall and thiscall attributes are not compatible");
5681 if (lookup_attribute ("fastcall", TYPE_ATTRIBUTES (*node)))
5683 error ("fastcall and thiscall attributes are not compatible");
5685 if (lookup_attribute ("cdecl", TYPE_ATTRIBUTES (*node)))
5687 error ("cdecl and thiscall attributes are not compatible");
5691 /* Can combine sseregparm with all attributes. */
5696 /* The transactional memory builtins are implicitly regparm or fastcall
5697 depending on the ABI. Override the generic do-nothing attribute that
5698 these builtins were declared with, and replace it with one of the two
5699 attributes that we expect elsewhere. */
5702 ix86_handle_tm_regparm_attribute (tree *node, tree, tree,
5703 int flags, bool *no_add_attrs)
5707 /* In no case do we want to add the placeholder attribute. */
5708 *no_add_attrs = true;
5710 /* The 64-bit ABI is unchanged for transactional memory. */
5714 /* ??? Is there a better way to validate 32-bit windows? We have
5715 cfun->machine->call_abi, but that seems to be set only for 64-bit. */
5716 if (CHECK_STACK_LIMIT > 0)
5717 alt = tree_cons (get_identifier ("fastcall"), NULL, NULL);
5720 alt = tree_cons (NULL, build_int_cst (NULL, 2), NULL);
5721 alt = tree_cons (get_identifier ("regparm"), alt, NULL);
5723 decl_attributes (node, alt, flags);
5728 /* This function determines from TYPE the calling-convention. */
5731 ix86_get_callcvt (const_tree type)
5733 unsigned int ret = 0;
5738 return IX86_CALLCVT_CDECL;
5740 attrs = TYPE_ATTRIBUTES (type);
5741 if (attrs != NULL_TREE)
5743 if (lookup_attribute ("cdecl", attrs))
5744 ret |= IX86_CALLCVT_CDECL;
5745 else if (lookup_attribute ("stdcall", attrs))
5746 ret |= IX86_CALLCVT_STDCALL;
5747 else if (lookup_attribute ("fastcall", attrs))
5748 ret |= IX86_CALLCVT_FASTCALL;
5749 else if (lookup_attribute ("thiscall", attrs))
5750 ret |= IX86_CALLCVT_THISCALL;
5752 /* Regparam isn't allowed for thiscall and fastcall. */
5753 if ((ret & (IX86_CALLCVT_THISCALL | IX86_CALLCVT_FASTCALL)) == 0)
5755 if (lookup_attribute ("regparm", attrs))
5756 ret |= IX86_CALLCVT_REGPARM;
5757 if (lookup_attribute ("sseregparm", attrs))
5758 ret |= IX86_CALLCVT_SSEREGPARM;
5761 if (IX86_BASE_CALLCVT(ret) != 0)
5765 is_stdarg = stdarg_p (type);
5766 if (TARGET_RTD && !is_stdarg)
5767 return IX86_CALLCVT_STDCALL | ret;
5771 || TREE_CODE (type) != METHOD_TYPE
5772 || ix86_function_type_abi (type) != MS_ABI)
5773 return IX86_CALLCVT_CDECL | ret;
5775 return IX86_CALLCVT_THISCALL;
5778 /* Return 0 if the attributes for two types are incompatible, 1 if they
5779 are compatible, and 2 if they are nearly compatible (which causes a
5780 warning to be generated). */
5783 ix86_comp_type_attributes (const_tree type1, const_tree type2)
5785 unsigned int ccvt1, ccvt2;
5787 if (TREE_CODE (type1) != FUNCTION_TYPE
5788 && TREE_CODE (type1) != METHOD_TYPE)
5791 ccvt1 = ix86_get_callcvt (type1);
5792 ccvt2 = ix86_get_callcvt (type2);
5795 if (ix86_function_regparm (type1, NULL)
5796 != ix86_function_regparm (type2, NULL))
5802 /* Return the regparm value for a function with the indicated TYPE and DECL.
5803 DECL may be NULL when calling function indirectly
5804 or considering a libcall. */
5807 ix86_function_regparm (const_tree type, const_tree decl)
5814 return (ix86_function_type_abi (type) == SYSV_ABI
5815 ? X86_64_REGPARM_MAX : X86_64_MS_REGPARM_MAX);
5816 ccvt = ix86_get_callcvt (type);
5817 regparm = ix86_regparm;
5819 if ((ccvt & IX86_CALLCVT_REGPARM) != 0)
5821 attr = lookup_attribute ("regparm", TYPE_ATTRIBUTES (type));
5824 regparm = TREE_INT_CST_LOW (TREE_VALUE (TREE_VALUE (attr)));
5828 else if ((ccvt & IX86_CALLCVT_FASTCALL) != 0)
5830 else if ((ccvt & IX86_CALLCVT_THISCALL) != 0)
5833 /* Use register calling convention for local functions when possible. */
5835 && TREE_CODE (decl) == FUNCTION_DECL)
5837 cgraph_node *target = cgraph_node::get (decl);
5839 target = target->function_symbol ();
5841 /* Caller and callee must agree on the calling convention, so
5842 checking here just optimize means that with
5843 __attribute__((optimize (...))) caller could use regparm convention
5844 and callee not, or vice versa. Instead look at whether the callee
5845 is optimized or not. */
5846 if (target && opt_for_fn (target->decl, optimize)
5847 && !(profile_flag && !flag_fentry))
5849 cgraph_local_info *i = &target->local;
5850 if (i && i->local && i->can_change_signature)
5852 int local_regparm, globals = 0, regno;
5854 /* Make sure no regparm register is taken by a
5855 fixed register variable. */
5856 for (local_regparm = 0; local_regparm < REGPARM_MAX;
5858 if (fixed_regs[local_regparm])
5861 /* We don't want to use regparm(3) for nested functions as
5862 these use a static chain pointer in the third argument. */
5863 if (local_regparm == 3 && DECL_STATIC_CHAIN (target->decl))
5866 /* Save a register for the split stack. */
5867 if (local_regparm == 3 && flag_split_stack)
5870 /* Each fixed register usage increases register pressure,
5871 so less registers should be used for argument passing.
5872 This functionality can be overriden by an explicit
5874 for (regno = AX_REG; regno <= DI_REG; regno++)
5875 if (fixed_regs[regno])
5879 = globals < local_regparm ? local_regparm - globals : 0;
5881 if (local_regparm > regparm)
5882 regparm = local_regparm;
5890 /* Return 1 or 2, if we can pass up to SSE_REGPARM_MAX SFmode (1) and
5891 DFmode (2) arguments in SSE registers for a function with the
5892 indicated TYPE and DECL. DECL may be NULL when calling function
5893 indirectly or considering a libcall. Return -1 if any FP parameter
5894 should be rejected by error. This is used in siutation we imply SSE
5895 calling convetion but the function is called from another function with
5896 SSE disabled. Otherwise return 0. */
5899 ix86_function_sseregparm (const_tree type, const_tree decl, bool warn)
5901 gcc_assert (!TARGET_64BIT);
5903 /* Use SSE registers to pass SFmode and DFmode arguments if requested
5904 by the sseregparm attribute. */
5905 if (TARGET_SSEREGPARM
5906 || (type && lookup_attribute ("sseregparm", TYPE_ATTRIBUTES (type))))
5913 error ("calling %qD with attribute sseregparm without "
5914 "SSE/SSE2 enabled", decl);
5916 error ("calling %qT with attribute sseregparm without "
5917 "SSE/SSE2 enabled", type);
5928 cgraph_node *target = cgraph_node::get (decl);
5930 target = target->function_symbol ();
5932 /* For local functions, pass up to SSE_REGPARM_MAX SFmode
5933 (and DFmode for SSE2) arguments in SSE registers. */
5935 /* TARGET_SSE_MATH */
5936 && (target_opts_for_fn (target->decl)->x_ix86_fpmath & FPMATH_SSE)
5937 && opt_for_fn (target->decl, optimize)
5938 && !(profile_flag && !flag_fentry))
5940 cgraph_local_info *i = &target->local;
5941 if (i && i->local && i->can_change_signature)
5943 /* Refuse to produce wrong code when local function with SSE enabled
5944 is called from SSE disabled function.
5945 FIXME: We need a way to detect these cases cross-ltrans partition
5946 and avoid using SSE calling conventions on local functions called
5947 from function with SSE disabled. For now at least delay the
5948 warning until we know we are going to produce wrong code.
5950 if (!TARGET_SSE && warn)
5952 return TARGET_SSE2_P (target_opts_for_fn (target->decl)
5953 ->x_ix86_isa_flags) ? 2 : 1;
5960 /* Return true if EAX is live at the start of the function. Used by
5961 ix86_expand_prologue to determine if we need special help before
5962 calling allocate_stack_worker. */
5965 ix86_eax_live_at_start_p (void)
5967 /* Cheat. Don't bother working forward from ix86_function_regparm
5968 to the function type to whether an actual argument is located in
5969 eax. Instead just look at cfg info, which is still close enough
5970 to correct at this point. This gives false positives for broken
5971 functions that might use uninitialized data that happens to be
5972 allocated in eax, but who cares? */
5973 return REGNO_REG_SET_P (df_get_live_out (ENTRY_BLOCK_PTR_FOR_FN (cfun)), 0);
5977 ix86_keep_aggregate_return_pointer (tree fntype)
5983 attr = lookup_attribute ("callee_pop_aggregate_return",
5984 TYPE_ATTRIBUTES (fntype));
5986 return (TREE_INT_CST_LOW (TREE_VALUE (TREE_VALUE (attr))) == 0);
5988 /* For 32-bit MS-ABI the default is to keep aggregate
5990 if (ix86_function_type_abi (fntype) == MS_ABI)
5993 return KEEP_AGGREGATE_RETURN_POINTER != 0;
5996 /* Value is the number of bytes of arguments automatically
5997 popped when returning from a subroutine call.
5998 FUNDECL is the declaration node of the function (as a tree),
5999 FUNTYPE is the data type of the function (as a tree),
6000 or for a library call it is an identifier node for the subroutine name.
6001 SIZE is the number of bytes of arguments passed on the stack.
6003 On the 80386, the RTD insn may be used to pop them if the number
6004 of args is fixed, but if the number is variable then the caller
6005 must pop them all. RTD can't be used for library calls now
6006 because the library is compiled with the Unix compiler.
6007 Use of RTD is a selectable option, since it is incompatible with
6008 standard Unix calling sequences. If the option is not selected,
6009 the caller must always pop the args.
6011 The attribute stdcall is equivalent to RTD on a per module basis. */
6014 ix86_return_pops_args (tree fundecl, tree funtype, int size)
6018 /* None of the 64-bit ABIs pop arguments. */
6022 ccvt = ix86_get_callcvt (funtype);
6024 if ((ccvt & (IX86_CALLCVT_STDCALL | IX86_CALLCVT_FASTCALL
6025 | IX86_CALLCVT_THISCALL)) != 0
6026 && ! stdarg_p (funtype))
6029 /* Lose any fake structure return argument if it is passed on the stack. */
6030 if (aggregate_value_p (TREE_TYPE (funtype), fundecl)
6031 && !ix86_keep_aggregate_return_pointer (funtype))
6033 int nregs = ix86_function_regparm (funtype, fundecl);
6035 return GET_MODE_SIZE (Pmode);
6041 /* Implement the TARGET_LEGITIMATE_COMBINED_INSN hook. */
6044 ix86_legitimate_combined_insn (rtx_insn *insn)
6046 /* Check operand constraints in case hard registers were propagated
6047 into insn pattern. This check prevents combine pass from
6048 generating insn patterns with invalid hard register operands.
6049 These invalid insns can eventually confuse reload to error out
6050 with a spill failure. See also PRs 46829 and 46843. */
6051 if ((INSN_CODE (insn) = recog (PATTERN (insn), insn, 0)) >= 0)
6055 extract_insn (insn);
6056 preprocess_constraints (insn);
6058 int n_operands = recog_data.n_operands;
6059 int n_alternatives = recog_data.n_alternatives;
6060 for (i = 0; i < n_operands; i++)
6062 rtx op = recog_data.operand[i];
6063 machine_mode mode = GET_MODE (op);
6064 const operand_alternative *op_alt;
6069 /* For pre-AVX disallow unaligned loads/stores where the
6070 instructions don't support it. */
6072 && VECTOR_MODE_P (GET_MODE (op))
6073 && misaligned_operand (op, GET_MODE (op)))
6075 int min_align = get_attr_ssememalign (insn);
6080 /* A unary operator may be accepted by the predicate, but it
6081 is irrelevant for matching constraints. */
6085 if (GET_CODE (op) == SUBREG)
6087 if (REG_P (SUBREG_REG (op))
6088 && REGNO (SUBREG_REG (op)) < FIRST_PSEUDO_REGISTER)
6089 offset = subreg_regno_offset (REGNO (SUBREG_REG (op)),
6090 GET_MODE (SUBREG_REG (op)),
6093 op = SUBREG_REG (op);
6096 if (!(REG_P (op) && HARD_REGISTER_P (op)))
6099 op_alt = recog_op_alt;
6101 /* Operand has no constraints, anything is OK. */
6102 win = !n_alternatives;
6104 alternative_mask preferred = get_preferred_alternatives (insn);
6105 for (j = 0; j < n_alternatives; j++, op_alt += n_operands)
6107 if (!TEST_BIT (preferred, j))
6109 if (op_alt[i].anything_ok
6110 || (op_alt[i].matches != -1
6112 (recog_data.operand[i],
6113 recog_data.operand[op_alt[i].matches]))
6114 || reg_fits_class_p (op, op_alt[i].cl, offset, mode))
6129 /* Implement the TARGET_ASAN_SHADOW_OFFSET hook. */
6131 static unsigned HOST_WIDE_INT
6132 ix86_asan_shadow_offset (void)
6134 return TARGET_LP64 ? (TARGET_MACHO ? (HOST_WIDE_INT_1 << 44)
6135 : HOST_WIDE_INT_C (0x7fff8000))
6136 : (HOST_WIDE_INT_1 << 29);
6139 /* Argument support functions. */
6141 /* Return true when register may be used to pass function parameters. */
6143 ix86_function_arg_regno_p (int regno)
6146 enum calling_abi call_abi;
6147 const int *parm_regs;
6149 if (TARGET_MPX && BND_REGNO_P (regno))
6155 return (regno < REGPARM_MAX
6156 || (TARGET_SSE && SSE_REGNO_P (regno) && !fixed_regs[regno]));
6158 return (regno < REGPARM_MAX
6159 || (TARGET_MMX && MMX_REGNO_P (regno)
6160 && (regno < FIRST_MMX_REG + MMX_REGPARM_MAX))
6161 || (TARGET_SSE && SSE_REGNO_P (regno)
6162 && (regno < FIRST_SSE_REG + SSE_REGPARM_MAX)));
6165 if (TARGET_SSE && SSE_REGNO_P (regno)
6166 && (regno < FIRST_SSE_REG + SSE_REGPARM_MAX))
6169 /* TODO: The function should depend on current function ABI but
6170 builtins.c would need updating then. Therefore we use the
6172 call_abi = ix86_cfun_abi ();
6174 /* RAX is used as hidden argument to va_arg functions. */
6175 if (call_abi == SYSV_ABI && regno == AX_REG)
6178 if (call_abi == MS_ABI)
6179 parm_regs = x86_64_ms_abi_int_parameter_registers;
6181 parm_regs = x86_64_int_parameter_registers;
6183 for (i = 0; i < (call_abi == MS_ABI
6184 ? X86_64_MS_REGPARM_MAX : X86_64_REGPARM_MAX); i++)
6185 if (regno == parm_regs[i])
6190 /* Return if we do not know how to pass TYPE solely in registers. */
6193 ix86_must_pass_in_stack (machine_mode mode, const_tree type)
6195 if (must_pass_in_stack_var_size_or_pad (mode, type))
6198 /* For 32-bit, we want TImode aggregates to go on the stack. But watch out!
6199 The layout_type routine is crafty and tries to trick us into passing
6200 currently unsupported vector types on the stack by using TImode. */
6201 return (!TARGET_64BIT && mode == TImode
6202 && type && TREE_CODE (type) != VECTOR_TYPE);
6205 /* It returns the size, in bytes, of the area reserved for arguments passed
6206 in registers for the function represented by fndecl dependent to the used
6209 ix86_reg_parm_stack_space (const_tree fndecl)
6211 enum calling_abi call_abi = SYSV_ABI;
6212 if (fndecl != NULL_TREE && TREE_CODE (fndecl) == FUNCTION_DECL)
6213 call_abi = ix86_function_abi (fndecl);
6215 call_abi = ix86_function_type_abi (fndecl);
6216 if (TARGET_64BIT && call_abi == MS_ABI)
6221 /* Returns value SYSV_ABI, MS_ABI dependent on fntype, specifying the
6224 ix86_function_type_abi (const_tree fntype)
6226 if (fntype != NULL_TREE && TYPE_ATTRIBUTES (fntype) != NULL_TREE)
6228 enum calling_abi abi = ix86_abi;
6229 if (abi == SYSV_ABI)
6231 if (lookup_attribute ("ms_abi", TYPE_ATTRIBUTES (fntype)))
6235 static bool warned = false;
6238 error ("X32 does not support ms_abi attribute");
6245 else if (lookup_attribute ("sysv_abi", TYPE_ATTRIBUTES (fntype)))
6252 /* We add this as a workaround in order to use libc_has_function
6255 ix86_libc_has_function (enum function_class fn_class)
6257 return targetm.libc_has_function (fn_class);
6261 ix86_function_ms_hook_prologue (const_tree fn)
6263 if (fn && lookup_attribute ("ms_hook_prologue", DECL_ATTRIBUTES (fn)))
6265 if (decl_function_context (fn) != NULL_TREE)
6266 error_at (DECL_SOURCE_LOCATION (fn),
6267 "ms_hook_prologue is not compatible with nested function");
6274 static enum calling_abi
6275 ix86_function_abi (const_tree fndecl)
6279 return ix86_function_type_abi (TREE_TYPE (fndecl));
6282 /* Returns value SYSV_ABI, MS_ABI dependent on cfun, specifying the
6285 ix86_cfun_abi (void)
6289 return cfun->machine->call_abi;
6292 /* Write the extra assembler code needed to declare a function properly. */
6295 ix86_asm_output_function_label (FILE *asm_out_file, const char *fname,
6298 bool is_ms_hook = ix86_function_ms_hook_prologue (decl);
6302 int i, filler_count = (TARGET_64BIT ? 32 : 16);
6303 unsigned int filler_cc = 0xcccccccc;
6305 for (i = 0; i < filler_count; i += 4)
6306 fprintf (asm_out_file, ASM_LONG " %#x\n", filler_cc);
6309 #ifdef SUBTARGET_ASM_UNWIND_INIT
6310 SUBTARGET_ASM_UNWIND_INIT (asm_out_file);
6313 ASM_OUTPUT_LABEL (asm_out_file, fname);
6315 /* Output magic byte marker, if hot-patch attribute is set. */
6320 /* leaq [%rsp + 0], %rsp */
6321 asm_fprintf (asm_out_file, ASM_BYTE
6322 "0x48, 0x8d, 0xa4, 0x24, 0x00, 0x00, 0x00, 0x00\n");
6326 /* movl.s %edi, %edi
6328 movl.s %esp, %ebp */
6329 asm_fprintf (asm_out_file, ASM_BYTE
6330 "0x8b, 0xff, 0x55, 0x8b, 0xec\n");
6336 extern void init_regs (void);
6338 /* Implementation of call abi switching target hook. Specific to FNDECL
6339 the specific call register sets are set. See also
6340 ix86_conditional_register_usage for more details. */
6342 ix86_call_abi_override (const_tree fndecl)
6344 if (fndecl == NULL_TREE)
6345 cfun->machine->call_abi = ix86_abi;
6347 cfun->machine->call_abi = ix86_function_type_abi (TREE_TYPE (fndecl));
6350 /* 64-bit MS and SYSV ABI have different set of call used registers. Avoid
6351 expensive re-initialization of init_regs each time we switch function context
6352 since this is needed only during RTL expansion. */
6354 ix86_maybe_switch_abi (void)
6357 call_used_regs[SI_REG] == (cfun->machine->call_abi == MS_ABI))
6361 /* Return 1 if pseudo register should be created and used to hold
6362 GOT address for PIC code. */
6364 ix86_use_pseudo_pic_reg (void)
6367 && (ix86_cmodel == CM_SMALL_PIC
6374 /* Initialize large model PIC register. */
6377 ix86_init_large_pic_reg (unsigned int tmp_regno)
6379 rtx_code_label *label;
6382 gcc_assert (Pmode == DImode);
6383 label = gen_label_rtx ();
6385 LABEL_PRESERVE_P (label) = 1;
6386 tmp_reg = gen_rtx_REG (Pmode, tmp_regno);
6387 gcc_assert (REGNO (pic_offset_table_rtx) != tmp_regno);
6388 emit_insn (gen_set_rip_rex64 (pic_offset_table_rtx,
6390 emit_insn (gen_set_got_offset_rex64 (tmp_reg, label));
6391 emit_insn (ix86_gen_add3 (pic_offset_table_rtx,
6392 pic_offset_table_rtx, tmp_reg));
6395 /* Create and initialize PIC register if required. */
6397 ix86_init_pic_reg (void)
6402 if (!ix86_use_pseudo_pic_reg ())
6409 if (ix86_cmodel == CM_LARGE_PIC)
6410 ix86_init_large_pic_reg (R11_REG);
6412 emit_insn (gen_set_got_rex64 (pic_offset_table_rtx));
6416 /* If there is future mcount call in the function it is more profitable
6417 to emit SET_GOT into ABI defined REAL_PIC_OFFSET_TABLE_REGNUM. */
6418 rtx reg = crtl->profile
6419 ? gen_rtx_REG (Pmode, REAL_PIC_OFFSET_TABLE_REGNUM)
6420 : pic_offset_table_rtx;
6421 rtx_insn *insn = emit_insn (gen_set_got (reg));
6422 RTX_FRAME_RELATED_P (insn) = 1;
6424 emit_move_insn (pic_offset_table_rtx, reg);
6425 add_reg_note (insn, REG_CFA_FLUSH_QUEUE, NULL_RTX);
6431 entry_edge = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun));
6432 insert_insn_on_edge (seq, entry_edge);
6433 commit_one_edge_insertion (entry_edge);
6436 /* Initialize a variable CUM of type CUMULATIVE_ARGS
6437 for a call to a function whose data type is FNTYPE.
6438 For a library call, FNTYPE is 0. */
6441 init_cumulative_args (CUMULATIVE_ARGS *cum, /* Argument info to initialize */
6442 tree fntype, /* tree ptr for function decl */
6443 rtx libname, /* SYMBOL_REF of library name or 0 */
6447 struct cgraph_local_info *i = NULL;
6448 struct cgraph_node *target = NULL;
6450 memset (cum, 0, sizeof (*cum));
6454 target = cgraph_node::get (fndecl);
6457 target = target->function_symbol ();
6458 i = cgraph_node::local_info (target->decl);
6459 cum->call_abi = ix86_function_abi (target->decl);
6462 cum->call_abi = ix86_function_abi (fndecl);
6465 cum->call_abi = ix86_function_type_abi (fntype);
6467 cum->caller = caller;
6469 /* Set up the number of registers to use for passing arguments. */
6470 cum->nregs = ix86_regparm;
6473 cum->nregs = (cum->call_abi == SYSV_ABI
6474 ? X86_64_REGPARM_MAX
6475 : X86_64_MS_REGPARM_MAX);
6479 cum->sse_nregs = SSE_REGPARM_MAX;
6482 cum->sse_nregs = (cum->call_abi == SYSV_ABI
6483 ? X86_64_SSE_REGPARM_MAX
6484 : X86_64_MS_SSE_REGPARM_MAX);
6488 cum->mmx_nregs = MMX_REGPARM_MAX;
6489 cum->warn_avx512f = true;
6490 cum->warn_avx = true;
6491 cum->warn_sse = true;
6492 cum->warn_mmx = true;
6494 /* Because type might mismatch in between caller and callee, we need to
6495 use actual type of function for local calls.
6496 FIXME: cgraph_analyze can be told to actually record if function uses
6497 va_start so for local functions maybe_vaarg can be made aggressive
6499 FIXME: once typesytem is fixed, we won't need this code anymore. */
6500 if (i && i->local && i->can_change_signature)
6501 fntype = TREE_TYPE (target->decl);
6502 cum->stdarg = stdarg_p (fntype);
6503 cum->maybe_vaarg = (fntype
6504 ? (!prototype_p (fntype) || stdarg_p (fntype))
6507 cum->bnd_regno = FIRST_BND_REG;
6508 cum->bnds_in_bt = 0;
6509 cum->force_bnd_pass = 0;
6514 /* If there are variable arguments, then we won't pass anything
6515 in registers in 32-bit mode. */
6516 if (stdarg_p (fntype))
6521 cum->warn_avx512f = false;
6522 cum->warn_avx = false;
6523 cum->warn_sse = false;
6524 cum->warn_mmx = false;
6528 /* Use ecx and edx registers if function has fastcall attribute,
6529 else look for regparm information. */
6532 unsigned int ccvt = ix86_get_callcvt (fntype);
6533 if ((ccvt & IX86_CALLCVT_THISCALL) != 0)
6536 cum->fastcall = 1; /* Same first register as in fastcall. */
6538 else if ((ccvt & IX86_CALLCVT_FASTCALL) != 0)
6544 cum->nregs = ix86_function_regparm (fntype, fndecl);
6547 /* Set up the number of SSE registers used for passing SFmode
6548 and DFmode arguments. Warn for mismatching ABI. */
6549 cum->float_in_sse = ix86_function_sseregparm (fntype, fndecl, true);
6553 /* Return the "natural" mode for TYPE. In most cases, this is just TYPE_MODE.
6554 But in the case of vector types, it is some vector mode.
6556 When we have only some of our vector isa extensions enabled, then there
6557 are some modes for which vector_mode_supported_p is false. For these
6558 modes, the generic vector support in gcc will choose some non-vector mode
6559 in order to implement the type. By computing the natural mode, we'll
6560 select the proper ABI location for the operand and not depend on whatever
6561 the middle-end decides to do with these vector types.
6563 The midde-end can't deal with the vector types > 16 bytes. In this
6564 case, we return the original mode and warn ABI change if CUM isn't
6567 If INT_RETURN is true, warn ABI change if the vector mode isn't
6568 available for function return value. */
6571 type_natural_mode (const_tree type, const CUMULATIVE_ARGS *cum,
6574 machine_mode mode = TYPE_MODE (type);
6576 if (TREE_CODE (type) == VECTOR_TYPE && !VECTOR_MODE_P (mode))
6578 HOST_WIDE_INT size = int_size_in_bytes (type);
6579 if ((size == 8 || size == 16 || size == 32 || size == 64)
6580 /* ??? Generic code allows us to create width 1 vectors. Ignore. */
6581 && TYPE_VECTOR_SUBPARTS (type) > 1)
6583 machine_mode innermode = TYPE_MODE (TREE_TYPE (type));
6585 if (TREE_CODE (TREE_TYPE (type)) == REAL_TYPE)
6586 mode = MIN_MODE_VECTOR_FLOAT;
6588 mode = MIN_MODE_VECTOR_INT;
6590 /* Get the mode which has this inner mode and number of units. */
6591 for (; mode != VOIDmode; mode = GET_MODE_WIDER_MODE (mode))
6592 if (GET_MODE_NUNITS (mode) == TYPE_VECTOR_SUBPARTS (type)
6593 && GET_MODE_INNER (mode) == innermode)
6595 if (size == 64 && !TARGET_AVX512F)
6597 static bool warnedavx512f;
6598 static bool warnedavx512f_ret;
6600 if (cum && cum->warn_avx512f && !warnedavx512f)
6602 if (warning (OPT_Wpsabi, "AVX512F vector argument "
6603 "without AVX512F enabled changes the ABI"))
6604 warnedavx512f = true;
6606 else if (in_return && !warnedavx512f_ret)
6608 if (warning (OPT_Wpsabi, "AVX512F vector return "
6609 "without AVX512F enabled changes the ABI"))
6610 warnedavx512f_ret = true;
6613 return TYPE_MODE (type);
6615 else if (size == 32 && !TARGET_AVX)
6617 static bool warnedavx;
6618 static bool warnedavx_ret;
6620 if (cum && cum->warn_avx && !warnedavx)
6622 if (warning (OPT_Wpsabi, "AVX vector argument "
6623 "without AVX enabled changes the ABI"))
6626 else if (in_return && !warnedavx_ret)
6628 if (warning (OPT_Wpsabi, "AVX vector return "
6629 "without AVX enabled changes the ABI"))
6630 warnedavx_ret = true;
6633 return TYPE_MODE (type);
6635 else if (((size == 8 && TARGET_64BIT) || size == 16)
6638 static bool warnedsse;
6639 static bool warnedsse_ret;
6641 if (cum && cum->warn_sse && !warnedsse)
6643 if (warning (OPT_Wpsabi, "SSE vector argument "
6644 "without SSE enabled changes the ABI"))
6647 else if (!TARGET_64BIT && in_return && !warnedsse_ret)
6649 if (warning (OPT_Wpsabi, "SSE vector return "
6650 "without SSE enabled changes the ABI"))
6651 warnedsse_ret = true;
6654 else if ((size == 8 && !TARGET_64BIT) && !TARGET_MMX)
6656 static bool warnedmmx;
6657 static bool warnedmmx_ret;
6659 if (cum && cum->warn_mmx && !warnedmmx)
6661 if (warning (OPT_Wpsabi, "MMX vector argument "
6662 "without MMX enabled changes the ABI"))
6665 else if (in_return && !warnedmmx_ret)
6667 if (warning (OPT_Wpsabi, "MMX vector return "
6668 "without MMX enabled changes the ABI"))
6669 warnedmmx_ret = true;
6682 /* We want to pass a value in REGNO whose "natural" mode is MODE. However,
6683 this may not agree with the mode that the type system has chosen for the
6684 register, which is ORIG_MODE. If ORIG_MODE is not BLKmode, then we can
6685 go ahead and use it. Otherwise we have to build a PARALLEL instead. */
6688 gen_reg_or_parallel (machine_mode mode, machine_mode orig_mode,
6693 if (orig_mode != BLKmode)
6694 tmp = gen_rtx_REG (orig_mode, regno);
6697 tmp = gen_rtx_REG (mode, regno);
6698 tmp = gen_rtx_EXPR_LIST (VOIDmode, tmp, const0_rtx);
6699 tmp = gen_rtx_PARALLEL (orig_mode, gen_rtvec (1, tmp));
6705 /* x86-64 register passing implementation. See x86-64 ABI for details. Goal
6706 of this code is to classify each 8bytes of incoming argument by the register
6707 class and assign registers accordingly. */
6709 /* Return the union class of CLASS1 and CLASS2.
6710 See the x86-64 PS ABI for details. */
6712 static enum x86_64_reg_class
6713 merge_classes (enum x86_64_reg_class class1, enum x86_64_reg_class class2)
6715 /* Rule #1: If both classes are equal, this is the resulting class. */
6716 if (class1 == class2)
6719 /* Rule #2: If one of the classes is NO_CLASS, the resulting class is
6721 if (class1 == X86_64_NO_CLASS)
6723 if (class2 == X86_64_NO_CLASS)
6726 /* Rule #3: If one of the classes is MEMORY, the result is MEMORY. */
6727 if (class1 == X86_64_MEMORY_CLASS || class2 == X86_64_MEMORY_CLASS)
6728 return X86_64_MEMORY_CLASS;
6730 /* Rule #4: If one of the classes is INTEGER, the result is INTEGER. */
6731 if ((class1 == X86_64_INTEGERSI_CLASS && class2 == X86_64_SSESF_CLASS)
6732 || (class2 == X86_64_INTEGERSI_CLASS && class1 == X86_64_SSESF_CLASS))
6733 return X86_64_INTEGERSI_CLASS;
6734 if (class1 == X86_64_INTEGER_CLASS || class1 == X86_64_INTEGERSI_CLASS
6735 || class2 == X86_64_INTEGER_CLASS || class2 == X86_64_INTEGERSI_CLASS)
6736 return X86_64_INTEGER_CLASS;
6738 /* Rule #5: If one of the classes is X87, X87UP, or COMPLEX_X87 class,
6740 if (class1 == X86_64_X87_CLASS
6741 || class1 == X86_64_X87UP_CLASS
6742 || class1 == X86_64_COMPLEX_X87_CLASS
6743 || class2 == X86_64_X87_CLASS
6744 || class2 == X86_64_X87UP_CLASS
6745 || class2 == X86_64_COMPLEX_X87_CLASS)
6746 return X86_64_MEMORY_CLASS;
6748 /* Rule #6: Otherwise class SSE is used. */
6749 return X86_64_SSE_CLASS;
6752 /* Classify the argument of type TYPE and mode MODE.
6753 CLASSES will be filled by the register class used to pass each word
6754 of the operand. The number of words is returned. In case the parameter
6755 should be passed in memory, 0 is returned. As a special case for zero
6756 sized containers, classes[0] will be NO_CLASS and 1 is returned.
6758 BIT_OFFSET is used internally for handling records and specifies offset
6759 of the offset in bits modulo 512 to avoid overflow cases.
6761 See the x86-64 PS ABI for details.
6765 classify_argument (machine_mode mode, const_tree type,
6766 enum x86_64_reg_class classes[MAX_CLASSES], int bit_offset)
6768 HOST_WIDE_INT bytes =
6769 (mode == BLKmode) ? int_size_in_bytes (type) : (int) GET_MODE_SIZE (mode);
6771 = (bytes + (bit_offset % 64) / 8 + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
6773 /* Variable sized entities are always passed/returned in memory. */
6777 if (mode != VOIDmode
6778 && targetm.calls.must_pass_in_stack (mode, type))
6781 if (type && AGGREGATE_TYPE_P (type))
6785 enum x86_64_reg_class subclasses[MAX_CLASSES];
6787 /* On x86-64 we pass structures larger than 64 bytes on the stack. */
6791 for (i = 0; i < words; i++)
6792 classes[i] = X86_64_NO_CLASS;
6794 /* Zero sized arrays or structures are NO_CLASS. We return 0 to
6795 signalize memory class, so handle it as special case. */
6798 classes[0] = X86_64_NO_CLASS;
6802 /* Classify each field of record and merge classes. */
6803 switch (TREE_CODE (type))
6806 /* And now merge the fields of structure. */
6807 for (field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field))
6809 if (TREE_CODE (field) == FIELD_DECL)
6813 if (TREE_TYPE (field) == error_mark_node)
6816 /* Bitfields are always classified as integer. Handle them
6817 early, since later code would consider them to be
6818 misaligned integers. */
6819 if (DECL_BIT_FIELD (field))
6821 for (i = (int_bit_position (field)
6822 + (bit_offset % 64)) / 8 / 8;
6823 i < ((int_bit_position (field) + (bit_offset % 64))
6824 + tree_to_shwi (DECL_SIZE (field))
6827 merge_classes (X86_64_INTEGER_CLASS,
6834 type = TREE_TYPE (field);
6836 /* Flexible array member is ignored. */
6837 if (TYPE_MODE (type) == BLKmode
6838 && TREE_CODE (type) == ARRAY_TYPE
6839 && TYPE_SIZE (type) == NULL_TREE
6840 && TYPE_DOMAIN (type) != NULL_TREE
6841 && (TYPE_MAX_VALUE (TYPE_DOMAIN (type))
6846 if (!warned && warn_psabi)
6849 inform (input_location,
6850 "the ABI of passing struct with"
6851 " a flexible array member has"
6852 " changed in GCC 4.4");
6856 num = classify_argument (TYPE_MODE (type), type,
6858 (int_bit_position (field)
6859 + bit_offset) % 512);
6862 pos = (int_bit_position (field)
6863 + (bit_offset % 64)) / 8 / 8;
6864 for (i = 0; i < num && (i + pos) < words; i++)
6866 merge_classes (subclasses[i], classes[i + pos]);
6873 /* Arrays are handled as small records. */
6876 num = classify_argument (TYPE_MODE (TREE_TYPE (type)),
6877 TREE_TYPE (type), subclasses, bit_offset);
6881 /* The partial classes are now full classes. */
6882 if (subclasses[0] == X86_64_SSESF_CLASS && bytes != 4)
6883 subclasses[0] = X86_64_SSE_CLASS;
6884 if (subclasses[0] == X86_64_INTEGERSI_CLASS
6885 && !((bit_offset % 64) == 0 && bytes == 4))
6886 subclasses[0] = X86_64_INTEGER_CLASS;
6888 for (i = 0; i < words; i++)
6889 classes[i] = subclasses[i % num];
6894 case QUAL_UNION_TYPE:
6895 /* Unions are similar to RECORD_TYPE but offset is always 0.
6897 for (field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field))
6899 if (TREE_CODE (field) == FIELD_DECL)
6903 if (TREE_TYPE (field) == error_mark_node)
6906 num = classify_argument (TYPE_MODE (TREE_TYPE (field)),
6907 TREE_TYPE (field), subclasses,
6911 for (i = 0; i < num && i < words; i++)
6912 classes[i] = merge_classes (subclasses[i], classes[i]);
6923 /* When size > 16 bytes, if the first one isn't
6924 X86_64_SSE_CLASS or any other ones aren't
6925 X86_64_SSEUP_CLASS, everything should be passed in
6927 if (classes[0] != X86_64_SSE_CLASS)
6930 for (i = 1; i < words; i++)
6931 if (classes[i] != X86_64_SSEUP_CLASS)
6935 /* Final merger cleanup. */
6936 for (i = 0; i < words; i++)
6938 /* If one class is MEMORY, everything should be passed in
6940 if (classes[i] == X86_64_MEMORY_CLASS)
6943 /* The X86_64_SSEUP_CLASS should be always preceded by
6944 X86_64_SSE_CLASS or X86_64_SSEUP_CLASS. */
6945 if (classes[i] == X86_64_SSEUP_CLASS
6946 && classes[i - 1] != X86_64_SSE_CLASS
6947 && classes[i - 1] != X86_64_SSEUP_CLASS)
6949 /* The first one should never be X86_64_SSEUP_CLASS. */
6950 gcc_assert (i != 0);
6951 classes[i] = X86_64_SSE_CLASS;
6954 /* If X86_64_X87UP_CLASS isn't preceded by X86_64_X87_CLASS,
6955 everything should be passed in memory. */
6956 if (classes[i] == X86_64_X87UP_CLASS
6957 && (classes[i - 1] != X86_64_X87_CLASS))
6961 /* The first one should never be X86_64_X87UP_CLASS. */
6962 gcc_assert (i != 0);
6963 if (!warned && warn_psabi)
6966 inform (input_location,
6967 "the ABI of passing union with long double"
6968 " has changed in GCC 4.4");
6976 /* Compute alignment needed. We align all types to natural boundaries with
6977 exception of XFmode that is aligned to 64bits. */
6978 if (mode != VOIDmode && mode != BLKmode)
6980 int mode_alignment = GET_MODE_BITSIZE (mode);
6983 mode_alignment = 128;
6984 else if (mode == XCmode)
6985 mode_alignment = 256;
6986 if (COMPLEX_MODE_P (mode))
6987 mode_alignment /= 2;
6988 /* Misaligned fields are always returned in memory. */
6989 if (bit_offset % mode_alignment)
6993 /* for V1xx modes, just use the base mode */
6994 if (VECTOR_MODE_P (mode) && mode != V1DImode && mode != V1TImode
6995 && GET_MODE_SIZE (GET_MODE_INNER (mode)) == bytes)
6996 mode = GET_MODE_INNER (mode);
6998 /* Classification of atomic types. */
7003 classes[0] = X86_64_SSE_CLASS;
7006 classes[0] = X86_64_SSE_CLASS;
7007 classes[1] = X86_64_SSEUP_CLASS;
7017 int size = bit_offset + (int) GET_MODE_BITSIZE (mode);
7019 /* Analyze last 128 bits only. */
7020 size = (size - 1) & 0x7f;
7024 classes[0] = X86_64_INTEGERSI_CLASS;
7029 classes[0] = X86_64_INTEGER_CLASS;
7032 else if (size < 64+32)
7034 classes[0] = X86_64_INTEGER_CLASS;
7035 classes[1] = X86_64_INTEGERSI_CLASS;
7038 else if (size < 64+64)
7040 classes[0] = classes[1] = X86_64_INTEGER_CLASS;
7048 classes[0] = classes[1] = X86_64_INTEGER_CLASS;
7052 /* OImode shouldn't be used directly. */
7057 if (!(bit_offset % 64))
7058 classes[0] = X86_64_SSESF_CLASS;
7060 classes[0] = X86_64_SSE_CLASS;
7063 classes[0] = X86_64_SSEDF_CLASS;
7066 classes[0] = X86_64_X87_CLASS;
7067 classes[1] = X86_64_X87UP_CLASS;
7070 classes[0] = X86_64_SSE_CLASS;
7071 classes[1] = X86_64_SSEUP_CLASS;
7074 classes[0] = X86_64_SSE_CLASS;
7075 if (!(bit_offset % 64))
7081 if (!warned && warn_psabi)
7084 inform (input_location,
7085 "the ABI of passing structure with complex float"
7086 " member has changed in GCC 4.4");
7088 classes[1] = X86_64_SSESF_CLASS;
7092 classes[0] = X86_64_SSEDF_CLASS;
7093 classes[1] = X86_64_SSEDF_CLASS;
7096 classes[0] = X86_64_COMPLEX_X87_CLASS;
7099 /* This modes is larger than 16 bytes. */
7107 classes[0] = X86_64_SSE_CLASS;
7108 classes[1] = X86_64_SSEUP_CLASS;
7109 classes[2] = X86_64_SSEUP_CLASS;
7110 classes[3] = X86_64_SSEUP_CLASS;
7118 classes[0] = X86_64_SSE_CLASS;
7119 classes[1] = X86_64_SSEUP_CLASS;
7120 classes[2] = X86_64_SSEUP_CLASS;
7121 classes[3] = X86_64_SSEUP_CLASS;
7122 classes[4] = X86_64_SSEUP_CLASS;
7123 classes[5] = X86_64_SSEUP_CLASS;
7124 classes[6] = X86_64_SSEUP_CLASS;
7125 classes[7] = X86_64_SSEUP_CLASS;
7133 classes[0] = X86_64_SSE_CLASS;
7134 classes[1] = X86_64_SSEUP_CLASS;
7142 classes[0] = X86_64_SSE_CLASS;
7148 gcc_assert (VECTOR_MODE_P (mode));
7153 gcc_assert (GET_MODE_CLASS (GET_MODE_INNER (mode)) == MODE_INT);
7155 if (bit_offset + GET_MODE_BITSIZE (mode) <= 32)
7156 classes[0] = X86_64_INTEGERSI_CLASS;
7158 classes[0] = X86_64_INTEGER_CLASS;
7159 classes[1] = X86_64_INTEGER_CLASS;
7160 return 1 + (bytes > 8);
7164 /* Examine the argument and return set number of register required in each
7165 class. Return true iff parameter should be passed in memory. */
7168 examine_argument (machine_mode mode, const_tree type, int in_return,
7169 int *int_nregs, int *sse_nregs)
7171 enum x86_64_reg_class regclass[MAX_CLASSES];
7172 int n = classify_argument (mode, type, regclass, 0);
7179 for (n--; n >= 0; n--)
7180 switch (regclass[n])
7182 case X86_64_INTEGER_CLASS:
7183 case X86_64_INTEGERSI_CLASS:
7186 case X86_64_SSE_CLASS:
7187 case X86_64_SSESF_CLASS:
7188 case X86_64_SSEDF_CLASS:
7191 case X86_64_NO_CLASS:
7192 case X86_64_SSEUP_CLASS:
7194 case X86_64_X87_CLASS:
7195 case X86_64_X87UP_CLASS:
7196 case X86_64_COMPLEX_X87_CLASS:
7200 case X86_64_MEMORY_CLASS:
7207 /* Construct container for the argument used by GCC interface. See
7208 FUNCTION_ARG for the detailed description. */
7211 construct_container (machine_mode mode, machine_mode orig_mode,
7212 const_tree type, int in_return, int nintregs, int nsseregs,
7213 const int *intreg, int sse_regno)
7215 /* The following variables hold the static issued_error state. */
7216 static bool issued_sse_arg_error;
7217 static bool issued_sse_ret_error;
7218 static bool issued_x87_ret_error;
7220 machine_mode tmpmode;
7222 (mode == BLKmode) ? int_size_in_bytes (type) : (int) GET_MODE_SIZE (mode);
7223 enum x86_64_reg_class regclass[MAX_CLASSES];
7227 int needed_sseregs, needed_intregs;
7228 rtx exp[MAX_CLASSES];
7231 n = classify_argument (mode, type, regclass, 0);
7234 if (examine_argument (mode, type, in_return, &needed_intregs,
7237 if (needed_intregs > nintregs || needed_sseregs > nsseregs)
7240 /* We allowed the user to turn off SSE for kernel mode. Don't crash if
7241 some less clueful developer tries to use floating-point anyway. */
7242 if (needed_sseregs && !TARGET_SSE)
7246 if (!issued_sse_ret_error)
7248 error ("SSE register return with SSE disabled");
7249 issued_sse_ret_error = true;
7252 else if (!issued_sse_arg_error)
7254 error ("SSE register argument with SSE disabled");
7255 issued_sse_arg_error = true;
7260 /* Likewise, error if the ABI requires us to return values in the
7261 x87 registers and the user specified -mno-80387. */
7262 if (!TARGET_FLOAT_RETURNS_IN_80387 && in_return)
7263 for (i = 0; i < n; i++)
7264 if (regclass[i] == X86_64_X87_CLASS
7265 || regclass[i] == X86_64_X87UP_CLASS
7266 || regclass[i] == X86_64_COMPLEX_X87_CLASS)
7268 if (!issued_x87_ret_error)
7270 error ("x87 register return with x87 disabled");
7271 issued_x87_ret_error = true;
7276 /* First construct simple cases. Avoid SCmode, since we want to use
7277 single register to pass this type. */
7278 if (n == 1 && mode != SCmode)
7279 switch (regclass[0])
7281 case X86_64_INTEGER_CLASS:
7282 case X86_64_INTEGERSI_CLASS:
7283 return gen_rtx_REG (mode, intreg[0]);
7284 case X86_64_SSE_CLASS:
7285 case X86_64_SSESF_CLASS:
7286 case X86_64_SSEDF_CLASS:
7287 if (mode != BLKmode)
7288 return gen_reg_or_parallel (mode, orig_mode,
7289 SSE_REGNO (sse_regno));
7291 case X86_64_X87_CLASS:
7292 case X86_64_COMPLEX_X87_CLASS:
7293 return gen_rtx_REG (mode, FIRST_STACK_REG);
7294 case X86_64_NO_CLASS:
7295 /* Zero sized array, struct or class. */
7301 && regclass[0] == X86_64_SSE_CLASS
7302 && regclass[1] == X86_64_SSEUP_CLASS
7304 return gen_reg_or_parallel (mode, orig_mode,
7305 SSE_REGNO (sse_regno));
7307 && regclass[0] == X86_64_SSE_CLASS
7308 && regclass[1] == X86_64_SSEUP_CLASS
7309 && regclass[2] == X86_64_SSEUP_CLASS
7310 && regclass[3] == X86_64_SSEUP_CLASS
7312 return gen_reg_or_parallel (mode, orig_mode,
7313 SSE_REGNO (sse_regno));
7315 && regclass[0] == X86_64_SSE_CLASS
7316 && regclass[1] == X86_64_SSEUP_CLASS
7317 && regclass[2] == X86_64_SSEUP_CLASS
7318 && regclass[3] == X86_64_SSEUP_CLASS
7319 && regclass[4] == X86_64_SSEUP_CLASS
7320 && regclass[5] == X86_64_SSEUP_CLASS
7321 && regclass[6] == X86_64_SSEUP_CLASS
7322 && regclass[7] == X86_64_SSEUP_CLASS
7324 return gen_reg_or_parallel (mode, orig_mode,
7325 SSE_REGNO (sse_regno));
7327 && regclass[0] == X86_64_X87_CLASS
7328 && regclass[1] == X86_64_X87UP_CLASS)
7329 return gen_rtx_REG (XFmode, FIRST_STACK_REG);
7332 && regclass[0] == X86_64_INTEGER_CLASS
7333 && regclass[1] == X86_64_INTEGER_CLASS
7334 && (mode == CDImode || mode == TImode)
7335 && intreg[0] + 1 == intreg[1])
7336 return gen_rtx_REG (mode, intreg[0]);
7338 /* Otherwise figure out the entries of the PARALLEL. */
7339 for (i = 0; i < n; i++)
7343 switch (regclass[i])
7345 case X86_64_NO_CLASS:
7347 case X86_64_INTEGER_CLASS:
7348 case X86_64_INTEGERSI_CLASS:
7349 /* Merge TImodes on aligned occasions here too. */
7350 if (i * 8 + 8 > bytes)
7352 = mode_for_size ((bytes - i * 8) * BITS_PER_UNIT, MODE_INT, 0);
7353 else if (regclass[i] == X86_64_INTEGERSI_CLASS)
7357 /* We've requested 24 bytes we
7358 don't have mode for. Use DImode. */
7359 if (tmpmode == BLKmode)
7362 = gen_rtx_EXPR_LIST (VOIDmode,
7363 gen_rtx_REG (tmpmode, *intreg),
7367 case X86_64_SSESF_CLASS:
7369 = gen_rtx_EXPR_LIST (VOIDmode,
7370 gen_rtx_REG (SFmode,
7371 SSE_REGNO (sse_regno)),
7375 case X86_64_SSEDF_CLASS:
7377 = gen_rtx_EXPR_LIST (VOIDmode,
7378 gen_rtx_REG (DFmode,
7379 SSE_REGNO (sse_regno)),
7383 case X86_64_SSE_CLASS:
7391 if (i == 0 && regclass[1] == X86_64_SSEUP_CLASS)
7401 && regclass[1] == X86_64_SSEUP_CLASS
7402 && regclass[2] == X86_64_SSEUP_CLASS
7403 && regclass[3] == X86_64_SSEUP_CLASS);
7409 && regclass[1] == X86_64_SSEUP_CLASS
7410 && regclass[2] == X86_64_SSEUP_CLASS
7411 && regclass[3] == X86_64_SSEUP_CLASS
7412 && regclass[4] == X86_64_SSEUP_CLASS
7413 && regclass[5] == X86_64_SSEUP_CLASS
7414 && regclass[6] == X86_64_SSEUP_CLASS
7415 && regclass[7] == X86_64_SSEUP_CLASS);
7423 = gen_rtx_EXPR_LIST (VOIDmode,
7424 gen_rtx_REG (tmpmode,
7425 SSE_REGNO (sse_regno)),
7434 /* Empty aligned struct, union or class. */
7438 ret = gen_rtx_PARALLEL (mode, rtvec_alloc (nexps));
7439 for (i = 0; i < nexps; i++)
7440 XVECEXP (ret, 0, i) = exp [i];
7444 /* Update the data in CUM to advance over an argument of mode MODE
7445 and data type TYPE. (TYPE is null for libcalls where that information
7446 may not be available.)
7448 Return a number of integer regsiters advanced over. */
7451 function_arg_advance_32 (CUMULATIVE_ARGS *cum, machine_mode mode,
7452 const_tree type, HOST_WIDE_INT bytes,
7453 HOST_WIDE_INT words)
7456 bool error_p = NULL;
7472 cum->words += words;
7473 cum->nregs -= words;
7474 cum->regno += words;
7475 if (cum->nregs >= 0)
7477 if (cum->nregs <= 0)
7485 /* OImode shouldn't be used directly. */
7489 if (cum->float_in_sse == -1)
7491 if (cum->float_in_sse < 2)
7494 if (cum->float_in_sse == -1)
7496 if (cum->float_in_sse < 1)
7519 if (!type || !AGGREGATE_TYPE_P (type))
7521 cum->sse_words += words;
7522 cum->sse_nregs -= 1;
7523 cum->sse_regno += 1;
7524 if (cum->sse_nregs <= 0)
7538 if (!type || !AGGREGATE_TYPE_P (type))
7540 cum->mmx_words += words;
7541 cum->mmx_nregs -= 1;
7542 cum->mmx_regno += 1;
7543 if (cum->mmx_nregs <= 0)
7553 cum->float_in_sse = 0;
7554 error ("calling %qD with SSE calling convention without "
7555 "SSE/SSE2 enabled", cum->decl);
7556 sorry ("this is a GCC bug that can be worked around by adding "
7557 "attribute used to function called");
7564 function_arg_advance_64 (CUMULATIVE_ARGS *cum, machine_mode mode,
7565 const_tree type, HOST_WIDE_INT words, bool named)
7567 int int_nregs, sse_nregs;
7569 /* Unnamed 512 and 256bit vector mode parameters are passed on stack. */
7570 if (!named && (VALID_AVX512F_REG_MODE (mode)
7571 || VALID_AVX256_REG_MODE (mode)))
7574 if (!examine_argument (mode, type, 0, &int_nregs, &sse_nregs)
7575 && sse_nregs <= cum->sse_nregs && int_nregs <= cum->nregs)
7577 cum->nregs -= int_nregs;
7578 cum->sse_nregs -= sse_nregs;
7579 cum->regno += int_nregs;
7580 cum->sse_regno += sse_nregs;
7585 int align = ix86_function_arg_boundary (mode, type) / BITS_PER_WORD;
7586 cum->words = (cum->words + align - 1) & ~(align - 1);
7587 cum->words += words;
7593 function_arg_advance_ms_64 (CUMULATIVE_ARGS *cum, HOST_WIDE_INT bytes,
7594 HOST_WIDE_INT words)
7596 /* Otherwise, this should be passed indirect. */
7597 gcc_assert (bytes == 1 || bytes == 2 || bytes == 4 || bytes == 8);
7599 cum->words += words;
7609 /* Update the data in CUM to advance over an argument of mode MODE and
7610 data type TYPE. (TYPE is null for libcalls where that information
7611 may not be available.) */
7614 ix86_function_arg_advance (cumulative_args_t cum_v, machine_mode mode,
7615 const_tree type, bool named)
7617 CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
7618 HOST_WIDE_INT bytes, words;
7621 if (mode == BLKmode)
7622 bytes = int_size_in_bytes (type);
7624 bytes = GET_MODE_SIZE (mode);
7625 words = (bytes + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
7628 mode = type_natural_mode (type, NULL, false);
7630 if ((type && POINTER_BOUNDS_TYPE_P (type))
7631 || POINTER_BOUNDS_MODE_P (mode))
7633 /* If we pass bounds in BT then just update remained bounds count. */
7634 if (cum->bnds_in_bt)
7640 /* Update remained number of bounds to force. */
7641 if (cum->force_bnd_pass)
7642 cum->force_bnd_pass--;
7649 /* The first arg not going to Bounds Tables resets this counter. */
7650 cum->bnds_in_bt = 0;
7651 /* For unnamed args we always pass bounds to avoid bounds mess when
7652 passed and received types do not match. If bounds do not follow
7653 unnamed arg, still pretend required number of bounds were passed. */
7654 if (cum->force_bnd_pass)
7656 cum->bnd_regno += cum->force_bnd_pass;
7657 cum->force_bnd_pass = 0;
7660 if (TARGET_64BIT && (cum ? cum->call_abi : ix86_abi) == MS_ABI)
7661 nregs = function_arg_advance_ms_64 (cum, bytes, words);
7662 else if (TARGET_64BIT)
7663 nregs = function_arg_advance_64 (cum, mode, type, words, named);
7665 nregs = function_arg_advance_32 (cum, mode, type, bytes, words);
7667 /* For stdarg we expect bounds to be passed for each value passed
7670 cum->force_bnd_pass = nregs;
7671 /* For pointers passed in memory we expect bounds passed in Bounds
7674 cum->bnds_in_bt = chkp_type_bounds_count (type);
7677 /* Define where to put the arguments to a function.
7678 Value is zero to push the argument on the stack,
7679 or a hard register in which to store the argument.
7681 MODE is the argument's machine mode.
7682 TYPE is the data type of the argument (as a tree).
7683 This is null for libcalls where that information may
7685 CUM is a variable of type CUMULATIVE_ARGS which gives info about
7686 the preceding args and about the function being called.
7687 NAMED is nonzero if this argument is a named parameter
7688 (otherwise it is an extra parameter matching an ellipsis). */
7691 function_arg_32 (CUMULATIVE_ARGS *cum, machine_mode mode,
7692 machine_mode orig_mode, const_tree type,
7693 HOST_WIDE_INT bytes, HOST_WIDE_INT words)
7695 bool error_p = false;
7696 /* Avoid the AL settings for the Unix64 ABI. */
7697 if (mode == VOIDmode)
7713 if (words <= cum->nregs)
7715 int regno = cum->regno;
7717 /* Fastcall allocates the first two DWORD (SImode) or
7718 smaller arguments to ECX and EDX if it isn't an
7724 || (type && AGGREGATE_TYPE_P (type)))
7727 /* ECX not EAX is the first allocated register. */
7728 if (regno == AX_REG)
7731 return gen_rtx_REG (mode, regno);
7736 if (cum->float_in_sse == -1)
7738 if (cum->float_in_sse < 2)
7741 if (cum->float_in_sse == -1)
7743 if (cum->float_in_sse < 1)
7747 /* In 32bit, we pass TImode in xmm registers. */
7754 if (!type || !AGGREGATE_TYPE_P (type))
7757 return gen_reg_or_parallel (mode, orig_mode,
7758 cum->sse_regno + FIRST_SSE_REG);
7764 /* OImode and XImode shouldn't be used directly. */
7779 if (!type || !AGGREGATE_TYPE_P (type))
7782 return gen_reg_or_parallel (mode, orig_mode,
7783 cum->sse_regno + FIRST_SSE_REG);
7793 if (!type || !AGGREGATE_TYPE_P (type))
7796 return gen_reg_or_parallel (mode, orig_mode,
7797 cum->mmx_regno + FIRST_MMX_REG);
7803 cum->float_in_sse = 0;
7804 error ("calling %qD with SSE calling convention without "
7805 "SSE/SSE2 enabled", cum->decl);
7806 sorry ("this is a GCC bug that can be worked around by adding "
7807 "attribute used to function called");
7814 function_arg_64 (const CUMULATIVE_ARGS *cum, machine_mode mode,
7815 machine_mode orig_mode, const_tree type, bool named)
7817 /* Handle a hidden AL argument containing number of registers
7818 for varargs x86-64 functions. */
7819 if (mode == VOIDmode)
7820 return GEN_INT (cum->maybe_vaarg
7821 ? (cum->sse_nregs < 0
7822 ? X86_64_SSE_REGPARM_MAX
7843 /* Unnamed 256 and 512bit vector mode parameters are passed on stack. */
7849 return construct_container (mode, orig_mode, type, 0, cum->nregs,
7851 &x86_64_int_parameter_registers [cum->regno],
7856 function_arg_ms_64 (const CUMULATIVE_ARGS *cum, machine_mode mode,
7857 machine_mode orig_mode, bool named,
7858 HOST_WIDE_INT bytes)
7862 /* We need to add clobber for MS_ABI->SYSV ABI calls in expand_call.
7863 We use value of -2 to specify that current function call is MSABI. */
7864 if (mode == VOIDmode)
7865 return GEN_INT (-2);
7867 /* If we've run out of registers, it goes on the stack. */
7868 if (cum->nregs == 0)
7871 regno = x86_64_ms_abi_int_parameter_registers[cum->regno];
7873 /* Only floating point modes are passed in anything but integer regs. */
7874 if (TARGET_SSE && (mode == SFmode || mode == DFmode))
7877 regno = cum->regno + FIRST_SSE_REG;
7882 /* Unnamed floating parameters are passed in both the
7883 SSE and integer registers. */
7884 t1 = gen_rtx_REG (mode, cum->regno + FIRST_SSE_REG);
7885 t2 = gen_rtx_REG (mode, regno);
7886 t1 = gen_rtx_EXPR_LIST (VOIDmode, t1, const0_rtx);
7887 t2 = gen_rtx_EXPR_LIST (VOIDmode, t2, const0_rtx);
7888 return gen_rtx_PARALLEL (mode, gen_rtvec (2, t1, t2));
7891 /* Handle aggregated types passed in register. */
7892 if (orig_mode == BLKmode)
7894 if (bytes > 0 && bytes <= 8)
7895 mode = (bytes > 4 ? DImode : SImode);
7896 if (mode == BLKmode)
7900 return gen_reg_or_parallel (mode, orig_mode, regno);
7903 /* Return where to put the arguments to a function.
7904 Return zero to push the argument on the stack, or a hard register in which to store the argument.
7906 MODE is the argument's machine mode. TYPE is the data type of the
7907 argument. It is null for libcalls where that information may not be
7908 available. CUM gives information about the preceding args and about
7909 the function being called. NAMED is nonzero if this argument is a
7910 named parameter (otherwise it is an extra parameter matching an
7914 ix86_function_arg (cumulative_args_t cum_v, machine_mode omode,
7915 const_tree type, bool named)
7917 CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
7918 machine_mode mode = omode;
7919 HOST_WIDE_INT bytes, words;
7922 /* All pointer bounds argumntas are handled separately here. */
7923 if ((type && POINTER_BOUNDS_TYPE_P (type))
7924 || POINTER_BOUNDS_MODE_P (mode))
7926 /* Return NULL if bounds are forced to go in Bounds Table. */
7927 if (cum->bnds_in_bt)
7929 /* Return the next available bound reg if any. */
7930 else if (cum->bnd_regno <= LAST_BND_REG)
7931 arg = gen_rtx_REG (BNDmode, cum->bnd_regno);
7932 /* Return the next special slot number otherwise. */
7934 arg = GEN_INT (cum->bnd_regno - LAST_BND_REG - 1);
7939 if (mode == BLKmode)
7940 bytes = int_size_in_bytes (type);
7942 bytes = GET_MODE_SIZE (mode);
7943 words = (bytes + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
7945 /* To simplify the code below, represent vector types with a vector mode
7946 even if MMX/SSE are not active. */
7947 if (type && TREE_CODE (type) == VECTOR_TYPE)
7948 mode = type_natural_mode (type, cum, false);
7950 if (TARGET_64BIT && (cum ? cum->call_abi : ix86_abi) == MS_ABI)
7951 arg = function_arg_ms_64 (cum, mode, omode, named, bytes);
7952 else if (TARGET_64BIT)
7953 arg = function_arg_64 (cum, mode, omode, type, named);
7955 arg = function_arg_32 (cum, mode, omode, type, bytes, words);
7960 /* A C expression that indicates when an argument must be passed by
7961 reference. If nonzero for an argument, a copy of that argument is
7962 made in memory and a pointer to the argument is passed instead of
7963 the argument itself. The pointer is passed in whatever way is
7964 appropriate for passing a pointer to that type. */
7967 ix86_pass_by_reference (cumulative_args_t cum_v, machine_mode mode,
7968 const_tree type, bool)
7970 CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
7972 /* Bounds are never passed by reference. */
7973 if ((type && POINTER_BOUNDS_TYPE_P (type))
7974 || POINTER_BOUNDS_MODE_P (mode))
7977 /* See Windows x64 Software Convention. */
7978 if (TARGET_64BIT && (cum ? cum->call_abi : ix86_abi) == MS_ABI)
7980 int msize = (int) GET_MODE_SIZE (mode);
7983 /* Arrays are passed by reference. */
7984 if (TREE_CODE (type) == ARRAY_TYPE)
7987 if (AGGREGATE_TYPE_P (type))
7989 /* Structs/unions of sizes other than 8, 16, 32, or 64 bits
7990 are passed by reference. */
7991 msize = int_size_in_bytes (type);
7995 /* __m128 is passed by reference. */
7997 case 1: case 2: case 4: case 8:
8003 else if (TARGET_64BIT && type && int_size_in_bytes (type) == -1)
8009 /* Return true when TYPE should be 128bit aligned for 32bit argument
8010 passing ABI. XXX: This function is obsolete and is only used for
8011 checking psABI compatibility with previous versions of GCC. */
8014 ix86_compat_aligned_value_p (const_tree type)
8016 machine_mode mode = TYPE_MODE (type);
8017 if (((TARGET_SSE && SSE_REG_MODE_P (mode))
8021 && (!TYPE_USER_ALIGN (type) || TYPE_ALIGN (type) > 128))
8023 if (TYPE_ALIGN (type) < 128)
8026 if (AGGREGATE_TYPE_P (type))
8028 /* Walk the aggregates recursively. */
8029 switch (TREE_CODE (type))
8033 case QUAL_UNION_TYPE:
8037 /* Walk all the structure fields. */
8038 for (field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field))
8040 if (TREE_CODE (field) == FIELD_DECL
8041 && ix86_compat_aligned_value_p (TREE_TYPE (field)))
8048 /* Just for use if some languages passes arrays by value. */
8049 if (ix86_compat_aligned_value_p (TREE_TYPE (type)))
8060 /* Return the alignment boundary for MODE and TYPE with alignment ALIGN.
8061 XXX: This function is obsolete and is only used for checking psABI
8062 compatibility with previous versions of GCC. */
8065 ix86_compat_function_arg_boundary (machine_mode mode,
8066 const_tree type, unsigned int align)
8068 /* In 32bit, only _Decimal128 and __float128 are aligned to their
8069 natural boundaries. */
8070 if (!TARGET_64BIT && mode != TDmode && mode != TFmode)
8072 /* i386 ABI defines all arguments to be 4 byte aligned. We have to
8073 make an exception for SSE modes since these require 128bit
8076 The handling here differs from field_alignment. ICC aligns MMX
8077 arguments to 4 byte boundaries, while structure fields are aligned
8078 to 8 byte boundaries. */
8081 if (!(TARGET_SSE && SSE_REG_MODE_P (mode)))
8082 align = PARM_BOUNDARY;
8086 if (!ix86_compat_aligned_value_p (type))
8087 align = PARM_BOUNDARY;
8090 if (align > BIGGEST_ALIGNMENT)
8091 align = BIGGEST_ALIGNMENT;
8095 /* Return true when TYPE should be 128bit aligned for 32bit argument
8099 ix86_contains_aligned_value_p (const_tree type)
8101 machine_mode mode = TYPE_MODE (type);
8103 if (mode == XFmode || mode == XCmode)
8106 if (TYPE_ALIGN (type) < 128)
8109 if (AGGREGATE_TYPE_P (type))
8111 /* Walk the aggregates recursively. */
8112 switch (TREE_CODE (type))
8116 case QUAL_UNION_TYPE:
8120 /* Walk all the structure fields. */
8121 for (field = TYPE_FIELDS (type);
8123 field = DECL_CHAIN (field))
8125 if (TREE_CODE (field) == FIELD_DECL
8126 && ix86_contains_aligned_value_p (TREE_TYPE (field)))
8133 /* Just for use if some languages passes arrays by value. */
8134 if (ix86_contains_aligned_value_p (TREE_TYPE (type)))
8143 return TYPE_ALIGN (type) >= 128;
8148 /* Gives the alignment boundary, in bits, of an argument with the
8149 specified mode and type. */
8152 ix86_function_arg_boundary (machine_mode mode, const_tree type)
8157 /* Since the main variant type is used for call, we convert it to
8158 the main variant type. */
8159 type = TYPE_MAIN_VARIANT (type);
8160 align = TYPE_ALIGN (type);
8163 align = GET_MODE_ALIGNMENT (mode);
8164 if (align < PARM_BOUNDARY)
8165 align = PARM_BOUNDARY;
8169 unsigned int saved_align = align;
8173 /* i386 ABI defines XFmode arguments to be 4 byte aligned. */
8176 if (mode == XFmode || mode == XCmode)
8177 align = PARM_BOUNDARY;
8179 else if (!ix86_contains_aligned_value_p (type))
8180 align = PARM_BOUNDARY;
8183 align = PARM_BOUNDARY;
8188 && align != ix86_compat_function_arg_boundary (mode, type,
8192 inform (input_location,
8193 "The ABI for passing parameters with %d-byte"
8194 " alignment has changed in GCC 4.6",
8195 align / BITS_PER_UNIT);
8202 /* Return true if N is a possible register number of function value. */
8205 ix86_function_value_regno_p (const unsigned int regno)
8212 return (!TARGET_64BIT || ix86_cfun_abi () != MS_ABI);
8215 return TARGET_64BIT && ix86_cfun_abi () != MS_ABI;
8219 return chkp_function_instrumented_p (current_function_decl);
8221 /* Complex values are returned in %st(0)/%st(1) pair. */
8224 /* TODO: The function should depend on current function ABI but
8225 builtins.c would need updating then. Therefore we use the
8227 if (TARGET_64BIT && ix86_cfun_abi () == MS_ABI)
8229 return TARGET_FLOAT_RETURNS_IN_80387;
8231 /* Complex values are returned in %xmm0/%xmm1 pair. */
8237 if (TARGET_MACHO || TARGET_64BIT)
8245 /* Define how to find the value returned by a function.
8246 VALTYPE is the data type of the value (as a tree).
8247 If the precise function being called is known, FUNC is its FUNCTION_DECL;
8248 otherwise, FUNC is 0. */
8251 function_value_32 (machine_mode orig_mode, machine_mode mode,
8252 const_tree fntype, const_tree fn)
8256 /* 8-byte vector modes in %mm0. See ix86_return_in_memory for where
8257 we normally prevent this case when mmx is not available. However
8258 some ABIs may require the result to be returned like DImode. */
8259 if (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 8)
8260 regno = FIRST_MMX_REG;
8262 /* 16-byte vector modes in %xmm0. See ix86_return_in_memory for where
8263 we prevent this case when sse is not available. However some ABIs
8264 may require the result to be returned like integer TImode. */
8265 else if (mode == TImode
8266 || (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 16))
8267 regno = FIRST_SSE_REG;
8269 /* 32-byte vector modes in %ymm0. */
8270 else if (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 32)
8271 regno = FIRST_SSE_REG;
8273 /* 64-byte vector modes in %zmm0. */
8274 else if (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 64)
8275 regno = FIRST_SSE_REG;
8277 /* Floating point return values in %st(0) (unless -mno-fp-ret-in-387). */
8278 else if (X87_FLOAT_MODE_P (mode) && TARGET_FLOAT_RETURNS_IN_80387)
8279 regno = FIRST_FLOAT_REG;
8281 /* Most things go in %eax. */
8284 /* Override FP return register with %xmm0 for local functions when
8285 SSE math is enabled or for functions with sseregparm attribute. */
8286 if ((fn || fntype) && (mode == SFmode || mode == DFmode))
8288 int sse_level = ix86_function_sseregparm (fntype, fn, false);
8289 if (sse_level == -1)
8291 error ("calling %qD with SSE caling convention without "
8292 "SSE/SSE2 enabled", fn);
8293 sorry ("this is a GCC bug that can be worked around by adding "
8294 "attribute used to function called");
8296 else if ((sse_level >= 1 && mode == SFmode)
8297 || (sse_level == 2 && mode == DFmode))
8298 regno = FIRST_SSE_REG;
8301 /* OImode shouldn't be used directly. */
8302 gcc_assert (mode != OImode);
8304 return gen_rtx_REG (orig_mode, regno);
8308 function_value_64 (machine_mode orig_mode, machine_mode mode,
8313 /* Handle libcalls, which don't provide a type node. */
8314 if (valtype == NULL)
8328 regno = FIRST_SSE_REG;
8332 regno = FIRST_FLOAT_REG;
8340 return gen_rtx_REG (mode, regno);
8342 else if (POINTER_TYPE_P (valtype))
8344 /* Pointers are always returned in word_mode. */
8348 ret = construct_container (mode, orig_mode, valtype, 1,
8349 X86_64_REGPARM_MAX, X86_64_SSE_REGPARM_MAX,
8350 x86_64_int_return_registers, 0);
8352 /* For zero sized structures, construct_container returns NULL, but we
8353 need to keep rest of compiler happy by returning meaningful value. */
8355 ret = gen_rtx_REG (orig_mode, AX_REG);
8361 function_value_ms_64 (machine_mode orig_mode, machine_mode mode,
8364 unsigned int regno = AX_REG;
8368 switch (GET_MODE_SIZE (mode))
8371 if (valtype != NULL_TREE
8372 && !VECTOR_INTEGER_TYPE_P (valtype)
8373 && !VECTOR_INTEGER_TYPE_P (valtype)
8374 && !INTEGRAL_TYPE_P (valtype)
8375 && !VECTOR_FLOAT_TYPE_P (valtype))
8377 if ((SCALAR_INT_MODE_P (mode) || VECTOR_MODE_P (mode))
8378 && !COMPLEX_MODE_P (mode))
8379 regno = FIRST_SSE_REG;
8383 if (mode == SFmode || mode == DFmode)
8384 regno = FIRST_SSE_REG;
8390 return gen_rtx_REG (orig_mode, regno);
8394 ix86_function_value_1 (const_tree valtype, const_tree fntype_or_decl,
8395 machine_mode orig_mode, machine_mode mode)
8397 const_tree fn, fntype;
8400 if (fntype_or_decl && DECL_P (fntype_or_decl))
8401 fn = fntype_or_decl;
8402 fntype = fn ? TREE_TYPE (fn) : fntype_or_decl;
8404 if ((valtype && POINTER_BOUNDS_TYPE_P (valtype))
8405 || POINTER_BOUNDS_MODE_P (mode))
8406 return gen_rtx_REG (BNDmode, FIRST_BND_REG);
8407 else if (TARGET_64BIT && ix86_function_type_abi (fntype) == MS_ABI)
8408 return function_value_ms_64 (orig_mode, mode, valtype);
8409 else if (TARGET_64BIT)
8410 return function_value_64 (orig_mode, mode, valtype);
8412 return function_value_32 (orig_mode, mode, fntype, fn);
8416 ix86_function_value (const_tree valtype, const_tree fntype_or_decl, bool)
8418 machine_mode mode, orig_mode;
8420 orig_mode = TYPE_MODE (valtype);
8421 mode = type_natural_mode (valtype, NULL, true);
8422 return ix86_function_value_1 (valtype, fntype_or_decl, orig_mode, mode);
8425 /* Return an RTX representing a place where a function returns
8426 or recieves pointer bounds or NULL if no bounds are returned.
8428 VALTYPE is a data type of a value returned by the function.
8430 FN_DECL_OR_TYPE is a tree node representing FUNCTION_DECL
8431 or FUNCTION_TYPE of the function.
8433 If OUTGOING is false, return a place in which the caller will
8434 see the return value. Otherwise, return a place where a
8435 function returns a value. */
8438 ix86_function_value_bounds (const_tree valtype,
8439 const_tree fntype_or_decl ATTRIBUTE_UNUSED,
8440 bool outgoing ATTRIBUTE_UNUSED)
8444 if (BOUNDED_TYPE_P (valtype))
8445 res = gen_rtx_REG (BNDmode, FIRST_BND_REG);
8446 else if (chkp_type_has_pointer (valtype))
8451 unsigned i, bnd_no = 0;
8453 bitmap_obstack_initialize (NULL);
8454 slots = BITMAP_ALLOC (NULL);
8455 chkp_find_bound_slots (valtype, slots);
8457 EXECUTE_IF_SET_IN_BITMAP (slots, 0, i, bi)
8459 rtx reg = gen_rtx_REG (BNDmode, FIRST_BND_REG + bnd_no);
8460 rtx offs = GEN_INT (i * POINTER_SIZE / BITS_PER_UNIT);
8461 gcc_assert (bnd_no < 2);
8462 bounds[bnd_no++] = gen_rtx_EXPR_LIST (VOIDmode, reg, offs);
8465 res = gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (bnd_no, bounds));
8467 BITMAP_FREE (slots);
8468 bitmap_obstack_release (NULL);
8476 /* Pointer function arguments and return values are promoted to
8480 ix86_promote_function_mode (const_tree type, machine_mode mode,
8481 int *punsignedp, const_tree fntype,
8484 if (type != NULL_TREE && POINTER_TYPE_P (type))
8486 *punsignedp = POINTERS_EXTEND_UNSIGNED;
8489 return default_promote_function_mode (type, mode, punsignedp, fntype,
8493 /* Return true if a structure, union or array with MODE containing FIELD
8494 should be accessed using BLKmode. */
8497 ix86_member_type_forces_blk (const_tree field, machine_mode mode)
8499 /* Union with XFmode must be in BLKmode. */
8500 return (mode == XFmode
8501 && (TREE_CODE (DECL_FIELD_CONTEXT (field)) == UNION_TYPE
8502 || TREE_CODE (DECL_FIELD_CONTEXT (field)) == QUAL_UNION_TYPE));
8506 ix86_libcall_value (machine_mode mode)
8508 return ix86_function_value_1 (NULL, NULL, mode, mode);
8511 /* Return true iff type is returned in memory. */
8514 ix86_return_in_memory (const_tree type, const_tree fntype ATTRIBUTE_UNUSED)
8516 #ifdef SUBTARGET_RETURN_IN_MEMORY
8517 return SUBTARGET_RETURN_IN_MEMORY (type, fntype);
8519 const machine_mode mode = type_natural_mode (type, NULL, true);
8522 if (POINTER_BOUNDS_TYPE_P (type))
8527 if (ix86_function_type_abi (fntype) == MS_ABI)
8529 size = int_size_in_bytes (type);
8531 /* __m128 is returned in xmm0. */
8532 if ((!type || VECTOR_INTEGER_TYPE_P (type)
8533 || INTEGRAL_TYPE_P (type)
8534 || VECTOR_FLOAT_TYPE_P (type))
8535 && (SCALAR_INT_MODE_P (mode) || VECTOR_MODE_P (mode))
8536 && !COMPLEX_MODE_P (mode)
8537 && (GET_MODE_SIZE (mode) == 16 || size == 16))
8540 /* Otherwise, the size must be exactly in [1248]. */
8541 return size != 1 && size != 2 && size != 4 && size != 8;
8545 int needed_intregs, needed_sseregs;
8547 return examine_argument (mode, type, 1,
8548 &needed_intregs, &needed_sseregs);
8553 if (mode == BLKmode)
8556 size = int_size_in_bytes (type);
8558 if (MS_AGGREGATE_RETURN && AGGREGATE_TYPE_P (type) && size <= 8)
8561 if (VECTOR_MODE_P (mode) || mode == TImode)
8563 /* User-created vectors small enough to fit in EAX. */
8567 /* Unless ABI prescibes otherwise,
8568 MMX/3dNow values are returned in MM0 if available. */
8571 return TARGET_VECT8_RETURNS || !TARGET_MMX;
8573 /* SSE values are returned in XMM0 if available. */
8577 /* AVX values are returned in YMM0 if available. */
8581 /* AVX512F values are returned in ZMM0 if available. */
8583 return !TARGET_AVX512F;
8592 /* OImode shouldn't be used directly. */
8593 gcc_assert (mode != OImode);
8601 /* Create the va_list data type. */
8603 /* Returns the calling convention specific va_list date type.
8604 The argument ABI can be DEFAULT_ABI, MS_ABI, or SYSV_ABI. */
8607 ix86_build_builtin_va_list_abi (enum calling_abi abi)
8609 tree f_gpr, f_fpr, f_ovf, f_sav, record, type_decl;
8611 /* For i386 we use plain pointer to argument area. */
8612 if (!TARGET_64BIT || abi == MS_ABI)
8613 return build_pointer_type (char_type_node);
8615 record = lang_hooks.types.make_type (RECORD_TYPE);
8616 type_decl = build_decl (BUILTINS_LOCATION,
8617 TYPE_DECL, get_identifier ("__va_list_tag"), record);
8619 f_gpr = build_decl (BUILTINS_LOCATION,
8620 FIELD_DECL, get_identifier ("gp_offset"),
8621 unsigned_type_node);
8622 f_fpr = build_decl (BUILTINS_LOCATION,
8623 FIELD_DECL, get_identifier ("fp_offset"),
8624 unsigned_type_node);
8625 f_ovf = build_decl (BUILTINS_LOCATION,
8626 FIELD_DECL, get_identifier ("overflow_arg_area"),
8628 f_sav = build_decl (BUILTINS_LOCATION,
8629 FIELD_DECL, get_identifier ("reg_save_area"),
8632 va_list_gpr_counter_field = f_gpr;
8633 va_list_fpr_counter_field = f_fpr;
8635 DECL_FIELD_CONTEXT (f_gpr) = record;
8636 DECL_FIELD_CONTEXT (f_fpr) = record;
8637 DECL_FIELD_CONTEXT (f_ovf) = record;
8638 DECL_FIELD_CONTEXT (f_sav) = record;
8640 TYPE_STUB_DECL (record) = type_decl;
8641 TYPE_NAME (record) = type_decl;
8642 TYPE_FIELDS (record) = f_gpr;
8643 DECL_CHAIN (f_gpr) = f_fpr;
8644 DECL_CHAIN (f_fpr) = f_ovf;
8645 DECL_CHAIN (f_ovf) = f_sav;
8647 layout_type (record);
8649 /* The correct type is an array type of one element. */
8650 return build_array_type (record, build_index_type (size_zero_node));
8653 /* Setup the builtin va_list data type and for 64-bit the additional
8654 calling convention specific va_list data types. */
8657 ix86_build_builtin_va_list (void)
8659 tree ret = ix86_build_builtin_va_list_abi (ix86_abi);
8661 /* Initialize abi specific va_list builtin types. */
8665 if (ix86_abi == MS_ABI)
8667 t = ix86_build_builtin_va_list_abi (SYSV_ABI);
8668 if (TREE_CODE (t) != RECORD_TYPE)
8669 t = build_variant_type_copy (t);
8670 sysv_va_list_type_node = t;
8675 if (TREE_CODE (t) != RECORD_TYPE)
8676 t = build_variant_type_copy (t);
8677 sysv_va_list_type_node = t;
8679 if (ix86_abi != MS_ABI)
8681 t = ix86_build_builtin_va_list_abi (MS_ABI);
8682 if (TREE_CODE (t) != RECORD_TYPE)
8683 t = build_variant_type_copy (t);
8684 ms_va_list_type_node = t;
8689 if (TREE_CODE (t) != RECORD_TYPE)
8690 t = build_variant_type_copy (t);
8691 ms_va_list_type_node = t;
8698 /* Worker function for TARGET_SETUP_INCOMING_VARARGS. */
8701 setup_incoming_varargs_64 (CUMULATIVE_ARGS *cum)
8707 /* GPR size of varargs save area. */
8708 if (cfun->va_list_gpr_size)
8709 ix86_varargs_gpr_size = X86_64_REGPARM_MAX * UNITS_PER_WORD;
8711 ix86_varargs_gpr_size = 0;
8713 /* FPR size of varargs save area. We don't need it if we don't pass
8714 anything in SSE registers. */
8715 if (TARGET_SSE && cfun->va_list_fpr_size)
8716 ix86_varargs_fpr_size = X86_64_SSE_REGPARM_MAX * 16;
8718 ix86_varargs_fpr_size = 0;
8720 if (! ix86_varargs_gpr_size && ! ix86_varargs_fpr_size)
8723 save_area = frame_pointer_rtx;
8724 set = get_varargs_alias_set ();
8726 max = cum->regno + cfun->va_list_gpr_size / UNITS_PER_WORD;
8727 if (max > X86_64_REGPARM_MAX)
8728 max = X86_64_REGPARM_MAX;
8730 for (i = cum->regno; i < max; i++)
8732 mem = gen_rtx_MEM (word_mode,
8733 plus_constant (Pmode, save_area, i * UNITS_PER_WORD));
8734 MEM_NOTRAP_P (mem) = 1;
8735 set_mem_alias_set (mem, set);
8736 emit_move_insn (mem,
8737 gen_rtx_REG (word_mode,
8738 x86_64_int_parameter_registers[i]));
8741 if (ix86_varargs_fpr_size)
8744 rtx_code_label *label;
8747 /* Now emit code to save SSE registers. The AX parameter contains number
8748 of SSE parameter registers used to call this function, though all we
8749 actually check here is the zero/non-zero status. */
8751 label = gen_label_rtx ();
8752 test = gen_rtx_EQ (VOIDmode, gen_rtx_REG (QImode, AX_REG), const0_rtx);
8753 emit_jump_insn (gen_cbranchqi4 (test, XEXP (test, 0), XEXP (test, 1),
8756 /* ??? If !TARGET_SSE_TYPELESS_STORES, would we perform better if
8757 we used movdqa (i.e. TImode) instead? Perhaps even better would
8758 be if we could determine the real mode of the data, via a hook
8759 into pass_stdarg. Ignore all that for now. */
8761 if (crtl->stack_alignment_needed < GET_MODE_ALIGNMENT (smode))
8762 crtl->stack_alignment_needed = GET_MODE_ALIGNMENT (smode);
8764 max = cum->sse_regno + cfun->va_list_fpr_size / 16;
8765 if (max > X86_64_SSE_REGPARM_MAX)
8766 max = X86_64_SSE_REGPARM_MAX;
8768 for (i = cum->sse_regno; i < max; ++i)
8770 mem = plus_constant (Pmode, save_area,
8771 i * 16 + ix86_varargs_gpr_size);
8772 mem = gen_rtx_MEM (smode, mem);
8773 MEM_NOTRAP_P (mem) = 1;
8774 set_mem_alias_set (mem, set);
8775 set_mem_align (mem, GET_MODE_ALIGNMENT (smode));
8777 emit_move_insn (mem, gen_rtx_REG (smode, SSE_REGNO (i)));
8785 setup_incoming_varargs_ms_64 (CUMULATIVE_ARGS *cum)
8787 alias_set_type set = get_varargs_alias_set ();
8790 /* Reset to zero, as there might be a sysv vaarg used
8792 ix86_varargs_gpr_size = 0;
8793 ix86_varargs_fpr_size = 0;
8795 for (i = cum->regno; i < X86_64_MS_REGPARM_MAX; i++)
8799 mem = gen_rtx_MEM (Pmode,
8800 plus_constant (Pmode, virtual_incoming_args_rtx,
8801 i * UNITS_PER_WORD));
8802 MEM_NOTRAP_P (mem) = 1;
8803 set_mem_alias_set (mem, set);
8805 reg = gen_rtx_REG (Pmode, x86_64_ms_abi_int_parameter_registers[i]);
8806 emit_move_insn (mem, reg);
8811 ix86_setup_incoming_varargs (cumulative_args_t cum_v, machine_mode mode,
8812 tree type, int *, int no_rtl)
8814 CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
8815 CUMULATIVE_ARGS next_cum;
8818 /* This argument doesn't appear to be used anymore. Which is good,
8819 because the old code here didn't suppress rtl generation. */
8820 gcc_assert (!no_rtl);
8825 fntype = TREE_TYPE (current_function_decl);
8827 /* For varargs, we do not want to skip the dummy va_dcl argument.
8828 For stdargs, we do want to skip the last named argument. */
8830 if (stdarg_p (fntype))
8831 ix86_function_arg_advance (pack_cumulative_args (&next_cum), mode, type,
8834 if (cum->call_abi == MS_ABI)
8835 setup_incoming_varargs_ms_64 (&next_cum);
8837 setup_incoming_varargs_64 (&next_cum);
8841 ix86_setup_incoming_vararg_bounds (cumulative_args_t cum_v,
8842 enum machine_mode mode,
8844 int *pretend_size ATTRIBUTE_UNUSED,
8847 CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
8848 CUMULATIVE_ARGS next_cum;
8851 int bnd_reg, i, max;
8853 gcc_assert (!no_rtl);
8855 /* Do nothing if we use plain pointer to argument area. */
8856 if (!TARGET_64BIT || cum->call_abi == MS_ABI)
8859 fntype = TREE_TYPE (current_function_decl);
8861 /* For varargs, we do not want to skip the dummy va_dcl argument.
8862 For stdargs, we do want to skip the last named argument. */
8864 if (stdarg_p (fntype))
8865 ix86_function_arg_advance (pack_cumulative_args (&next_cum), mode, type,
8867 save_area = frame_pointer_rtx;
8869 max = cum->regno + cfun->va_list_gpr_size / UNITS_PER_WORD;
8870 if (max > X86_64_REGPARM_MAX)
8871 max = X86_64_REGPARM_MAX;
8873 bnd_reg = cum->bnd_regno + cum->force_bnd_pass;
8874 if (chkp_function_instrumented_p (current_function_decl))
8875 for (i = cum->regno; i < max; i++)
8877 rtx addr = plus_constant (Pmode, save_area, i * UNITS_PER_WORD);
8878 rtx reg = gen_rtx_REG (DImode,
8879 x86_64_int_parameter_registers[i]);
8883 if (bnd_reg <= LAST_BND_REG)
8884 bounds = gen_rtx_REG (BNDmode, bnd_reg);
8888 plus_constant (Pmode, arg_pointer_rtx,
8889 (LAST_BND_REG - bnd_reg) * GET_MODE_SIZE (Pmode));
8890 bounds = gen_reg_rtx (BNDmode);
8891 emit_insn (BNDmode == BND64mode
8892 ? gen_bnd64_ldx (bounds, ldx_addr, ptr)
8893 : gen_bnd32_ldx (bounds, ldx_addr, ptr));
8896 emit_insn (BNDmode == BND64mode
8897 ? gen_bnd64_stx (addr, ptr, bounds)
8898 : gen_bnd32_stx (addr, ptr, bounds));
8905 /* Checks if TYPE is of kind va_list char *. */
8908 is_va_list_char_pointer (tree type)
8912 /* For 32-bit it is always true. */
8915 canonic = ix86_canonical_va_list_type (type);
8916 return (canonic == ms_va_list_type_node
8917 || (ix86_abi == MS_ABI && canonic == va_list_type_node));
8920 /* Implement va_start. */
8923 ix86_va_start (tree valist, rtx nextarg)
8925 HOST_WIDE_INT words, n_gpr, n_fpr;
8926 tree f_gpr, f_fpr, f_ovf, f_sav;
8927 tree gpr, fpr, ovf, sav, t;
8931 if (flag_split_stack
8932 && cfun->machine->split_stack_varargs_pointer == NULL_RTX)
8934 unsigned int scratch_regno;
8936 /* When we are splitting the stack, we can't refer to the stack
8937 arguments using internal_arg_pointer, because they may be on
8938 the old stack. The split stack prologue will arrange to
8939 leave a pointer to the old stack arguments in a scratch
8940 register, which we here copy to a pseudo-register. The split
8941 stack prologue can't set the pseudo-register directly because
8942 it (the prologue) runs before any registers have been saved. */
8944 scratch_regno = split_stack_prologue_scratch_regno ();
8945 if (scratch_regno != INVALID_REGNUM)
8950 reg = gen_reg_rtx (Pmode);
8951 cfun->machine->split_stack_varargs_pointer = reg;
8954 emit_move_insn (reg, gen_rtx_REG (Pmode, scratch_regno));
8958 push_topmost_sequence ();
8959 emit_insn_after (seq, entry_of_function ());
8960 pop_topmost_sequence ();
8964 /* Only 64bit target needs something special. */
8965 if (!TARGET_64BIT || is_va_list_char_pointer (TREE_TYPE (valist)))
8967 if (cfun->machine->split_stack_varargs_pointer == NULL_RTX)
8968 std_expand_builtin_va_start (valist, nextarg);
8973 va_r = expand_expr (valist, NULL_RTX, VOIDmode, EXPAND_WRITE);
8974 next = expand_binop (ptr_mode, add_optab,
8975 cfun->machine->split_stack_varargs_pointer,
8976 crtl->args.arg_offset_rtx,
8977 NULL_RTX, 0, OPTAB_LIB_WIDEN);
8978 convert_move (va_r, next, 0);
8980 /* Store zero bounds for va_list. */
8981 if (chkp_function_instrumented_p (current_function_decl))
8982 chkp_expand_bounds_reset_for_mem (valist,
8983 make_tree (TREE_TYPE (valist),
8990 f_gpr = TYPE_FIELDS (TREE_TYPE (sysv_va_list_type_node));
8991 f_fpr = DECL_CHAIN (f_gpr);
8992 f_ovf = DECL_CHAIN (f_fpr);
8993 f_sav = DECL_CHAIN (f_ovf);
8995 valist = build_simple_mem_ref (valist);
8996 TREE_TYPE (valist) = TREE_TYPE (sysv_va_list_type_node);
8997 /* The following should be folded into the MEM_REF offset. */
8998 gpr = build3 (COMPONENT_REF, TREE_TYPE (f_gpr), unshare_expr (valist),
9000 fpr = build3 (COMPONENT_REF, TREE_TYPE (f_fpr), unshare_expr (valist),
9002 ovf = build3 (COMPONENT_REF, TREE_TYPE (f_ovf), unshare_expr (valist),
9004 sav = build3 (COMPONENT_REF, TREE_TYPE (f_sav), unshare_expr (valist),
9007 /* Count number of gp and fp argument registers used. */
9008 words = crtl->args.info.words;
9009 n_gpr = crtl->args.info.regno;
9010 n_fpr = crtl->args.info.sse_regno;
9012 if (cfun->va_list_gpr_size)
9014 type = TREE_TYPE (gpr);
9015 t = build2 (MODIFY_EXPR, type,
9016 gpr, build_int_cst (type, n_gpr * 8));
9017 TREE_SIDE_EFFECTS (t) = 1;
9018 expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL);
9021 if (TARGET_SSE && cfun->va_list_fpr_size)
9023 type = TREE_TYPE (fpr);
9024 t = build2 (MODIFY_EXPR, type, fpr,
9025 build_int_cst (type, n_fpr * 16 + 8*X86_64_REGPARM_MAX));
9026 TREE_SIDE_EFFECTS (t) = 1;
9027 expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL);
9030 /* Find the overflow area. */
9031 type = TREE_TYPE (ovf);
9032 if (cfun->machine->split_stack_varargs_pointer == NULL_RTX)
9033 ovf_rtx = crtl->args.internal_arg_pointer;
9035 ovf_rtx = cfun->machine->split_stack_varargs_pointer;
9036 t = make_tree (type, ovf_rtx);
9038 t = fold_build_pointer_plus_hwi (t, words * UNITS_PER_WORD);
9040 /* Store zero bounds for overflow area pointer. */
9041 if (chkp_function_instrumented_p (current_function_decl))
9042 chkp_expand_bounds_reset_for_mem (ovf, t);
9044 t = build2 (MODIFY_EXPR, type, ovf, t);
9045 TREE_SIDE_EFFECTS (t) = 1;
9046 expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL);
9048 if (ix86_varargs_gpr_size || ix86_varargs_fpr_size)
9050 /* Find the register save area.
9051 Prologue of the function save it right above stack frame. */
9052 type = TREE_TYPE (sav);
9053 t = make_tree (type, frame_pointer_rtx);
9054 if (!ix86_varargs_gpr_size)
9055 t = fold_build_pointer_plus_hwi (t, -8 * X86_64_REGPARM_MAX);
9057 /* Store zero bounds for save area pointer. */
9058 if (chkp_function_instrumented_p (current_function_decl))
9059 chkp_expand_bounds_reset_for_mem (sav, t);
9061 t = build2 (MODIFY_EXPR, type, sav, t);
9062 TREE_SIDE_EFFECTS (t) = 1;
9063 expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL);
9067 /* Implement va_arg. */
9070 ix86_gimplify_va_arg (tree valist, tree type, gimple_seq *pre_p,
9073 static const int intreg[6] = { 0, 1, 2, 3, 4, 5 };
9074 tree f_gpr, f_fpr, f_ovf, f_sav;
9075 tree gpr, fpr, ovf, sav, t;
9077 tree lab_false, lab_over = NULL_TREE;
9082 machine_mode nat_mode;
9083 unsigned int arg_boundary;
9085 /* Only 64bit target needs something special. */
9086 if (!TARGET_64BIT || is_va_list_char_pointer (TREE_TYPE (valist)))
9087 return std_gimplify_va_arg_expr (valist, type, pre_p, post_p);
9089 f_gpr = TYPE_FIELDS (TREE_TYPE (sysv_va_list_type_node));
9090 f_fpr = DECL_CHAIN (f_gpr);
9091 f_ovf = DECL_CHAIN (f_fpr);
9092 f_sav = DECL_CHAIN (f_ovf);
9094 gpr = build3 (COMPONENT_REF, TREE_TYPE (f_gpr),
9095 valist, f_gpr, NULL_TREE);
9097 fpr = build3 (COMPONENT_REF, TREE_TYPE (f_fpr), valist, f_fpr, NULL_TREE);
9098 ovf = build3 (COMPONENT_REF, TREE_TYPE (f_ovf), valist, f_ovf, NULL_TREE);
9099 sav = build3 (COMPONENT_REF, TREE_TYPE (f_sav), valist, f_sav, NULL_TREE);
9101 indirect_p = pass_by_reference (NULL, TYPE_MODE (type), type, false);
9103 type = build_pointer_type (type);
9104 size = int_size_in_bytes (type);
9105 rsize = (size + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
9107 nat_mode = type_natural_mode (type, NULL, false);
9122 /* Unnamed 256 and 512bit vector mode parameters are passed on stack. */
9123 if (!TARGET_64BIT_MS_ABI)
9130 container = construct_container (nat_mode, TYPE_MODE (type),
9131 type, 0, X86_64_REGPARM_MAX,
9132 X86_64_SSE_REGPARM_MAX, intreg,
9137 /* Pull the value out of the saved registers. */
9139 addr = create_tmp_var (ptr_type_node, "addr");
9143 int needed_intregs, needed_sseregs;
9145 tree int_addr, sse_addr;
9147 lab_false = create_artificial_label (UNKNOWN_LOCATION);
9148 lab_over = create_artificial_label (UNKNOWN_LOCATION);
9150 examine_argument (nat_mode, type, 0, &needed_intregs, &needed_sseregs);
9152 need_temp = (!REG_P (container)
9153 && ((needed_intregs && TYPE_ALIGN (type) > 64)
9154 || TYPE_ALIGN (type) > 128));
9156 /* In case we are passing structure, verify that it is consecutive block
9157 on the register save area. If not we need to do moves. */
9158 if (!need_temp && !REG_P (container))
9160 /* Verify that all registers are strictly consecutive */
9161 if (SSE_REGNO_P (REGNO (XEXP (XVECEXP (container, 0, 0), 0))))
9165 for (i = 0; i < XVECLEN (container, 0) && !need_temp; i++)
9167 rtx slot = XVECEXP (container, 0, i);
9168 if (REGNO (XEXP (slot, 0)) != FIRST_SSE_REG + (unsigned int) i
9169 || INTVAL (XEXP (slot, 1)) != i * 16)
9177 for (i = 0; i < XVECLEN (container, 0) && !need_temp; i++)
9179 rtx slot = XVECEXP (container, 0, i);
9180 if (REGNO (XEXP (slot, 0)) != (unsigned int) i
9181 || INTVAL (XEXP (slot, 1)) != i * 8)
9193 int_addr = create_tmp_var (ptr_type_node, "int_addr");
9194 sse_addr = create_tmp_var (ptr_type_node, "sse_addr");
9197 /* First ensure that we fit completely in registers. */
9200 t = build_int_cst (TREE_TYPE (gpr),
9201 (X86_64_REGPARM_MAX - needed_intregs + 1) * 8);
9202 t = build2 (GE_EXPR, boolean_type_node, gpr, t);
9203 t2 = build1 (GOTO_EXPR, void_type_node, lab_false);
9204 t = build3 (COND_EXPR, void_type_node, t, t2, NULL_TREE);
9205 gimplify_and_add (t, pre_p);
9209 t = build_int_cst (TREE_TYPE (fpr),
9210 (X86_64_SSE_REGPARM_MAX - needed_sseregs + 1) * 16
9211 + X86_64_REGPARM_MAX * 8);
9212 t = build2 (GE_EXPR, boolean_type_node, fpr, t);
9213 t2 = build1 (GOTO_EXPR, void_type_node, lab_false);
9214 t = build3 (COND_EXPR, void_type_node, t, t2, NULL_TREE);
9215 gimplify_and_add (t, pre_p);
9218 /* Compute index to start of area used for integer regs. */
9221 /* int_addr = gpr + sav; */
9222 t = fold_build_pointer_plus (sav, gpr);
9223 gimplify_assign (int_addr, t, pre_p);
9227 /* sse_addr = fpr + sav; */
9228 t = fold_build_pointer_plus (sav, fpr);
9229 gimplify_assign (sse_addr, t, pre_p);
9233 int i, prev_size = 0;
9234 tree temp = create_tmp_var (type, "va_arg_tmp");
9237 t = build1 (ADDR_EXPR, build_pointer_type (type), temp);
9238 gimplify_assign (addr, t, pre_p);
9240 for (i = 0; i < XVECLEN (container, 0); i++)
9242 rtx slot = XVECEXP (container, 0, i);
9243 rtx reg = XEXP (slot, 0);
9244 machine_mode mode = GET_MODE (reg);
9250 tree dest_addr, dest;
9251 int cur_size = GET_MODE_SIZE (mode);
9253 gcc_assert (prev_size <= INTVAL (XEXP (slot, 1)));
9254 prev_size = INTVAL (XEXP (slot, 1));
9255 if (prev_size + cur_size > size)
9257 cur_size = size - prev_size;
9258 mode = mode_for_size (cur_size * BITS_PER_UNIT, MODE_INT, 1);
9259 if (mode == BLKmode)
9262 piece_type = lang_hooks.types.type_for_mode (mode, 1);
9263 if (mode == GET_MODE (reg))
9264 addr_type = build_pointer_type (piece_type);
9266 addr_type = build_pointer_type_for_mode (piece_type, ptr_mode,
9268 daddr_type = build_pointer_type_for_mode (piece_type, ptr_mode,
9271 if (SSE_REGNO_P (REGNO (reg)))
9273 src_addr = sse_addr;
9274 src_offset = (REGNO (reg) - FIRST_SSE_REG) * 16;
9278 src_addr = int_addr;
9279 src_offset = REGNO (reg) * 8;
9281 src_addr = fold_convert (addr_type, src_addr);
9282 src_addr = fold_build_pointer_plus_hwi (src_addr, src_offset);
9284 dest_addr = fold_convert (daddr_type, addr);
9285 dest_addr = fold_build_pointer_plus_hwi (dest_addr, prev_size);
9286 if (cur_size == GET_MODE_SIZE (mode))
9288 src = build_va_arg_indirect_ref (src_addr);
9289 dest = build_va_arg_indirect_ref (dest_addr);
9291 gimplify_assign (dest, src, pre_p);
9296 = build_call_expr (builtin_decl_implicit (BUILT_IN_MEMCPY),
9297 3, dest_addr, src_addr,
9298 size_int (cur_size));
9299 gimplify_and_add (copy, pre_p);
9301 prev_size += cur_size;
9307 t = build2 (PLUS_EXPR, TREE_TYPE (gpr), gpr,
9308 build_int_cst (TREE_TYPE (gpr), needed_intregs * 8));
9309 gimplify_assign (gpr, t, pre_p);
9314 t = build2 (PLUS_EXPR, TREE_TYPE (fpr), fpr,
9315 build_int_cst (TREE_TYPE (fpr), needed_sseregs * 16));
9316 gimplify_assign (unshare_expr (fpr), t, pre_p);
9319 gimple_seq_add_stmt (pre_p, gimple_build_goto (lab_over));
9321 gimple_seq_add_stmt (pre_p, gimple_build_label (lab_false));
9324 /* ... otherwise out of the overflow area. */
9326 /* When we align parameter on stack for caller, if the parameter
9327 alignment is beyond MAX_SUPPORTED_STACK_ALIGNMENT, it will be
9328 aligned at MAX_SUPPORTED_STACK_ALIGNMENT. We will match callee
9329 here with caller. */
9330 arg_boundary = ix86_function_arg_boundary (VOIDmode, type);
9331 if ((unsigned int) arg_boundary > MAX_SUPPORTED_STACK_ALIGNMENT)
9332 arg_boundary = MAX_SUPPORTED_STACK_ALIGNMENT;
9334 /* Care for on-stack alignment if needed. */
9335 if (arg_boundary <= 64 || size == 0)
9339 HOST_WIDE_INT align = arg_boundary / 8;
9340 t = fold_build_pointer_plus_hwi (ovf, align - 1);
9341 t = build2 (BIT_AND_EXPR, TREE_TYPE (t), t,
9342 build_int_cst (TREE_TYPE (t), -align));
9345 gimplify_expr (&t, pre_p, NULL, is_gimple_val, fb_rvalue);
9346 gimplify_assign (addr, t, pre_p);
9348 t = fold_build_pointer_plus_hwi (t, rsize * UNITS_PER_WORD);
9349 gimplify_assign (unshare_expr (ovf), t, pre_p);
9352 gimple_seq_add_stmt (pre_p, gimple_build_label (lab_over));
9354 ptrtype = build_pointer_type_for_mode (type, ptr_mode, true);
9355 addr = fold_convert (ptrtype, addr);
9358 addr = build_va_arg_indirect_ref (addr);
9359 return build_va_arg_indirect_ref (addr);
9362 /* Return true if OPNUM's MEM should be matched
9363 in movabs* patterns. */
9366 ix86_check_movabs (rtx insn, int opnum)
9370 set = PATTERN (insn);
9371 if (GET_CODE (set) == PARALLEL)
9372 set = XVECEXP (set, 0, 0);
9373 gcc_assert (GET_CODE (set) == SET);
9374 mem = XEXP (set, opnum);
9375 while (GET_CODE (mem) == SUBREG)
9376 mem = SUBREG_REG (mem);
9377 gcc_assert (MEM_P (mem));
9378 return volatile_ok || !MEM_VOLATILE_P (mem);
9381 /* Initialize the table of extra 80387 mathematical constants. */
9384 init_ext_80387_constants (void)
9386 static const char * cst[5] =
9388 "0.3010299956639811952256464283594894482", /* 0: fldlg2 */
9389 "0.6931471805599453094286904741849753009", /* 1: fldln2 */
9390 "1.4426950408889634073876517827983434472", /* 2: fldl2e */
9391 "3.3219280948873623478083405569094566090", /* 3: fldl2t */
9392 "3.1415926535897932385128089594061862044", /* 4: fldpi */
9396 for (i = 0; i < 5; i++)
9398 real_from_string (&ext_80387_constants_table[i], cst[i]);
9399 /* Ensure each constant is rounded to XFmode precision. */
9400 real_convert (&ext_80387_constants_table[i],
9401 XFmode, &ext_80387_constants_table[i]);
9404 ext_80387_constants_init = 1;
9407 /* Return non-zero if the constant is something that
9408 can be loaded with a special instruction. */
9411 standard_80387_constant_p (rtx x)
9413 machine_mode mode = GET_MODE (x);
9417 if (!(CONST_DOUBLE_P (x) && X87_FLOAT_MODE_P (mode)))
9420 if (x == CONST0_RTX (mode))
9422 if (x == CONST1_RTX (mode))
9425 REAL_VALUE_FROM_CONST_DOUBLE (r, x);
9427 /* For XFmode constants, try to find a special 80387 instruction when
9428 optimizing for size or on those CPUs that benefit from them. */
9430 && (optimize_function_for_size_p (cfun) || TARGET_EXT_80387_CONSTANTS))
9434 if (! ext_80387_constants_init)
9435 init_ext_80387_constants ();
9437 for (i = 0; i < 5; i++)
9438 if (real_identical (&r, &ext_80387_constants_table[i]))
9442 /* Load of the constant -0.0 or -1.0 will be split as
9443 fldz;fchs or fld1;fchs sequence. */
9444 if (real_isnegzero (&r))
9446 if (real_identical (&r, &dconstm1))
9452 /* Return the opcode of the special instruction to be used to load
9456 standard_80387_constant_opcode (rtx x)
9458 switch (standard_80387_constant_p (x))
9482 /* Return the CONST_DOUBLE representing the 80387 constant that is
9483 loaded by the specified special instruction. The argument IDX
9484 matches the return value from standard_80387_constant_p. */
9487 standard_80387_constant_rtx (int idx)
9491 if (! ext_80387_constants_init)
9492 init_ext_80387_constants ();
9508 return CONST_DOUBLE_FROM_REAL_VALUE (ext_80387_constants_table[i],
9512 /* Return 1 if X is all 0s and 2 if x is all 1s
9513 in supported SSE/AVX vector mode. */
9516 standard_sse_constant_p (rtx x)
9523 mode = GET_MODE (x);
9525 if (x == const0_rtx || x == CONST0_RTX (mode))
9527 if (vector_all_ones_operand (x, mode))
9555 /* Return the opcode of the special instruction to be used to load
9559 standard_sse_constant_opcode (rtx_insn *insn, rtx x)
9561 switch (standard_sse_constant_p (x))
9564 switch (get_attr_mode (insn))
9567 return "vpxord\t%g0, %g0, %g0";
9569 return TARGET_AVX512DQ ? "vxorps\t%g0, %g0, %g0"
9570 : "vpxord\t%g0, %g0, %g0";
9572 return TARGET_AVX512DQ ? "vxorpd\t%g0, %g0, %g0"
9573 : "vpxorq\t%g0, %g0, %g0";
9575 return TARGET_AVX512VL ? "vpxord\t%t0, %t0, %t0"
9576 : "%vpxor\t%0, %d0";
9578 return "%vxorpd\t%0, %d0";
9580 return "%vxorps\t%0, %d0";
9583 return TARGET_AVX512VL ? "vpxord\t%x0, %x0, %x0"
9584 : "vpxor\t%x0, %x0, %x0";
9586 return "vxorpd\t%x0, %x0, %x0";
9588 return "vxorps\t%x0, %x0, %x0";
9596 || get_attr_mode (insn) == MODE_XI
9597 || get_attr_mode (insn) == MODE_V8DF
9598 || get_attr_mode (insn) == MODE_V16SF)
9599 return "vpternlogd\t{$0xFF, %g0, %g0, %g0|%g0, %g0, %g0, 0xFF}";
9601 return "vpcmpeqd\t%0, %0, %0";
9603 return "pcmpeqd\t%0, %0";
9611 /* Returns true if OP contains a symbol reference */
9614 symbolic_reference_mentioned_p (rtx op)
9619 if (GET_CODE (op) == SYMBOL_REF || GET_CODE (op) == LABEL_REF)
9622 fmt = GET_RTX_FORMAT (GET_CODE (op));
9623 for (i = GET_RTX_LENGTH (GET_CODE (op)) - 1; i >= 0; i--)
9629 for (j = XVECLEN (op, i) - 1; j >= 0; j--)
9630 if (symbolic_reference_mentioned_p (XVECEXP (op, i, j)))
9634 else if (fmt[i] == 'e' && symbolic_reference_mentioned_p (XEXP (op, i)))
9641 /* Return true if it is appropriate to emit `ret' instructions in the
9642 body of a function. Do this only if the epilogue is simple, needing a
9643 couple of insns. Prior to reloading, we can't tell how many registers
9644 must be saved, so return false then. Return false if there is no frame
9645 marker to de-allocate. */
9648 ix86_can_use_return_insn_p (void)
9650 struct ix86_frame frame;
9652 if (! reload_completed || frame_pointer_needed)
9655 /* Don't allow more than 32k pop, since that's all we can do
9656 with one instruction. */
9657 if (crtl->args.pops_args && crtl->args.size >= 32768)
9660 ix86_compute_frame_layout (&frame);
9661 return (frame.stack_pointer_offset == UNITS_PER_WORD
9662 && (frame.nregs + frame.nsseregs) == 0);
9665 /* Value should be nonzero if functions must have frame pointers.
9666 Zero means the frame pointer need not be set up (and parms may
9667 be accessed via the stack pointer) in functions that seem suitable. */
9670 ix86_frame_pointer_required (void)
9672 /* If we accessed previous frames, then the generated code expects
9673 to be able to access the saved ebp value in our frame. */
9674 if (cfun->machine->accesses_prev_frame)
9677 /* Several x86 os'es need a frame pointer for other reasons,
9678 usually pertaining to setjmp. */
9679 if (SUBTARGET_FRAME_POINTER_REQUIRED)
9682 /* For older 32-bit runtimes setjmp requires valid frame-pointer. */
9683 if (TARGET_32BIT_MS_ABI && cfun->calls_setjmp)
9686 /* Win64 SEH, very large frames need a frame-pointer as maximum stack
9687 allocation is 4GB. */
9688 if (TARGET_64BIT_MS_ABI && get_frame_size () > SEH_MAX_FRAME_SIZE)
9691 /* In ix86_option_override_internal, TARGET_OMIT_LEAF_FRAME_POINTER
9692 turns off the frame pointer by default. Turn it back on now if
9693 we've not got a leaf function. */
9694 if (TARGET_OMIT_LEAF_FRAME_POINTER
9696 || ix86_current_function_calls_tls_descriptor))
9699 if (crtl->profile && !flag_fentry)
9705 /* Record that the current function accesses previous call frames. */
9708 ix86_setup_frame_addresses (void)
9710 cfun->machine->accesses_prev_frame = 1;
9713 #ifndef USE_HIDDEN_LINKONCE
9714 # if defined(HAVE_GAS_HIDDEN) && (SUPPORTS_ONE_ONLY - 0)
9715 # define USE_HIDDEN_LINKONCE 1
9717 # define USE_HIDDEN_LINKONCE 0
9721 static int pic_labels_used;
9723 /* Fills in the label name that should be used for a pc thunk for
9724 the given register. */
9727 get_pc_thunk_name (char name[32], unsigned int regno)
9729 gcc_assert (!TARGET_64BIT);
9731 if (USE_HIDDEN_LINKONCE)
9732 sprintf (name, "__x86.get_pc_thunk.%s", reg_names[regno]);
9734 ASM_GENERATE_INTERNAL_LABEL (name, "LPR", regno);
9738 /* This function generates code for -fpic that loads %ebx with
9739 the return address of the caller and then returns. */
9742 ix86_code_end (void)
9747 for (regno = AX_REG; regno <= SP_REG; regno++)
9752 if (!(pic_labels_used & (1 << regno)))
9755 get_pc_thunk_name (name, regno);
9757 decl = build_decl (BUILTINS_LOCATION, FUNCTION_DECL,
9758 get_identifier (name),
9759 build_function_type_list (void_type_node, NULL_TREE));
9760 DECL_RESULT (decl) = build_decl (BUILTINS_LOCATION, RESULT_DECL,
9761 NULL_TREE, void_type_node);
9762 TREE_PUBLIC (decl) = 1;
9763 TREE_STATIC (decl) = 1;
9764 DECL_IGNORED_P (decl) = 1;
9769 switch_to_section (darwin_sections[text_coal_section]);
9770 fputs ("\t.weak_definition\t", asm_out_file);
9771 assemble_name (asm_out_file, name);
9772 fputs ("\n\t.private_extern\t", asm_out_file);
9773 assemble_name (asm_out_file, name);
9774 putc ('\n', asm_out_file);
9775 ASM_OUTPUT_LABEL (asm_out_file, name);
9776 DECL_WEAK (decl) = 1;
9780 if (USE_HIDDEN_LINKONCE)
9782 cgraph_node::create (decl)->set_comdat_group (DECL_ASSEMBLER_NAME (decl));
9784 targetm.asm_out.unique_section (decl, 0);
9785 switch_to_section (get_named_section (decl, NULL, 0));
9787 targetm.asm_out.globalize_label (asm_out_file, name);
9788 fputs ("\t.hidden\t", asm_out_file);
9789 assemble_name (asm_out_file, name);
9790 putc ('\n', asm_out_file);
9791 ASM_DECLARE_FUNCTION_NAME (asm_out_file, name, decl);
9795 switch_to_section (text_section);
9796 ASM_OUTPUT_LABEL (asm_out_file, name);
9799 DECL_INITIAL (decl) = make_node (BLOCK);
9800 current_function_decl = decl;
9801 init_function_start (decl);
9802 first_function_block_is_cold = false;
9803 /* Make sure unwind info is emitted for the thunk if needed. */
9804 final_start_function (emit_barrier (), asm_out_file, 1);
9806 /* Pad stack IP move with 4 instructions (two NOPs count
9807 as one instruction). */
9808 if (TARGET_PAD_SHORT_FUNCTION)
9813 fputs ("\tnop\n", asm_out_file);
9816 xops[0] = gen_rtx_REG (Pmode, regno);
9817 xops[1] = gen_rtx_MEM (Pmode, stack_pointer_rtx);
9818 output_asm_insn ("mov%z0\t{%1, %0|%0, %1}", xops);
9819 output_asm_insn ("%!ret", NULL);
9820 final_end_function ();
9821 init_insn_lengths ();
9822 free_after_compilation (cfun);
9824 current_function_decl = NULL;
9827 if (flag_split_stack)
9828 file_end_indicate_split_stack ();
9831 /* Emit code for the SET_GOT patterns. */
9834 output_set_got (rtx dest, rtx label)
9840 if (TARGET_VXWORKS_RTP && flag_pic)
9842 /* Load (*VXWORKS_GOTT_BASE) into the PIC register. */
9843 xops[2] = gen_rtx_MEM (Pmode,
9844 gen_rtx_SYMBOL_REF (Pmode, VXWORKS_GOTT_BASE));
9845 output_asm_insn ("mov{l}\t{%2, %0|%0, %2}", xops);
9847 /* Load (*VXWORKS_GOTT_BASE)[VXWORKS_GOTT_INDEX] into the PIC register.
9848 Use %P and a local symbol in order to print VXWORKS_GOTT_INDEX as
9849 an unadorned address. */
9850 xops[2] = gen_rtx_SYMBOL_REF (Pmode, VXWORKS_GOTT_INDEX);
9851 SYMBOL_REF_FLAGS (xops[2]) |= SYMBOL_FLAG_LOCAL;
9852 output_asm_insn ("mov{l}\t{%P2(%0), %0|%0, DWORD PTR %P2[%0]}", xops);
9856 xops[1] = gen_rtx_SYMBOL_REF (Pmode, GOT_SYMBOL_NAME);
9861 /* We don't need a pic base, we're not producing pic. */
9864 xops[2] = gen_rtx_LABEL_REF (Pmode, label ? label : gen_label_rtx ());
9865 output_asm_insn ("mov%z0\t{%2, %0|%0, %2}", xops);
9866 targetm.asm_out.internal_label (asm_out_file, "L",
9867 CODE_LABEL_NUMBER (XEXP (xops[2], 0)));
9872 get_pc_thunk_name (name, REGNO (dest));
9873 pic_labels_used |= 1 << REGNO (dest);
9875 xops[2] = gen_rtx_SYMBOL_REF (Pmode, ggc_strdup (name));
9876 xops[2] = gen_rtx_MEM (QImode, xops[2]);
9877 output_asm_insn ("%!call\t%X2", xops);
9880 /* Output the Mach-O "canonical" pic base label name ("Lxx$pb") here.
9881 This is what will be referenced by the Mach-O PIC subsystem. */
9882 if (machopic_should_output_picbase_label () || !label)
9883 ASM_OUTPUT_LABEL (asm_out_file, MACHOPIC_FUNCTION_BASE_NAME);
9885 /* When we are restoring the pic base at the site of a nonlocal label,
9886 and we decided to emit the pic base above, we will still output a
9887 local label used for calculating the correction offset (even though
9888 the offset will be 0 in that case). */
9890 targetm.asm_out.internal_label (asm_out_file, "L",
9891 CODE_LABEL_NUMBER (label));
9896 output_asm_insn ("add%z0\t{%1, %0|%0, %1}", xops);
9901 /* Generate an "push" pattern for input ARG. */
9906 struct machine_function *m = cfun->machine;
9908 if (m->fs.cfa_reg == stack_pointer_rtx)
9909 m->fs.cfa_offset += UNITS_PER_WORD;
9910 m->fs.sp_offset += UNITS_PER_WORD;
9912 if (REG_P (arg) && GET_MODE (arg) != word_mode)
9913 arg = gen_rtx_REG (word_mode, REGNO (arg));
9915 return gen_rtx_SET (gen_rtx_MEM (word_mode,
9916 gen_rtx_PRE_DEC (Pmode,
9917 stack_pointer_rtx)),
9921 /* Generate an "pop" pattern for input ARG. */
9926 if (REG_P (arg) && GET_MODE (arg) != word_mode)
9927 arg = gen_rtx_REG (word_mode, REGNO (arg));
9929 return gen_rtx_SET (arg,
9930 gen_rtx_MEM (word_mode,
9931 gen_rtx_POST_INC (Pmode,
9932 stack_pointer_rtx)));
9935 /* Return >= 0 if there is an unused call-clobbered register available
9936 for the entire function. */
9939 ix86_select_alt_pic_regnum (void)
9941 if (ix86_use_pseudo_pic_reg ())
9942 return INVALID_REGNUM;
9946 && !ix86_current_function_calls_tls_descriptor)
9949 /* Can't use the same register for both PIC and DRAP. */
9951 drap = REGNO (crtl->drap_reg);
9954 for (i = 2; i >= 0; --i)
9955 if (i != drap && !df_regs_ever_live_p (i))
9959 return INVALID_REGNUM;
9962 /* Return TRUE if we need to save REGNO. */
9965 ix86_save_reg (unsigned int regno, bool maybe_eh_return)
9967 if (regno == REAL_PIC_OFFSET_TABLE_REGNUM
9968 && pic_offset_table_rtx)
9970 if (ix86_use_pseudo_pic_reg ())
9972 /* REAL_PIC_OFFSET_TABLE_REGNUM used by call to
9973 _mcount in prologue. */
9974 if (!TARGET_64BIT && flag_pic && crtl->profile)
9977 else if (df_regs_ever_live_p (REAL_PIC_OFFSET_TABLE_REGNUM)
9979 || crtl->calls_eh_return
9980 || crtl->uses_const_pool
9981 || cfun->has_nonlocal_label)
9982 return ix86_select_alt_pic_regnum () == INVALID_REGNUM;
9985 if (crtl->calls_eh_return && maybe_eh_return)
9990 unsigned test = EH_RETURN_DATA_REGNO (i);
9991 if (test == INVALID_REGNUM)
9999 && regno == REGNO (crtl->drap_reg)
10000 && !cfun->machine->no_drap_save_restore)
10003 return (df_regs_ever_live_p (regno)
10004 && !call_used_regs[regno]
10005 && !fixed_regs[regno]
10006 && (regno != HARD_FRAME_POINTER_REGNUM || !frame_pointer_needed));
10009 /* Return number of saved general prupose registers. */
10012 ix86_nsaved_regs (void)
10017 for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
10018 if (!SSE_REGNO_P (regno) && ix86_save_reg (regno, true))
10023 /* Return number of saved SSE registrers. */
10026 ix86_nsaved_sseregs (void)
10031 if (!TARGET_64BIT_MS_ABI)
10033 for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
10034 if (SSE_REGNO_P (regno) && ix86_save_reg (regno, true))
10039 /* Given FROM and TO register numbers, say whether this elimination is
10040 allowed. If stack alignment is needed, we can only replace argument
10041 pointer with hard frame pointer, or replace frame pointer with stack
10042 pointer. Otherwise, frame pointer elimination is automatically
10043 handled and all other eliminations are valid. */
10046 ix86_can_eliminate (const int from, const int to)
10048 if (stack_realign_fp)
10049 return ((from == ARG_POINTER_REGNUM
10050 && to == HARD_FRAME_POINTER_REGNUM)
10051 || (from == FRAME_POINTER_REGNUM
10052 && to == STACK_POINTER_REGNUM));
10054 return to == STACK_POINTER_REGNUM ? !frame_pointer_needed : true;
10057 /* Return the offset between two registers, one to be eliminated, and the other
10058 its replacement, at the start of a routine. */
10061 ix86_initial_elimination_offset (int from, int to)
10063 struct ix86_frame frame;
10064 ix86_compute_frame_layout (&frame);
10066 if (from == ARG_POINTER_REGNUM && to == HARD_FRAME_POINTER_REGNUM)
10067 return frame.hard_frame_pointer_offset;
10068 else if (from == FRAME_POINTER_REGNUM
10069 && to == HARD_FRAME_POINTER_REGNUM)
10070 return frame.hard_frame_pointer_offset - frame.frame_pointer_offset;
10073 gcc_assert (to == STACK_POINTER_REGNUM);
10075 if (from == ARG_POINTER_REGNUM)
10076 return frame.stack_pointer_offset;
10078 gcc_assert (from == FRAME_POINTER_REGNUM);
10079 return frame.stack_pointer_offset - frame.frame_pointer_offset;
10083 /* In a dynamically-aligned function, we can't know the offset from
10084 stack pointer to frame pointer, so we must ensure that setjmp
10085 eliminates fp against the hard fp (%ebp) rather than trying to
10086 index from %esp up to the top of the frame across a gap that is
10087 of unknown (at compile-time) size. */
10089 ix86_builtin_setjmp_frame_value (void)
10091 return stack_realign_fp ? hard_frame_pointer_rtx : virtual_stack_vars_rtx;
10094 /* When using -fsplit-stack, the allocation routines set a field in
10095 the TCB to the bottom of the stack plus this much space, measured
10098 #define SPLIT_STACK_AVAILABLE 256
10100 /* Fill structure ix86_frame about frame of currently computed function. */
10103 ix86_compute_frame_layout (struct ix86_frame *frame)
10105 unsigned HOST_WIDE_INT stack_alignment_needed;
10106 HOST_WIDE_INT offset;
10107 unsigned HOST_WIDE_INT preferred_alignment;
10108 HOST_WIDE_INT size = get_frame_size ();
10109 HOST_WIDE_INT to_allocate;
10111 frame->nregs = ix86_nsaved_regs ();
10112 frame->nsseregs = ix86_nsaved_sseregs ();
10114 /* 64-bit MS ABI seem to require stack alignment to be always 16 except for
10115 function prologues and leaf. */
10116 if ((TARGET_64BIT_MS_ABI && crtl->preferred_stack_boundary < 128)
10117 && (!crtl->is_leaf || cfun->calls_alloca != 0
10118 || ix86_current_function_calls_tls_descriptor))
10120 crtl->preferred_stack_boundary = 128;
10121 crtl->stack_alignment_needed = 128;
10123 /* preferred_stack_boundary is never updated for call
10124 expanded from tls descriptor. Update it here. We don't update it in
10125 expand stage because according to the comments before
10126 ix86_current_function_calls_tls_descriptor, tls calls may be optimized
10128 else if (ix86_current_function_calls_tls_descriptor
10129 && crtl->preferred_stack_boundary < PREFERRED_STACK_BOUNDARY)
10131 crtl->preferred_stack_boundary = PREFERRED_STACK_BOUNDARY;
10132 if (crtl->stack_alignment_needed < PREFERRED_STACK_BOUNDARY)
10133 crtl->stack_alignment_needed = PREFERRED_STACK_BOUNDARY;
10136 stack_alignment_needed = crtl->stack_alignment_needed / BITS_PER_UNIT;
10137 preferred_alignment = crtl->preferred_stack_boundary / BITS_PER_UNIT;
10139 gcc_assert (!size || stack_alignment_needed);
10140 gcc_assert (preferred_alignment >= STACK_BOUNDARY / BITS_PER_UNIT);
10141 gcc_assert (preferred_alignment <= stack_alignment_needed);
10143 /* For SEH we have to limit the amount of code movement into the prologue.
10144 At present we do this via a BLOCKAGE, at which point there's very little
10145 scheduling that can be done, which means that there's very little point
10146 in doing anything except PUSHs. */
10148 cfun->machine->use_fast_prologue_epilogue = false;
10150 /* During reload iteration the amount of registers saved can change.
10151 Recompute the value as needed. Do not recompute when amount of registers
10152 didn't change as reload does multiple calls to the function and does not
10153 expect the decision to change within single iteration. */
10154 else if (!optimize_bb_for_size_p (ENTRY_BLOCK_PTR_FOR_FN (cfun))
10155 && cfun->machine->use_fast_prologue_epilogue_nregs != frame->nregs)
10157 int count = frame->nregs;
10158 struct cgraph_node *node = cgraph_node::get (current_function_decl);
10160 cfun->machine->use_fast_prologue_epilogue_nregs = count;
10162 /* The fast prologue uses move instead of push to save registers. This
10163 is significantly longer, but also executes faster as modern hardware
10164 can execute the moves in parallel, but can't do that for push/pop.
10166 Be careful about choosing what prologue to emit: When function takes
10167 many instructions to execute we may use slow version as well as in
10168 case function is known to be outside hot spot (this is known with
10169 feedback only). Weight the size of function by number of registers
10170 to save as it is cheap to use one or two push instructions but very
10171 slow to use many of them. */
10173 count = (count - 1) * FAST_PROLOGUE_INSN_COUNT;
10174 if (node->frequency < NODE_FREQUENCY_NORMAL
10175 || (flag_branch_probabilities
10176 && node->frequency < NODE_FREQUENCY_HOT))
10177 cfun->machine->use_fast_prologue_epilogue = false;
10179 cfun->machine->use_fast_prologue_epilogue
10180 = !expensive_function_p (count);
10183 frame->save_regs_using_mov
10184 = (TARGET_PROLOGUE_USING_MOVE && cfun->machine->use_fast_prologue_epilogue
10185 /* If static stack checking is enabled and done with probes,
10186 the registers need to be saved before allocating the frame. */
10187 && flag_stack_check != STATIC_BUILTIN_STACK_CHECK);
10189 /* Skip return address. */
10190 offset = UNITS_PER_WORD;
10192 /* Skip pushed static chain. */
10193 if (ix86_static_chain_on_stack)
10194 offset += UNITS_PER_WORD;
10196 /* Skip saved base pointer. */
10197 if (frame_pointer_needed)
10198 offset += UNITS_PER_WORD;
10199 frame->hfp_save_offset = offset;
10201 /* The traditional frame pointer location is at the top of the frame. */
10202 frame->hard_frame_pointer_offset = offset;
10204 /* Register save area */
10205 offset += frame->nregs * UNITS_PER_WORD;
10206 frame->reg_save_offset = offset;
10208 /* On SEH target, registers are pushed just before the frame pointer
10211 frame->hard_frame_pointer_offset = offset;
10213 /* Align and set SSE register save area. */
10214 if (frame->nsseregs)
10216 /* The only ABI that has saved SSE registers (Win64) also has a
10217 16-byte aligned default stack, and thus we don't need to be
10218 within the re-aligned local stack frame to save them. */
10219 gcc_assert (INCOMING_STACK_BOUNDARY >= 128);
10220 offset = (offset + 16 - 1) & -16;
10221 offset += frame->nsseregs * 16;
10223 frame->sse_reg_save_offset = offset;
10225 /* The re-aligned stack starts here. Values before this point are not
10226 directly comparable with values below this point. In order to make
10227 sure that no value happens to be the same before and after, force
10228 the alignment computation below to add a non-zero value. */
10229 if (stack_realign_fp)
10230 offset = (offset + stack_alignment_needed) & -stack_alignment_needed;
10233 frame->va_arg_size = ix86_varargs_gpr_size + ix86_varargs_fpr_size;
10234 offset += frame->va_arg_size;
10236 /* Align start of frame for local function. */
10237 if (stack_realign_fp
10238 || offset != frame->sse_reg_save_offset
10241 || cfun->calls_alloca
10242 || ix86_current_function_calls_tls_descriptor)
10243 offset = (offset + stack_alignment_needed - 1) & -stack_alignment_needed;
10245 /* Frame pointer points here. */
10246 frame->frame_pointer_offset = offset;
10250 /* Add outgoing arguments area. Can be skipped if we eliminated
10251 all the function calls as dead code.
10252 Skipping is however impossible when function calls alloca. Alloca
10253 expander assumes that last crtl->outgoing_args_size
10254 of stack frame are unused. */
10255 if (ACCUMULATE_OUTGOING_ARGS
10256 && (!crtl->is_leaf || cfun->calls_alloca
10257 || ix86_current_function_calls_tls_descriptor))
10259 offset += crtl->outgoing_args_size;
10260 frame->outgoing_arguments_size = crtl->outgoing_args_size;
10263 frame->outgoing_arguments_size = 0;
10265 /* Align stack boundary. Only needed if we're calling another function
10266 or using alloca. */
10267 if (!crtl->is_leaf || cfun->calls_alloca
10268 || ix86_current_function_calls_tls_descriptor)
10269 offset = (offset + preferred_alignment - 1) & -preferred_alignment;
10271 /* We've reached end of stack frame. */
10272 frame->stack_pointer_offset = offset;
10274 /* Size prologue needs to allocate. */
10275 to_allocate = offset - frame->sse_reg_save_offset;
10277 if ((!to_allocate && frame->nregs <= 1)
10278 || (TARGET_64BIT && to_allocate >= (HOST_WIDE_INT) 0x80000000))
10279 frame->save_regs_using_mov = false;
10281 if (ix86_using_red_zone ()
10282 && crtl->sp_is_unchanging
10284 && !ix86_current_function_calls_tls_descriptor)
10286 frame->red_zone_size = to_allocate;
10287 if (frame->save_regs_using_mov)
10288 frame->red_zone_size += frame->nregs * UNITS_PER_WORD;
10289 if (frame->red_zone_size > RED_ZONE_SIZE - RED_ZONE_RESERVE)
10290 frame->red_zone_size = RED_ZONE_SIZE - RED_ZONE_RESERVE;
10293 frame->red_zone_size = 0;
10294 frame->stack_pointer_offset -= frame->red_zone_size;
10296 /* The SEH frame pointer location is near the bottom of the frame.
10297 This is enforced by the fact that the difference between the
10298 stack pointer and the frame pointer is limited to 240 bytes in
10299 the unwind data structure. */
10302 HOST_WIDE_INT diff;
10304 /* If we can leave the frame pointer where it is, do so. Also, returns
10305 the establisher frame for __builtin_frame_address (0). */
10306 diff = frame->stack_pointer_offset - frame->hard_frame_pointer_offset;
10307 if (diff <= SEH_MAX_FRAME_SIZE
10308 && (diff > 240 || (diff & 15) != 0)
10309 && !crtl->accesses_prior_frames)
10311 /* Ideally we'd determine what portion of the local stack frame
10312 (within the constraint of the lowest 240) is most heavily used.
10313 But without that complication, simply bias the frame pointer
10314 by 128 bytes so as to maximize the amount of the local stack
10315 frame that is addressable with 8-bit offsets. */
10316 frame->hard_frame_pointer_offset = frame->stack_pointer_offset - 128;
10321 /* This is semi-inlined memory_address_length, but simplified
10322 since we know that we're always dealing with reg+offset, and
10323 to avoid having to create and discard all that rtl. */
10326 choose_baseaddr_len (unsigned int regno, HOST_WIDE_INT offset)
10332 /* EBP and R13 cannot be encoded without an offset. */
10333 len = (regno == BP_REG || regno == R13_REG);
10335 else if (IN_RANGE (offset, -128, 127))
10338 /* ESP and R12 must be encoded with a SIB byte. */
10339 if (regno == SP_REG || regno == R12_REG)
10345 /* Return an RTX that points to CFA_OFFSET within the stack frame.
10346 The valid base registers are taken from CFUN->MACHINE->FS. */
10349 choose_baseaddr (HOST_WIDE_INT cfa_offset)
10351 const struct machine_function *m = cfun->machine;
10352 rtx base_reg = NULL;
10353 HOST_WIDE_INT base_offset = 0;
10355 if (m->use_fast_prologue_epilogue)
10357 /* Choose the base register most likely to allow the most scheduling
10358 opportunities. Generally FP is valid throughout the function,
10359 while DRAP must be reloaded within the epilogue. But choose either
10360 over the SP due to increased encoding size. */
10362 if (m->fs.fp_valid)
10364 base_reg = hard_frame_pointer_rtx;
10365 base_offset = m->fs.fp_offset - cfa_offset;
10367 else if (m->fs.drap_valid)
10369 base_reg = crtl->drap_reg;
10370 base_offset = 0 - cfa_offset;
10372 else if (m->fs.sp_valid)
10374 base_reg = stack_pointer_rtx;
10375 base_offset = m->fs.sp_offset - cfa_offset;
10380 HOST_WIDE_INT toffset;
10381 int len = 16, tlen;
10383 /* Choose the base register with the smallest address encoding.
10384 With a tie, choose FP > DRAP > SP. */
10385 if (m->fs.sp_valid)
10387 base_reg = stack_pointer_rtx;
10388 base_offset = m->fs.sp_offset - cfa_offset;
10389 len = choose_baseaddr_len (STACK_POINTER_REGNUM, base_offset);
10391 if (m->fs.drap_valid)
10393 toffset = 0 - cfa_offset;
10394 tlen = choose_baseaddr_len (REGNO (crtl->drap_reg), toffset);
10397 base_reg = crtl->drap_reg;
10398 base_offset = toffset;
10402 if (m->fs.fp_valid)
10404 toffset = m->fs.fp_offset - cfa_offset;
10405 tlen = choose_baseaddr_len (HARD_FRAME_POINTER_REGNUM, toffset);
10408 base_reg = hard_frame_pointer_rtx;
10409 base_offset = toffset;
10414 gcc_assert (base_reg != NULL);
10416 return plus_constant (Pmode, base_reg, base_offset);
10419 /* Emit code to save registers in the prologue. */
10422 ix86_emit_save_regs (void)
10424 unsigned int regno;
10427 for (regno = FIRST_PSEUDO_REGISTER - 1; regno-- > 0; )
10428 if (!SSE_REGNO_P (regno) && ix86_save_reg (regno, true))
10430 insn = emit_insn (gen_push (gen_rtx_REG (word_mode, regno)));
10431 RTX_FRAME_RELATED_P (insn) = 1;
10435 /* Emit a single register save at CFA - CFA_OFFSET. */
10438 ix86_emit_save_reg_using_mov (machine_mode mode, unsigned int regno,
10439 HOST_WIDE_INT cfa_offset)
10441 struct machine_function *m = cfun->machine;
10442 rtx reg = gen_rtx_REG (mode, regno);
10443 rtx mem, addr, base, insn;
10445 addr = choose_baseaddr (cfa_offset);
10446 mem = gen_frame_mem (mode, addr);
10448 /* For SSE saves, we need to indicate the 128-bit alignment. */
10449 set_mem_align (mem, GET_MODE_ALIGNMENT (mode));
10451 insn = emit_move_insn (mem, reg);
10452 RTX_FRAME_RELATED_P (insn) = 1;
10455 if (GET_CODE (base) == PLUS)
10456 base = XEXP (base, 0);
10457 gcc_checking_assert (REG_P (base));
10459 /* When saving registers into a re-aligned local stack frame, avoid
10460 any tricky guessing by dwarf2out. */
10461 if (m->fs.realigned)
10463 gcc_checking_assert (stack_realign_drap);
10465 if (regno == REGNO (crtl->drap_reg))
10467 /* A bit of a hack. We force the DRAP register to be saved in
10468 the re-aligned stack frame, which provides us with a copy
10469 of the CFA that will last past the prologue. Install it. */
10470 gcc_checking_assert (cfun->machine->fs.fp_valid);
10471 addr = plus_constant (Pmode, hard_frame_pointer_rtx,
10472 cfun->machine->fs.fp_offset - cfa_offset);
10473 mem = gen_rtx_MEM (mode, addr);
10474 add_reg_note (insn, REG_CFA_DEF_CFA, mem);
10478 /* The frame pointer is a stable reference within the
10479 aligned frame. Use it. */
10480 gcc_checking_assert (cfun->machine->fs.fp_valid);
10481 addr = plus_constant (Pmode, hard_frame_pointer_rtx,
10482 cfun->machine->fs.fp_offset - cfa_offset);
10483 mem = gen_rtx_MEM (mode, addr);
10484 add_reg_note (insn, REG_CFA_EXPRESSION, gen_rtx_SET (mem, reg));
10488 /* The memory may not be relative to the current CFA register,
10489 which means that we may need to generate a new pattern for
10490 use by the unwind info. */
10491 else if (base != m->fs.cfa_reg)
10493 addr = plus_constant (Pmode, m->fs.cfa_reg,
10494 m->fs.cfa_offset - cfa_offset);
10495 mem = gen_rtx_MEM (mode, addr);
10496 add_reg_note (insn, REG_CFA_OFFSET, gen_rtx_SET (mem, reg));
10500 /* Emit code to save registers using MOV insns.
10501 First register is stored at CFA - CFA_OFFSET. */
10503 ix86_emit_save_regs_using_mov (HOST_WIDE_INT cfa_offset)
10505 unsigned int regno;
10507 for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
10508 if (!SSE_REGNO_P (regno) && ix86_save_reg (regno, true))
10510 ix86_emit_save_reg_using_mov (word_mode, regno, cfa_offset);
10511 cfa_offset -= UNITS_PER_WORD;
10515 /* Emit code to save SSE registers using MOV insns.
10516 First register is stored at CFA - CFA_OFFSET. */
10518 ix86_emit_save_sse_regs_using_mov (HOST_WIDE_INT cfa_offset)
10520 unsigned int regno;
10522 for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
10523 if (SSE_REGNO_P (regno) && ix86_save_reg (regno, true))
10525 ix86_emit_save_reg_using_mov (V4SFmode, regno, cfa_offset);
10530 static GTY(()) rtx queued_cfa_restores;
10532 /* Add a REG_CFA_RESTORE REG note to INSN or queue them until next stack
10533 manipulation insn. The value is on the stack at CFA - CFA_OFFSET.
10534 Don't add the note if the previously saved value will be left untouched
10535 within stack red-zone till return, as unwinders can find the same value
10536 in the register and on the stack. */
10539 ix86_add_cfa_restore_note (rtx_insn *insn, rtx reg, HOST_WIDE_INT cfa_offset)
10541 if (!crtl->shrink_wrapped
10542 && cfa_offset <= cfun->machine->fs.red_zone_offset)
10547 add_reg_note (insn, REG_CFA_RESTORE, reg);
10548 RTX_FRAME_RELATED_P (insn) = 1;
10551 queued_cfa_restores
10552 = alloc_reg_note (REG_CFA_RESTORE, reg, queued_cfa_restores);
10555 /* Add queued REG_CFA_RESTORE notes if any to INSN. */
10558 ix86_add_queued_cfa_restore_notes (rtx insn)
10561 if (!queued_cfa_restores)
10563 for (last = queued_cfa_restores; XEXP (last, 1); last = XEXP (last, 1))
10565 XEXP (last, 1) = REG_NOTES (insn);
10566 REG_NOTES (insn) = queued_cfa_restores;
10567 queued_cfa_restores = NULL_RTX;
10568 RTX_FRAME_RELATED_P (insn) = 1;
10571 /* Expand prologue or epilogue stack adjustment.
10572 The pattern exist to put a dependency on all ebp-based memory accesses.
10573 STYLE should be negative if instructions should be marked as frame related,
10574 zero if %r11 register is live and cannot be freely used and positive
10578 pro_epilogue_adjust_stack (rtx dest, rtx src, rtx offset,
10579 int style, bool set_cfa)
10581 struct machine_function *m = cfun->machine;
10583 bool add_frame_related_expr = false;
10585 if (Pmode == SImode)
10586 insn = gen_pro_epilogue_adjust_stack_si_add (dest, src, offset);
10587 else if (x86_64_immediate_operand (offset, DImode))
10588 insn = gen_pro_epilogue_adjust_stack_di_add (dest, src, offset);
10592 /* r11 is used by indirect sibcall return as well, set before the
10593 epilogue and used after the epilogue. */
10595 tmp = gen_rtx_REG (DImode, R11_REG);
10598 gcc_assert (src != hard_frame_pointer_rtx
10599 && dest != hard_frame_pointer_rtx);
10600 tmp = hard_frame_pointer_rtx;
10602 insn = emit_insn (gen_rtx_SET (tmp, offset));
10604 add_frame_related_expr = true;
10606 insn = gen_pro_epilogue_adjust_stack_di_add (dest, src, tmp);
10609 insn = emit_insn (insn);
10611 ix86_add_queued_cfa_restore_notes (insn);
10617 gcc_assert (m->fs.cfa_reg == src);
10618 m->fs.cfa_offset += INTVAL (offset);
10619 m->fs.cfa_reg = dest;
10621 r = gen_rtx_PLUS (Pmode, src, offset);
10622 r = gen_rtx_SET (dest, r);
10623 add_reg_note (insn, REG_CFA_ADJUST_CFA, r);
10624 RTX_FRAME_RELATED_P (insn) = 1;
10626 else if (style < 0)
10628 RTX_FRAME_RELATED_P (insn) = 1;
10629 if (add_frame_related_expr)
10631 rtx r = gen_rtx_PLUS (Pmode, src, offset);
10632 r = gen_rtx_SET (dest, r);
10633 add_reg_note (insn, REG_FRAME_RELATED_EXPR, r);
10637 if (dest == stack_pointer_rtx)
10639 HOST_WIDE_INT ooffset = m->fs.sp_offset;
10640 bool valid = m->fs.sp_valid;
10642 if (src == hard_frame_pointer_rtx)
10644 valid = m->fs.fp_valid;
10645 ooffset = m->fs.fp_offset;
10647 else if (src == crtl->drap_reg)
10649 valid = m->fs.drap_valid;
10654 /* Else there are two possibilities: SP itself, which we set
10655 up as the default above. Or EH_RETURN_STACKADJ_RTX, which is
10656 taken care of this by hand along the eh_return path. */
10657 gcc_checking_assert (src == stack_pointer_rtx
10658 || offset == const0_rtx);
10661 m->fs.sp_offset = ooffset - INTVAL (offset);
10662 m->fs.sp_valid = valid;
10666 /* Find an available register to be used as dynamic realign argument
10667 pointer regsiter. Such a register will be written in prologue and
10668 used in begin of body, so it must not be
10669 1. parameter passing register.
10671 We reuse static-chain register if it is available. Otherwise, we
10672 use DI for i386 and R13 for x86-64. We chose R13 since it has
10675 Return: the regno of chosen register. */
10677 static unsigned int
10678 find_drap_reg (void)
10680 tree decl = cfun->decl;
10684 /* Use R13 for nested function or function need static chain.
10685 Since function with tail call may use any caller-saved
10686 registers in epilogue, DRAP must not use caller-saved
10687 register in such case. */
10688 if (DECL_STATIC_CHAIN (decl) || crtl->tail_call_emit)
10695 /* Use DI for nested function or function need static chain.
10696 Since function with tail call may use any caller-saved
10697 registers in epilogue, DRAP must not use caller-saved
10698 register in such case. */
10699 if (DECL_STATIC_CHAIN (decl) || crtl->tail_call_emit)
10702 /* Reuse static chain register if it isn't used for parameter
10704 if (ix86_function_regparm (TREE_TYPE (decl), decl) <= 2)
10706 unsigned int ccvt = ix86_get_callcvt (TREE_TYPE (decl));
10707 if ((ccvt & (IX86_CALLCVT_FASTCALL | IX86_CALLCVT_THISCALL)) == 0)
10714 /* Return minimum incoming stack alignment. */
10716 static unsigned int
10717 ix86_minimum_incoming_stack_boundary (bool sibcall)
10719 unsigned int incoming_stack_boundary;
10721 /* Prefer the one specified at command line. */
10722 if (ix86_user_incoming_stack_boundary)
10723 incoming_stack_boundary = ix86_user_incoming_stack_boundary;
10724 /* In 32bit, use MIN_STACK_BOUNDARY for incoming stack boundary
10725 if -mstackrealign is used, it isn't used for sibcall check and
10726 estimated stack alignment is 128bit. */
10729 && ix86_force_align_arg_pointer
10730 && crtl->stack_alignment_estimated == 128)
10731 incoming_stack_boundary = MIN_STACK_BOUNDARY;
10733 incoming_stack_boundary = ix86_default_incoming_stack_boundary;
10735 /* Incoming stack alignment can be changed on individual functions
10736 via force_align_arg_pointer attribute. We use the smallest
10737 incoming stack boundary. */
10738 if (incoming_stack_boundary > MIN_STACK_BOUNDARY
10739 && lookup_attribute (ix86_force_align_arg_pointer_string,
10740 TYPE_ATTRIBUTES (TREE_TYPE (current_function_decl))))
10741 incoming_stack_boundary = MIN_STACK_BOUNDARY;
10743 /* The incoming stack frame has to be aligned at least at
10744 parm_stack_boundary. */
10745 if (incoming_stack_boundary < crtl->parm_stack_boundary)
10746 incoming_stack_boundary = crtl->parm_stack_boundary;
10748 /* Stack at entrance of main is aligned by runtime. We use the
10749 smallest incoming stack boundary. */
10750 if (incoming_stack_boundary > MAIN_STACK_BOUNDARY
10751 && DECL_NAME (current_function_decl)
10752 && MAIN_NAME_P (DECL_NAME (current_function_decl))
10753 && DECL_FILE_SCOPE_P (current_function_decl))
10754 incoming_stack_boundary = MAIN_STACK_BOUNDARY;
10756 return incoming_stack_boundary;
10759 /* Update incoming stack boundary and estimated stack alignment. */
10762 ix86_update_stack_boundary (void)
10764 ix86_incoming_stack_boundary
10765 = ix86_minimum_incoming_stack_boundary (false);
10767 /* x86_64 vararg needs 16byte stack alignment for register save
10771 && crtl->stack_alignment_estimated < 128)
10772 crtl->stack_alignment_estimated = 128;
10775 /* Handle the TARGET_GET_DRAP_RTX hook. Return NULL if no DRAP is
10776 needed or an rtx for DRAP otherwise. */
10779 ix86_get_drap_rtx (void)
10781 if (ix86_force_drap || !ACCUMULATE_OUTGOING_ARGS)
10782 crtl->need_drap = true;
10784 if (stack_realign_drap)
10786 /* Assign DRAP to vDRAP and returns vDRAP */
10787 unsigned int regno = find_drap_reg ();
10790 rtx_insn *seq, *insn;
10792 arg_ptr = gen_rtx_REG (Pmode, regno);
10793 crtl->drap_reg = arg_ptr;
10796 drap_vreg = copy_to_reg (arg_ptr);
10797 seq = get_insns ();
10800 insn = emit_insn_before (seq, NEXT_INSN (entry_of_function ()));
10803 add_reg_note (insn, REG_CFA_SET_VDRAP, drap_vreg);
10804 RTX_FRAME_RELATED_P (insn) = 1;
10812 /* Handle the TARGET_INTERNAL_ARG_POINTER hook. */
10815 ix86_internal_arg_pointer (void)
10817 return virtual_incoming_args_rtx;
10820 struct scratch_reg {
10825 /* Return a short-lived scratch register for use on function entry.
10826 In 32-bit mode, it is valid only after the registers are saved
10827 in the prologue. This register must be released by means of
10828 release_scratch_register_on_entry once it is dead. */
10831 get_scratch_register_on_entry (struct scratch_reg *sr)
10839 /* We always use R11 in 64-bit mode. */
10844 tree decl = current_function_decl, fntype = TREE_TYPE (decl);
10846 = lookup_attribute ("fastcall", TYPE_ATTRIBUTES (fntype)) != NULL_TREE;
10848 = lookup_attribute ("thiscall", TYPE_ATTRIBUTES (fntype)) != NULL_TREE;
10849 bool static_chain_p = DECL_STATIC_CHAIN (decl);
10850 int regparm = ix86_function_regparm (fntype, decl);
10852 = crtl->drap_reg ? REGNO (crtl->drap_reg) : INVALID_REGNUM;
10854 /* 'fastcall' sets regparm to 2, uses ecx/edx for arguments and eax
10855 for the static chain register. */
10856 if ((regparm < 1 || (fastcall_p && !static_chain_p))
10857 && drap_regno != AX_REG)
10859 /* 'thiscall' sets regparm to 1, uses ecx for arguments and edx
10860 for the static chain register. */
10861 else if (thiscall_p && !static_chain_p && drap_regno != AX_REG)
10863 else if (regparm < 2 && !thiscall_p && drap_regno != DX_REG)
10865 /* ecx is the static chain register. */
10866 else if (regparm < 3 && !fastcall_p && !thiscall_p
10868 && drap_regno != CX_REG)
10870 else if (ix86_save_reg (BX_REG, true))
10872 /* esi is the static chain register. */
10873 else if (!(regparm == 3 && static_chain_p)
10874 && ix86_save_reg (SI_REG, true))
10876 else if (ix86_save_reg (DI_REG, true))
10880 regno = (drap_regno == AX_REG ? DX_REG : AX_REG);
10885 sr->reg = gen_rtx_REG (Pmode, regno);
10888 rtx_insn *insn = emit_insn (gen_push (sr->reg));
10889 RTX_FRAME_RELATED_P (insn) = 1;
10893 /* Release a scratch register obtained from the preceding function. */
10896 release_scratch_register_on_entry (struct scratch_reg *sr)
10900 struct machine_function *m = cfun->machine;
10901 rtx x, insn = emit_insn (gen_pop (sr->reg));
10903 /* The RTX_FRAME_RELATED_P mechanism doesn't know about pop. */
10904 RTX_FRAME_RELATED_P (insn) = 1;
10905 x = gen_rtx_PLUS (Pmode, stack_pointer_rtx, GEN_INT (UNITS_PER_WORD));
10906 x = gen_rtx_SET (stack_pointer_rtx, x);
10907 add_reg_note (insn, REG_FRAME_RELATED_EXPR, x);
10908 m->fs.sp_offset -= UNITS_PER_WORD;
10912 #define PROBE_INTERVAL (1 << STACK_CHECK_PROBE_INTERVAL_EXP)
10914 /* Emit code to adjust the stack pointer by SIZE bytes while probing it. */
10917 ix86_adjust_stack_and_probe (const HOST_WIDE_INT size)
10919 /* We skip the probe for the first interval + a small dope of 4 words and
10920 probe that many bytes past the specified size to maintain a protection
10921 area at the botton of the stack. */
10922 const int dope = 4 * UNITS_PER_WORD;
10923 rtx size_rtx = GEN_INT (size), last;
10925 /* See if we have a constant small number of probes to generate. If so,
10926 that's the easy case. The run-time loop is made up of 11 insns in the
10927 generic case while the compile-time loop is made up of 3+2*(n-1) insns
10928 for n # of intervals. */
10929 if (size <= 5 * PROBE_INTERVAL)
10931 HOST_WIDE_INT i, adjust;
10932 bool first_probe = true;
10934 /* Adjust SP and probe at PROBE_INTERVAL + N * PROBE_INTERVAL for
10935 values of N from 1 until it exceeds SIZE. If only one probe is
10936 needed, this will not generate any code. Then adjust and probe
10937 to PROBE_INTERVAL + SIZE. */
10938 for (i = PROBE_INTERVAL; i < size; i += PROBE_INTERVAL)
10942 adjust = 2 * PROBE_INTERVAL + dope;
10943 first_probe = false;
10946 adjust = PROBE_INTERVAL;
10948 emit_insn (gen_rtx_SET (stack_pointer_rtx,
10949 plus_constant (Pmode, stack_pointer_rtx,
10951 emit_stack_probe (stack_pointer_rtx);
10955 adjust = size + PROBE_INTERVAL + dope;
10957 adjust = size + PROBE_INTERVAL - i;
10959 emit_insn (gen_rtx_SET (stack_pointer_rtx,
10960 plus_constant (Pmode, stack_pointer_rtx,
10962 emit_stack_probe (stack_pointer_rtx);
10964 /* Adjust back to account for the additional first interval. */
10965 last = emit_insn (gen_rtx_SET (stack_pointer_rtx,
10966 plus_constant (Pmode, stack_pointer_rtx,
10967 PROBE_INTERVAL + dope)));
10970 /* Otherwise, do the same as above, but in a loop. Note that we must be
10971 extra careful with variables wrapping around because we might be at
10972 the very top (or the very bottom) of the address space and we have
10973 to be able to handle this case properly; in particular, we use an
10974 equality test for the loop condition. */
10977 HOST_WIDE_INT rounded_size;
10978 struct scratch_reg sr;
10980 get_scratch_register_on_entry (&sr);
10983 /* Step 1: round SIZE to the previous multiple of the interval. */
10985 rounded_size = size & -PROBE_INTERVAL;
10988 /* Step 2: compute initial and final value of the loop counter. */
10990 /* SP = SP_0 + PROBE_INTERVAL. */
10991 emit_insn (gen_rtx_SET (stack_pointer_rtx,
10992 plus_constant (Pmode, stack_pointer_rtx,
10993 - (PROBE_INTERVAL + dope))));
10995 /* LAST_ADDR = SP_0 + PROBE_INTERVAL + ROUNDED_SIZE. */
10996 emit_move_insn (sr.reg, GEN_INT (-rounded_size));
10997 emit_insn (gen_rtx_SET (sr.reg,
10998 gen_rtx_PLUS (Pmode, sr.reg,
10999 stack_pointer_rtx)));
11002 /* Step 3: the loop
11004 while (SP != LAST_ADDR)
11006 SP = SP + PROBE_INTERVAL
11010 adjusts SP and probes to PROBE_INTERVAL + N * PROBE_INTERVAL for
11011 values of N from 1 until it is equal to ROUNDED_SIZE. */
11013 emit_insn (ix86_gen_adjust_stack_and_probe (sr.reg, sr.reg, size_rtx));
11016 /* Step 4: adjust SP and probe at PROBE_INTERVAL + SIZE if we cannot
11017 assert at compile-time that SIZE is equal to ROUNDED_SIZE. */
11019 if (size != rounded_size)
11021 emit_insn (gen_rtx_SET (stack_pointer_rtx,
11022 plus_constant (Pmode, stack_pointer_rtx,
11023 rounded_size - size)));
11024 emit_stack_probe (stack_pointer_rtx);
11027 /* Adjust back to account for the additional first interval. */
11028 last = emit_insn (gen_rtx_SET (stack_pointer_rtx,
11029 plus_constant (Pmode, stack_pointer_rtx,
11030 PROBE_INTERVAL + dope)));
11032 release_scratch_register_on_entry (&sr);
11035 gcc_assert (cfun->machine->fs.cfa_reg != stack_pointer_rtx);
11037 /* Even if the stack pointer isn't the CFA register, we need to correctly
11038 describe the adjustments made to it, in particular differentiate the
11039 frame-related ones from the frame-unrelated ones. */
11042 rtx expr = gen_rtx_SEQUENCE (VOIDmode, rtvec_alloc (2));
11043 XVECEXP (expr, 0, 0)
11044 = gen_rtx_SET (stack_pointer_rtx,
11045 plus_constant (Pmode, stack_pointer_rtx, -size));
11046 XVECEXP (expr, 0, 1)
11047 = gen_rtx_SET (stack_pointer_rtx,
11048 plus_constant (Pmode, stack_pointer_rtx,
11049 PROBE_INTERVAL + dope + size));
11050 add_reg_note (last, REG_FRAME_RELATED_EXPR, expr);
11051 RTX_FRAME_RELATED_P (last) = 1;
11053 cfun->machine->fs.sp_offset += size;
11056 /* Make sure nothing is scheduled before we are done. */
11057 emit_insn (gen_blockage ());
11060 /* Adjust the stack pointer up to REG while probing it. */
11063 output_adjust_stack_and_probe (rtx reg)
11065 static int labelno = 0;
11066 char loop_lab[32], end_lab[32];
11069 ASM_GENERATE_INTERNAL_LABEL (loop_lab, "LPSRL", labelno);
11070 ASM_GENERATE_INTERNAL_LABEL (end_lab, "LPSRE", labelno++);
11072 ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, loop_lab);
11074 /* Jump to END_LAB if SP == LAST_ADDR. */
11075 xops[0] = stack_pointer_rtx;
11077 output_asm_insn ("cmp%z0\t{%1, %0|%0, %1}", xops);
11078 fputs ("\tje\t", asm_out_file);
11079 assemble_name_raw (asm_out_file, end_lab);
11080 fputc ('\n', asm_out_file);
11082 /* SP = SP + PROBE_INTERVAL. */
11083 xops[1] = GEN_INT (PROBE_INTERVAL);
11084 output_asm_insn ("sub%z0\t{%1, %0|%0, %1}", xops);
11087 xops[1] = const0_rtx;
11088 output_asm_insn ("or%z0\t{%1, (%0)|DWORD PTR [%0], %1}", xops);
11090 fprintf (asm_out_file, "\tjmp\t");
11091 assemble_name_raw (asm_out_file, loop_lab);
11092 fputc ('\n', asm_out_file);
11094 ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, end_lab);
11099 /* Emit code to probe a range of stack addresses from FIRST to FIRST+SIZE,
11100 inclusive. These are offsets from the current stack pointer. */
11103 ix86_emit_probe_stack_range (HOST_WIDE_INT first, HOST_WIDE_INT size)
11105 /* See if we have a constant small number of probes to generate. If so,
11106 that's the easy case. The run-time loop is made up of 7 insns in the
11107 generic case while the compile-time loop is made up of n insns for n #
11109 if (size <= 7 * PROBE_INTERVAL)
11113 /* Probe at FIRST + N * PROBE_INTERVAL for values of N from 1 until
11114 it exceeds SIZE. If only one probe is needed, this will not
11115 generate any code. Then probe at FIRST + SIZE. */
11116 for (i = PROBE_INTERVAL; i < size; i += PROBE_INTERVAL)
11117 emit_stack_probe (plus_constant (Pmode, stack_pointer_rtx,
11120 emit_stack_probe (plus_constant (Pmode, stack_pointer_rtx,
11124 /* Otherwise, do the same as above, but in a loop. Note that we must be
11125 extra careful with variables wrapping around because we might be at
11126 the very top (or the very bottom) of the address space and we have
11127 to be able to handle this case properly; in particular, we use an
11128 equality test for the loop condition. */
11131 HOST_WIDE_INT rounded_size, last;
11132 struct scratch_reg sr;
11134 get_scratch_register_on_entry (&sr);
11137 /* Step 1: round SIZE to the previous multiple of the interval. */
11139 rounded_size = size & -PROBE_INTERVAL;
11142 /* Step 2: compute initial and final value of the loop counter. */
11144 /* TEST_OFFSET = FIRST. */
11145 emit_move_insn (sr.reg, GEN_INT (-first));
11147 /* LAST_OFFSET = FIRST + ROUNDED_SIZE. */
11148 last = first + rounded_size;
11151 /* Step 3: the loop
11153 while (TEST_ADDR != LAST_ADDR)
11155 TEST_ADDR = TEST_ADDR + PROBE_INTERVAL
11159 probes at FIRST + N * PROBE_INTERVAL for values of N from 1
11160 until it is equal to ROUNDED_SIZE. */
11162 emit_insn (ix86_gen_probe_stack_range (sr.reg, sr.reg, GEN_INT (-last)));
11165 /* Step 4: probe at FIRST + SIZE if we cannot assert at compile-time
11166 that SIZE is equal to ROUNDED_SIZE. */
11168 if (size != rounded_size)
11169 emit_stack_probe (plus_constant (Pmode,
11170 gen_rtx_PLUS (Pmode,
11173 rounded_size - size));
11175 release_scratch_register_on_entry (&sr);
11178 /* Make sure nothing is scheduled before we are done. */
11179 emit_insn (gen_blockage ());
11182 /* Probe a range of stack addresses from REG to END, inclusive. These are
11183 offsets from the current stack pointer. */
11186 output_probe_stack_range (rtx reg, rtx end)
11188 static int labelno = 0;
11189 char loop_lab[32], end_lab[32];
11192 ASM_GENERATE_INTERNAL_LABEL (loop_lab, "LPSRL", labelno);
11193 ASM_GENERATE_INTERNAL_LABEL (end_lab, "LPSRE", labelno++);
11195 ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, loop_lab);
11197 /* Jump to END_LAB if TEST_ADDR == LAST_ADDR. */
11200 output_asm_insn ("cmp%z0\t{%1, %0|%0, %1}", xops);
11201 fputs ("\tje\t", asm_out_file);
11202 assemble_name_raw (asm_out_file, end_lab);
11203 fputc ('\n', asm_out_file);
11205 /* TEST_ADDR = TEST_ADDR + PROBE_INTERVAL. */
11206 xops[1] = GEN_INT (PROBE_INTERVAL);
11207 output_asm_insn ("sub%z0\t{%1, %0|%0, %1}", xops);
11209 /* Probe at TEST_ADDR. */
11210 xops[0] = stack_pointer_rtx;
11212 xops[2] = const0_rtx;
11213 output_asm_insn ("or%z0\t{%2, (%0,%1)|DWORD PTR [%0+%1], %2}", xops);
11215 fprintf (asm_out_file, "\tjmp\t");
11216 assemble_name_raw (asm_out_file, loop_lab);
11217 fputc ('\n', asm_out_file);
11219 ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, end_lab);
11224 /* Finalize stack_realign_needed flag, which will guide prologue/epilogue
11225 to be generated in correct form. */
11227 ix86_finalize_stack_realign_flags (void)
11229 /* Check if stack realign is really needed after reload, and
11230 stores result in cfun */
11231 unsigned int incoming_stack_boundary
11232 = (crtl->parm_stack_boundary > ix86_incoming_stack_boundary
11233 ? crtl->parm_stack_boundary : ix86_incoming_stack_boundary);
11234 unsigned int stack_realign = (incoming_stack_boundary
11236 ? crtl->max_used_stack_slot_alignment
11237 : crtl->stack_alignment_needed));
11239 if (crtl->stack_realign_finalized)
11241 /* After stack_realign_needed is finalized, we can't no longer
11243 gcc_assert (crtl->stack_realign_needed == stack_realign);
11247 /* If the only reason for frame_pointer_needed is that we conservatively
11248 assumed stack realignment might be needed, but in the end nothing that
11249 needed the stack alignment had been spilled, clear frame_pointer_needed
11250 and say we don't need stack realignment. */
11252 && frame_pointer_needed
11254 && flag_omit_frame_pointer
11255 && crtl->sp_is_unchanging
11256 && !ix86_current_function_calls_tls_descriptor
11257 && !crtl->accesses_prior_frames
11258 && !cfun->calls_alloca
11259 && !crtl->calls_eh_return
11260 && !(flag_stack_check && STACK_CHECK_MOVING_SP)
11261 && !ix86_frame_pointer_required ()
11262 && get_frame_size () == 0
11263 && ix86_nsaved_sseregs () == 0
11264 && ix86_varargs_gpr_size + ix86_varargs_fpr_size == 0)
11266 HARD_REG_SET set_up_by_prologue, prologue_used;
11269 CLEAR_HARD_REG_SET (prologue_used);
11270 CLEAR_HARD_REG_SET (set_up_by_prologue);
11271 add_to_hard_reg_set (&set_up_by_prologue, Pmode, STACK_POINTER_REGNUM);
11272 add_to_hard_reg_set (&set_up_by_prologue, Pmode, ARG_POINTER_REGNUM);
11273 add_to_hard_reg_set (&set_up_by_prologue, Pmode,
11274 HARD_FRAME_POINTER_REGNUM);
11275 FOR_EACH_BB_FN (bb, cfun)
11278 FOR_BB_INSNS (bb, insn)
11279 if (NONDEBUG_INSN_P (insn)
11280 && requires_stack_frame_p (insn, prologue_used,
11281 set_up_by_prologue))
11283 crtl->stack_realign_needed = stack_realign;
11284 crtl->stack_realign_finalized = true;
11289 /* If drap has been set, but it actually isn't live at the start
11290 of the function, there is no reason to set it up. */
11291 if (crtl->drap_reg)
11293 basic_block bb = ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb;
11294 if (! REGNO_REG_SET_P (DF_LR_IN (bb), REGNO (crtl->drap_reg)))
11296 crtl->drap_reg = NULL_RTX;
11297 crtl->need_drap = false;
11301 cfun->machine->no_drap_save_restore = true;
11303 frame_pointer_needed = false;
11304 stack_realign = false;
11305 crtl->max_used_stack_slot_alignment = incoming_stack_boundary;
11306 crtl->stack_alignment_needed = incoming_stack_boundary;
11307 crtl->stack_alignment_estimated = incoming_stack_boundary;
11308 if (crtl->preferred_stack_boundary > incoming_stack_boundary)
11309 crtl->preferred_stack_boundary = incoming_stack_boundary;
11310 df_finish_pass (true);
11311 df_scan_alloc (NULL);
11313 df_compute_regs_ever_live (true);
11317 crtl->stack_realign_needed = stack_realign;
11318 crtl->stack_realign_finalized = true;
11321 /* Delete SET_GOT right after entry block if it is allocated to reg. */
11324 ix86_elim_entry_set_got (rtx reg)
11326 basic_block bb = ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb;
11327 rtx_insn *c_insn = BB_HEAD (bb);
11328 if (!NONDEBUG_INSN_P (c_insn))
11329 c_insn = next_nonnote_nondebug_insn (c_insn);
11330 if (c_insn && NONJUMP_INSN_P (c_insn))
11332 rtx pat = PATTERN (c_insn);
11333 if (GET_CODE (pat) == PARALLEL)
11335 rtx vec = XVECEXP (pat, 0, 0);
11336 if (GET_CODE (vec) == SET
11337 && XINT (XEXP (vec, 1), 1) == UNSPEC_SET_GOT
11338 && REGNO (XEXP (vec, 0)) == REGNO (reg))
11339 delete_insn (c_insn);
11344 /* Expand the prologue into a bunch of separate insns. */
11347 ix86_expand_prologue (void)
11349 struct machine_function *m = cfun->machine;
11351 struct ix86_frame frame;
11352 HOST_WIDE_INT allocate;
11353 bool int_registers_saved;
11354 bool sse_registers_saved;
11356 ix86_finalize_stack_realign_flags ();
11358 /* DRAP should not coexist with stack_realign_fp */
11359 gcc_assert (!(crtl->drap_reg && stack_realign_fp));
11361 memset (&m->fs, 0, sizeof (m->fs));
11363 /* Initialize CFA state for before the prologue. */
11364 m->fs.cfa_reg = stack_pointer_rtx;
11365 m->fs.cfa_offset = INCOMING_FRAME_SP_OFFSET;
11367 /* Track SP offset to the CFA. We continue tracking this after we've
11368 swapped the CFA register away from SP. In the case of re-alignment
11369 this is fudged; we're interested to offsets within the local frame. */
11370 m->fs.sp_offset = INCOMING_FRAME_SP_OFFSET;
11371 m->fs.sp_valid = true;
11373 ix86_compute_frame_layout (&frame);
11375 if (!TARGET_64BIT && ix86_function_ms_hook_prologue (current_function_decl))
11377 /* We should have already generated an error for any use of
11378 ms_hook on a nested function. */
11379 gcc_checking_assert (!ix86_static_chain_on_stack);
11381 /* Check if profiling is active and we shall use profiling before
11382 prologue variant. If so sorry. */
11383 if (crtl->profile && flag_fentry != 0)
11384 sorry ("ms_hook_prologue attribute isn%'t compatible "
11385 "with -mfentry for 32-bit");
11387 /* In ix86_asm_output_function_label we emitted:
11388 8b ff movl.s %edi,%edi
11390 8b ec movl.s %esp,%ebp
11392 This matches the hookable function prologue in Win32 API
11393 functions in Microsoft Windows XP Service Pack 2 and newer.
11394 Wine uses this to enable Windows apps to hook the Win32 API
11395 functions provided by Wine.
11397 What that means is that we've already set up the frame pointer. */
11399 if (frame_pointer_needed
11400 && !(crtl->drap_reg && crtl->stack_realign_needed))
11404 /* We've decided to use the frame pointer already set up.
11405 Describe this to the unwinder by pretending that both
11406 push and mov insns happen right here.
11408 Putting the unwind info here at the end of the ms_hook
11409 is done so that we can make absolutely certain we get
11410 the required byte sequence at the start of the function,
11411 rather than relying on an assembler that can produce
11412 the exact encoding required.
11414 However it does mean (in the unpatched case) that we have
11415 a 1 insn window where the asynchronous unwind info is
11416 incorrect. However, if we placed the unwind info at
11417 its correct location we would have incorrect unwind info
11418 in the patched case. Which is probably all moot since
11419 I don't expect Wine generates dwarf2 unwind info for the
11420 system libraries that use this feature. */
11422 insn = emit_insn (gen_blockage ());
11424 push = gen_push (hard_frame_pointer_rtx);
11425 mov = gen_rtx_SET (hard_frame_pointer_rtx,
11426 stack_pointer_rtx);
11427 RTX_FRAME_RELATED_P (push) = 1;
11428 RTX_FRAME_RELATED_P (mov) = 1;
11430 RTX_FRAME_RELATED_P (insn) = 1;
11431 add_reg_note (insn, REG_FRAME_RELATED_EXPR,
11432 gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, push, mov)));
11434 /* Note that gen_push incremented m->fs.cfa_offset, even
11435 though we didn't emit the push insn here. */
11436 m->fs.cfa_reg = hard_frame_pointer_rtx;
11437 m->fs.fp_offset = m->fs.cfa_offset;
11438 m->fs.fp_valid = true;
11442 /* The frame pointer is not needed so pop %ebp again.
11443 This leaves us with a pristine state. */
11444 emit_insn (gen_pop (hard_frame_pointer_rtx));
11448 /* The first insn of a function that accepts its static chain on the
11449 stack is to push the register that would be filled in by a direct
11450 call. This insn will be skipped by the trampoline. */
11451 else if (ix86_static_chain_on_stack)
11453 insn = emit_insn (gen_push (ix86_static_chain (cfun->decl, false)));
11454 emit_insn (gen_blockage ());
11456 /* We don't want to interpret this push insn as a register save,
11457 only as a stack adjustment. The real copy of the register as
11458 a save will be done later, if needed. */
11459 t = plus_constant (Pmode, stack_pointer_rtx, -UNITS_PER_WORD);
11460 t = gen_rtx_SET (stack_pointer_rtx, t);
11461 add_reg_note (insn, REG_CFA_ADJUST_CFA, t);
11462 RTX_FRAME_RELATED_P (insn) = 1;
11465 /* Emit prologue code to adjust stack alignment and setup DRAP, in case
11466 of DRAP is needed and stack realignment is really needed after reload */
11467 if (stack_realign_drap)
11469 int align_bytes = crtl->stack_alignment_needed / BITS_PER_UNIT;
11471 /* Only need to push parameter pointer reg if it is caller saved. */
11472 if (!call_used_regs[REGNO (crtl->drap_reg)])
11474 /* Push arg pointer reg */
11475 insn = emit_insn (gen_push (crtl->drap_reg));
11476 RTX_FRAME_RELATED_P (insn) = 1;
11479 /* Grab the argument pointer. */
11480 t = plus_constant (Pmode, stack_pointer_rtx, m->fs.sp_offset);
11481 insn = emit_insn (gen_rtx_SET (crtl->drap_reg, t));
11482 RTX_FRAME_RELATED_P (insn) = 1;
11483 m->fs.cfa_reg = crtl->drap_reg;
11484 m->fs.cfa_offset = 0;
11486 /* Align the stack. */
11487 insn = emit_insn (ix86_gen_andsp (stack_pointer_rtx,
11489 GEN_INT (-align_bytes)));
11490 RTX_FRAME_RELATED_P (insn) = 1;
11492 /* Replicate the return address on the stack so that return
11493 address can be reached via (argp - 1) slot. This is needed
11494 to implement macro RETURN_ADDR_RTX and intrinsic function
11495 expand_builtin_return_addr etc. */
11496 t = plus_constant (Pmode, crtl->drap_reg, -UNITS_PER_WORD);
11497 t = gen_frame_mem (word_mode, t);
11498 insn = emit_insn (gen_push (t));
11499 RTX_FRAME_RELATED_P (insn) = 1;
11501 /* For the purposes of frame and register save area addressing,
11502 we've started over with a new frame. */
11503 m->fs.sp_offset = INCOMING_FRAME_SP_OFFSET;
11504 m->fs.realigned = true;
11507 int_registers_saved = (frame.nregs == 0);
11508 sse_registers_saved = (frame.nsseregs == 0);
11510 if (frame_pointer_needed && !m->fs.fp_valid)
11512 /* Note: AT&T enter does NOT have reversed args. Enter is probably
11513 slower on all targets. Also sdb doesn't like it. */
11514 insn = emit_insn (gen_push (hard_frame_pointer_rtx));
11515 RTX_FRAME_RELATED_P (insn) = 1;
11517 /* Push registers now, before setting the frame pointer
11519 if (!int_registers_saved
11521 && !frame.save_regs_using_mov)
11523 ix86_emit_save_regs ();
11524 int_registers_saved = true;
11525 gcc_assert (m->fs.sp_offset == frame.reg_save_offset);
11528 if (m->fs.sp_offset == frame.hard_frame_pointer_offset)
11530 insn = emit_move_insn (hard_frame_pointer_rtx, stack_pointer_rtx);
11531 RTX_FRAME_RELATED_P (insn) = 1;
11533 if (m->fs.cfa_reg == stack_pointer_rtx)
11534 m->fs.cfa_reg = hard_frame_pointer_rtx;
11535 m->fs.fp_offset = m->fs.sp_offset;
11536 m->fs.fp_valid = true;
11540 if (!int_registers_saved)
11542 /* If saving registers via PUSH, do so now. */
11543 if (!frame.save_regs_using_mov)
11545 ix86_emit_save_regs ();
11546 int_registers_saved = true;
11547 gcc_assert (m->fs.sp_offset == frame.reg_save_offset);
11550 /* When using red zone we may start register saving before allocating
11551 the stack frame saving one cycle of the prologue. However, avoid
11552 doing this if we have to probe the stack; at least on x86_64 the
11553 stack probe can turn into a call that clobbers a red zone location. */
11554 else if (ix86_using_red_zone ()
11555 && (! TARGET_STACK_PROBE
11556 || frame.stack_pointer_offset < CHECK_STACK_LIMIT))
11558 ix86_emit_save_regs_using_mov (frame.reg_save_offset);
11559 int_registers_saved = true;
11563 if (stack_realign_fp)
11565 int align_bytes = crtl->stack_alignment_needed / BITS_PER_UNIT;
11566 gcc_assert (align_bytes > MIN_STACK_BOUNDARY / BITS_PER_UNIT);
11568 /* The computation of the size of the re-aligned stack frame means
11569 that we must allocate the size of the register save area before
11570 performing the actual alignment. Otherwise we cannot guarantee
11571 that there's enough storage above the realignment point. */
11572 if (m->fs.sp_offset != frame.sse_reg_save_offset)
11573 pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
11574 GEN_INT (m->fs.sp_offset
11575 - frame.sse_reg_save_offset),
11578 /* Align the stack. */
11579 insn = emit_insn (ix86_gen_andsp (stack_pointer_rtx,
11581 GEN_INT (-align_bytes)));
11583 /* For the purposes of register save area addressing, the stack
11584 pointer is no longer valid. As for the value of sp_offset,
11585 see ix86_compute_frame_layout, which we need to match in order
11586 to pass verification of stack_pointer_offset at the end. */
11587 m->fs.sp_offset = (m->fs.sp_offset + align_bytes) & -align_bytes;
11588 m->fs.sp_valid = false;
11591 allocate = frame.stack_pointer_offset - m->fs.sp_offset;
11593 if (flag_stack_usage_info)
11595 /* We start to count from ARG_POINTER. */
11596 HOST_WIDE_INT stack_size = frame.stack_pointer_offset;
11598 /* If it was realigned, take into account the fake frame. */
11599 if (stack_realign_drap)
11601 if (ix86_static_chain_on_stack)
11602 stack_size += UNITS_PER_WORD;
11604 if (!call_used_regs[REGNO (crtl->drap_reg)])
11605 stack_size += UNITS_PER_WORD;
11607 /* This over-estimates by 1 minimal-stack-alignment-unit but
11608 mitigates that by counting in the new return address slot. */
11609 current_function_dynamic_stack_size
11610 += crtl->stack_alignment_needed / BITS_PER_UNIT;
11613 current_function_static_stack_size = stack_size;
11616 /* On SEH target with very large frame size, allocate an area to save
11617 SSE registers (as the very large allocation won't be described). */
11619 && frame.stack_pointer_offset > SEH_MAX_FRAME_SIZE
11620 && !sse_registers_saved)
11622 HOST_WIDE_INT sse_size =
11623 frame.sse_reg_save_offset - frame.reg_save_offset;
11625 gcc_assert (int_registers_saved);
11627 /* No need to do stack checking as the area will be immediately
11629 pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
11630 GEN_INT (-sse_size), -1,
11631 m->fs.cfa_reg == stack_pointer_rtx);
11632 allocate -= sse_size;
11633 ix86_emit_save_sse_regs_using_mov (frame.sse_reg_save_offset);
11634 sse_registers_saved = true;
11637 /* The stack has already been decremented by the instruction calling us
11638 so probe if the size is non-negative to preserve the protection area. */
11639 if (allocate >= 0 && flag_stack_check == STATIC_BUILTIN_STACK_CHECK)
11641 /* We expect the registers to be saved when probes are used. */
11642 gcc_assert (int_registers_saved);
11644 if (STACK_CHECK_MOVING_SP)
11646 if (!(crtl->is_leaf && !cfun->calls_alloca
11647 && allocate <= PROBE_INTERVAL))
11649 ix86_adjust_stack_and_probe (allocate);
11655 HOST_WIDE_INT size = allocate;
11657 if (TARGET_64BIT && size >= (HOST_WIDE_INT) 0x80000000)
11658 size = 0x80000000 - STACK_CHECK_PROTECT - 1;
11660 if (TARGET_STACK_PROBE)
11662 if (crtl->is_leaf && !cfun->calls_alloca)
11664 if (size > PROBE_INTERVAL)
11665 ix86_emit_probe_stack_range (0, size);
11668 ix86_emit_probe_stack_range (0, size + STACK_CHECK_PROTECT);
11672 if (crtl->is_leaf && !cfun->calls_alloca)
11674 if (size > PROBE_INTERVAL && size > STACK_CHECK_PROTECT)
11675 ix86_emit_probe_stack_range (STACK_CHECK_PROTECT,
11676 size - STACK_CHECK_PROTECT);
11679 ix86_emit_probe_stack_range (STACK_CHECK_PROTECT, size);
11686 else if (!ix86_target_stack_probe ()
11687 || frame.stack_pointer_offset < CHECK_STACK_LIMIT)
11689 pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
11690 GEN_INT (-allocate), -1,
11691 m->fs.cfa_reg == stack_pointer_rtx);
11695 rtx eax = gen_rtx_REG (Pmode, AX_REG);
11697 rtx (*adjust_stack_insn)(rtx, rtx, rtx);
11698 const bool sp_is_cfa_reg = (m->fs.cfa_reg == stack_pointer_rtx);
11699 bool eax_live = ix86_eax_live_at_start_p ();
11700 bool r10_live = false;
11703 r10_live = (DECL_STATIC_CHAIN (current_function_decl) != 0);
11707 insn = emit_insn (gen_push (eax));
11708 allocate -= UNITS_PER_WORD;
11709 /* Note that SEH directives need to continue tracking the stack
11710 pointer even after the frame pointer has been set up. */
11711 if (sp_is_cfa_reg || TARGET_SEH)
11714 m->fs.cfa_offset += UNITS_PER_WORD;
11715 RTX_FRAME_RELATED_P (insn) = 1;
11716 add_reg_note (insn, REG_FRAME_RELATED_EXPR,
11717 gen_rtx_SET (stack_pointer_rtx,
11718 plus_constant (Pmode, stack_pointer_rtx,
11719 -UNITS_PER_WORD)));
11725 r10 = gen_rtx_REG (Pmode, R10_REG);
11726 insn = emit_insn (gen_push (r10));
11727 allocate -= UNITS_PER_WORD;
11728 if (sp_is_cfa_reg || TARGET_SEH)
11731 m->fs.cfa_offset += UNITS_PER_WORD;
11732 RTX_FRAME_RELATED_P (insn) = 1;
11733 add_reg_note (insn, REG_FRAME_RELATED_EXPR,
11734 gen_rtx_SET (stack_pointer_rtx,
11735 plus_constant (Pmode, stack_pointer_rtx,
11736 -UNITS_PER_WORD)));
11740 emit_move_insn (eax, GEN_INT (allocate));
11741 emit_insn (ix86_gen_allocate_stack_worker (eax, eax));
11743 /* Use the fact that AX still contains ALLOCATE. */
11744 adjust_stack_insn = (Pmode == DImode
11745 ? gen_pro_epilogue_adjust_stack_di_sub
11746 : gen_pro_epilogue_adjust_stack_si_sub);
11748 insn = emit_insn (adjust_stack_insn (stack_pointer_rtx,
11749 stack_pointer_rtx, eax));
11751 if (sp_is_cfa_reg || TARGET_SEH)
11754 m->fs.cfa_offset += allocate;
11755 RTX_FRAME_RELATED_P (insn) = 1;
11756 add_reg_note (insn, REG_FRAME_RELATED_EXPR,
11757 gen_rtx_SET (stack_pointer_rtx,
11758 plus_constant (Pmode, stack_pointer_rtx,
11761 m->fs.sp_offset += allocate;
11763 /* Use stack_pointer_rtx for relative addressing so that code
11764 works for realigned stack, too. */
11765 if (r10_live && eax_live)
11767 t = gen_rtx_PLUS (Pmode, stack_pointer_rtx, eax);
11768 emit_move_insn (gen_rtx_REG (word_mode, R10_REG),
11769 gen_frame_mem (word_mode, t));
11770 t = plus_constant (Pmode, t, UNITS_PER_WORD);
11771 emit_move_insn (gen_rtx_REG (word_mode, AX_REG),
11772 gen_frame_mem (word_mode, t));
11774 else if (eax_live || r10_live)
11776 t = gen_rtx_PLUS (Pmode, stack_pointer_rtx, eax);
11777 emit_move_insn (gen_rtx_REG (word_mode,
11778 (eax_live ? AX_REG : R10_REG)),
11779 gen_frame_mem (word_mode, t));
11782 gcc_assert (m->fs.sp_offset == frame.stack_pointer_offset);
11784 /* If we havn't already set up the frame pointer, do so now. */
11785 if (frame_pointer_needed && !m->fs.fp_valid)
11787 insn = ix86_gen_add3 (hard_frame_pointer_rtx, stack_pointer_rtx,
11788 GEN_INT (frame.stack_pointer_offset
11789 - frame.hard_frame_pointer_offset));
11790 insn = emit_insn (insn);
11791 RTX_FRAME_RELATED_P (insn) = 1;
11792 add_reg_note (insn, REG_CFA_ADJUST_CFA, NULL);
11794 if (m->fs.cfa_reg == stack_pointer_rtx)
11795 m->fs.cfa_reg = hard_frame_pointer_rtx;
11796 m->fs.fp_offset = frame.hard_frame_pointer_offset;
11797 m->fs.fp_valid = true;
11800 if (!int_registers_saved)
11801 ix86_emit_save_regs_using_mov (frame.reg_save_offset);
11802 if (!sse_registers_saved)
11803 ix86_emit_save_sse_regs_using_mov (frame.sse_reg_save_offset);
11805 /* For the mcount profiling on 32 bit PIC mode we need to emit SET_GOT
11807 if (!TARGET_64BIT && pic_offset_table_rtx && crtl->profile && !flag_fentry)
11809 rtx pic = gen_rtx_REG (Pmode, REAL_PIC_OFFSET_TABLE_REGNUM);
11810 insn = emit_insn (gen_set_got (pic));
11811 RTX_FRAME_RELATED_P (insn) = 1;
11812 add_reg_note (insn, REG_CFA_FLUSH_QUEUE, NULL_RTX);
11813 emit_insn (gen_prologue_use (pic));
11814 /* Deleting already emmitted SET_GOT if exist and allocated to
11815 REAL_PIC_OFFSET_TABLE_REGNUM. */
11816 ix86_elim_entry_set_got (pic);
11819 if (crtl->drap_reg && !crtl->stack_realign_needed)
11821 /* vDRAP is setup but after reload it turns out stack realign
11822 isn't necessary, here we will emit prologue to setup DRAP
11823 without stack realign adjustment */
11824 t = choose_baseaddr (0);
11825 emit_insn (gen_rtx_SET (crtl->drap_reg, t));
11828 /* Prevent instructions from being scheduled into register save push
11829 sequence when access to the redzone area is done through frame pointer.
11830 The offset between the frame pointer and the stack pointer is calculated
11831 relative to the value of the stack pointer at the end of the function
11832 prologue, and moving instructions that access redzone area via frame
11833 pointer inside push sequence violates this assumption. */
11834 if (frame_pointer_needed && frame.red_zone_size)
11835 emit_insn (gen_memory_blockage ());
11837 /* Emit cld instruction if stringops are used in the function. */
11838 if (TARGET_CLD && ix86_current_function_needs_cld)
11839 emit_insn (gen_cld ());
11841 /* SEH requires that the prologue end within 256 bytes of the start of
11842 the function. Prevent instruction schedules that would extend that.
11843 Further, prevent alloca modifications to the stack pointer from being
11844 combined with prologue modifications. */
11846 emit_insn (gen_prologue_use (stack_pointer_rtx));
11849 /* Emit code to restore REG using a POP insn. */
11852 ix86_emit_restore_reg_using_pop (rtx reg)
11854 struct machine_function *m = cfun->machine;
11855 rtx_insn *insn = emit_insn (gen_pop (reg));
11857 ix86_add_cfa_restore_note (insn, reg, m->fs.sp_offset);
11858 m->fs.sp_offset -= UNITS_PER_WORD;
11860 if (m->fs.cfa_reg == crtl->drap_reg
11861 && REGNO (reg) == REGNO (crtl->drap_reg))
11863 /* Previously we'd represented the CFA as an expression
11864 like *(%ebp - 8). We've just popped that value from
11865 the stack, which means we need to reset the CFA to
11866 the drap register. This will remain until we restore
11867 the stack pointer. */
11868 add_reg_note (insn, REG_CFA_DEF_CFA, reg);
11869 RTX_FRAME_RELATED_P (insn) = 1;
11871 /* This means that the DRAP register is valid for addressing too. */
11872 m->fs.drap_valid = true;
11876 if (m->fs.cfa_reg == stack_pointer_rtx)
11878 rtx x = plus_constant (Pmode, stack_pointer_rtx, UNITS_PER_WORD);
11879 x = gen_rtx_SET (stack_pointer_rtx, x);
11880 add_reg_note (insn, REG_CFA_ADJUST_CFA, x);
11881 RTX_FRAME_RELATED_P (insn) = 1;
11883 m->fs.cfa_offset -= UNITS_PER_WORD;
11886 /* When the frame pointer is the CFA, and we pop it, we are
11887 swapping back to the stack pointer as the CFA. This happens
11888 for stack frames that don't allocate other data, so we assume
11889 the stack pointer is now pointing at the return address, i.e.
11890 the function entry state, which makes the offset be 1 word. */
11891 if (reg == hard_frame_pointer_rtx)
11893 m->fs.fp_valid = false;
11894 if (m->fs.cfa_reg == hard_frame_pointer_rtx)
11896 m->fs.cfa_reg = stack_pointer_rtx;
11897 m->fs.cfa_offset -= UNITS_PER_WORD;
11899 add_reg_note (insn, REG_CFA_DEF_CFA,
11900 gen_rtx_PLUS (Pmode, stack_pointer_rtx,
11901 GEN_INT (m->fs.cfa_offset)));
11902 RTX_FRAME_RELATED_P (insn) = 1;
11907 /* Emit code to restore saved registers using POP insns. */
11910 ix86_emit_restore_regs_using_pop (void)
11912 unsigned int regno;
11914 for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
11915 if (!SSE_REGNO_P (regno) && ix86_save_reg (regno, false))
11916 ix86_emit_restore_reg_using_pop (gen_rtx_REG (word_mode, regno));
11919 /* Emit code and notes for the LEAVE instruction. */
11922 ix86_emit_leave (void)
11924 struct machine_function *m = cfun->machine;
11925 rtx_insn *insn = emit_insn (ix86_gen_leave ());
11927 ix86_add_queued_cfa_restore_notes (insn);
11929 gcc_assert (m->fs.fp_valid);
11930 m->fs.sp_valid = true;
11931 m->fs.sp_offset = m->fs.fp_offset - UNITS_PER_WORD;
11932 m->fs.fp_valid = false;
11934 if (m->fs.cfa_reg == hard_frame_pointer_rtx)
11936 m->fs.cfa_reg = stack_pointer_rtx;
11937 m->fs.cfa_offset = m->fs.sp_offset;
11939 add_reg_note (insn, REG_CFA_DEF_CFA,
11940 plus_constant (Pmode, stack_pointer_rtx,
11942 RTX_FRAME_RELATED_P (insn) = 1;
11944 ix86_add_cfa_restore_note (insn, hard_frame_pointer_rtx,
11948 /* Emit code to restore saved registers using MOV insns.
11949 First register is restored from CFA - CFA_OFFSET. */
11951 ix86_emit_restore_regs_using_mov (HOST_WIDE_INT cfa_offset,
11952 bool maybe_eh_return)
11954 struct machine_function *m = cfun->machine;
11955 unsigned int regno;
11957 for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
11958 if (!SSE_REGNO_P (regno) && ix86_save_reg (regno, maybe_eh_return))
11960 rtx reg = gen_rtx_REG (word_mode, regno);
11964 mem = choose_baseaddr (cfa_offset);
11965 mem = gen_frame_mem (word_mode, mem);
11966 insn = emit_move_insn (reg, mem);
11968 if (m->fs.cfa_reg == crtl->drap_reg && regno == REGNO (crtl->drap_reg))
11970 /* Previously we'd represented the CFA as an expression
11971 like *(%ebp - 8). We've just popped that value from
11972 the stack, which means we need to reset the CFA to
11973 the drap register. This will remain until we restore
11974 the stack pointer. */
11975 add_reg_note (insn, REG_CFA_DEF_CFA, reg);
11976 RTX_FRAME_RELATED_P (insn) = 1;
11978 /* This means that the DRAP register is valid for addressing. */
11979 m->fs.drap_valid = true;
11982 ix86_add_cfa_restore_note (NULL, reg, cfa_offset);
11984 cfa_offset -= UNITS_PER_WORD;
11988 /* Emit code to restore saved registers using MOV insns.
11989 First register is restored from CFA - CFA_OFFSET. */
11991 ix86_emit_restore_sse_regs_using_mov (HOST_WIDE_INT cfa_offset,
11992 bool maybe_eh_return)
11994 unsigned int regno;
11996 for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
11997 if (SSE_REGNO_P (regno) && ix86_save_reg (regno, maybe_eh_return))
11999 rtx reg = gen_rtx_REG (V4SFmode, regno);
12002 mem = choose_baseaddr (cfa_offset);
12003 mem = gen_rtx_MEM (V4SFmode, mem);
12004 set_mem_align (mem, 128);
12005 emit_move_insn (reg, mem);
12007 ix86_add_cfa_restore_note (NULL, reg, cfa_offset);
12013 /* Restore function stack, frame, and registers. */
12016 ix86_expand_epilogue (int style)
12018 struct machine_function *m = cfun->machine;
12019 struct machine_frame_state frame_state_save = m->fs;
12020 struct ix86_frame frame;
12021 bool restore_regs_via_mov;
12024 ix86_finalize_stack_realign_flags ();
12025 ix86_compute_frame_layout (&frame);
12027 m->fs.sp_valid = (!frame_pointer_needed
12028 || (crtl->sp_is_unchanging
12029 && !stack_realign_fp));
12030 gcc_assert (!m->fs.sp_valid
12031 || m->fs.sp_offset == frame.stack_pointer_offset);
12033 /* The FP must be valid if the frame pointer is present. */
12034 gcc_assert (frame_pointer_needed == m->fs.fp_valid);
12035 gcc_assert (!m->fs.fp_valid
12036 || m->fs.fp_offset == frame.hard_frame_pointer_offset);
12038 /* We must have *some* valid pointer to the stack frame. */
12039 gcc_assert (m->fs.sp_valid || m->fs.fp_valid);
12041 /* The DRAP is never valid at this point. */
12042 gcc_assert (!m->fs.drap_valid);
12044 /* See the comment about red zone and frame
12045 pointer usage in ix86_expand_prologue. */
12046 if (frame_pointer_needed && frame.red_zone_size)
12047 emit_insn (gen_memory_blockage ());
12049 using_drap = crtl->drap_reg && crtl->stack_realign_needed;
12050 gcc_assert (!using_drap || m->fs.cfa_reg == crtl->drap_reg);
12052 /* Determine the CFA offset of the end of the red-zone. */
12053 m->fs.red_zone_offset = 0;
12054 if (ix86_using_red_zone () && crtl->args.pops_args < 65536)
12056 /* The red-zone begins below the return address. */
12057 m->fs.red_zone_offset = RED_ZONE_SIZE + UNITS_PER_WORD;
12059 /* When the register save area is in the aligned portion of
12060 the stack, determine the maximum runtime displacement that
12061 matches up with the aligned frame. */
12062 if (stack_realign_drap)
12063 m->fs.red_zone_offset -= (crtl->stack_alignment_needed / BITS_PER_UNIT
12067 /* Special care must be taken for the normal return case of a function
12068 using eh_return: the eax and edx registers are marked as saved, but
12069 not restored along this path. Adjust the save location to match. */
12070 if (crtl->calls_eh_return && style != 2)
12071 frame.reg_save_offset -= 2 * UNITS_PER_WORD;
12073 /* EH_RETURN requires the use of moves to function properly. */
12074 if (crtl->calls_eh_return)
12075 restore_regs_via_mov = true;
12076 /* SEH requires the use of pops to identify the epilogue. */
12077 else if (TARGET_SEH)
12078 restore_regs_via_mov = false;
12079 /* If we're only restoring one register and sp is not valid then
12080 using a move instruction to restore the register since it's
12081 less work than reloading sp and popping the register. */
12082 else if (!m->fs.sp_valid && frame.nregs <= 1)
12083 restore_regs_via_mov = true;
12084 else if (TARGET_EPILOGUE_USING_MOVE
12085 && cfun->machine->use_fast_prologue_epilogue
12086 && (frame.nregs > 1
12087 || m->fs.sp_offset != frame.reg_save_offset))
12088 restore_regs_via_mov = true;
12089 else if (frame_pointer_needed
12091 && m->fs.sp_offset != frame.reg_save_offset)
12092 restore_regs_via_mov = true;
12093 else if (frame_pointer_needed
12094 && TARGET_USE_LEAVE
12095 && cfun->machine->use_fast_prologue_epilogue
12096 && frame.nregs == 1)
12097 restore_regs_via_mov = true;
12099 restore_regs_via_mov = false;
12101 if (restore_regs_via_mov || frame.nsseregs)
12103 /* Ensure that the entire register save area is addressable via
12104 the stack pointer, if we will restore via sp. */
12106 && m->fs.sp_offset > 0x7fffffff
12107 && !(m->fs.fp_valid || m->fs.drap_valid)
12108 && (frame.nsseregs + frame.nregs) != 0)
12110 pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
12111 GEN_INT (m->fs.sp_offset
12112 - frame.sse_reg_save_offset),
12114 m->fs.cfa_reg == stack_pointer_rtx);
12118 /* If there are any SSE registers to restore, then we have to do it
12119 via moves, since there's obviously no pop for SSE regs. */
12120 if (frame.nsseregs)
12121 ix86_emit_restore_sse_regs_using_mov (frame.sse_reg_save_offset,
12124 if (restore_regs_via_mov)
12129 ix86_emit_restore_regs_using_mov (frame.reg_save_offset, style == 2);
12131 /* eh_return epilogues need %ecx added to the stack pointer. */
12134 rtx sa = EH_RETURN_STACKADJ_RTX;
12137 /* Stack align doesn't work with eh_return. */
12138 gcc_assert (!stack_realign_drap);
12139 /* Neither does regparm nested functions. */
12140 gcc_assert (!ix86_static_chain_on_stack);
12142 if (frame_pointer_needed)
12144 t = gen_rtx_PLUS (Pmode, hard_frame_pointer_rtx, sa);
12145 t = plus_constant (Pmode, t, m->fs.fp_offset - UNITS_PER_WORD);
12146 emit_insn (gen_rtx_SET (sa, t));
12148 t = gen_frame_mem (Pmode, hard_frame_pointer_rtx);
12149 insn = emit_move_insn (hard_frame_pointer_rtx, t);
12151 /* Note that we use SA as a temporary CFA, as the return
12152 address is at the proper place relative to it. We
12153 pretend this happens at the FP restore insn because
12154 prior to this insn the FP would be stored at the wrong
12155 offset relative to SA, and after this insn we have no
12156 other reasonable register to use for the CFA. We don't
12157 bother resetting the CFA to the SP for the duration of
12158 the return insn. */
12159 add_reg_note (insn, REG_CFA_DEF_CFA,
12160 plus_constant (Pmode, sa, UNITS_PER_WORD));
12161 ix86_add_queued_cfa_restore_notes (insn);
12162 add_reg_note (insn, REG_CFA_RESTORE, hard_frame_pointer_rtx);
12163 RTX_FRAME_RELATED_P (insn) = 1;
12165 m->fs.cfa_reg = sa;
12166 m->fs.cfa_offset = UNITS_PER_WORD;
12167 m->fs.fp_valid = false;
12169 pro_epilogue_adjust_stack (stack_pointer_rtx, sa,
12170 const0_rtx, style, false);
12174 t = gen_rtx_PLUS (Pmode, stack_pointer_rtx, sa);
12175 t = plus_constant (Pmode, t, m->fs.sp_offset - UNITS_PER_WORD);
12176 insn = emit_insn (gen_rtx_SET (stack_pointer_rtx, t));
12177 ix86_add_queued_cfa_restore_notes (insn);
12179 gcc_assert (m->fs.cfa_reg == stack_pointer_rtx);
12180 if (m->fs.cfa_offset != UNITS_PER_WORD)
12182 m->fs.cfa_offset = UNITS_PER_WORD;
12183 add_reg_note (insn, REG_CFA_DEF_CFA,
12184 plus_constant (Pmode, stack_pointer_rtx,
12186 RTX_FRAME_RELATED_P (insn) = 1;
12189 m->fs.sp_offset = UNITS_PER_WORD;
12190 m->fs.sp_valid = true;
12195 /* SEH requires that the function end with (1) a stack adjustment
12196 if necessary, (2) a sequence of pops, and (3) a return or
12197 jump instruction. Prevent insns from the function body from
12198 being scheduled into this sequence. */
12201 /* Prevent a catch region from being adjacent to the standard
12202 epilogue sequence. Unfortuantely crtl->uses_eh_lsda nor
12203 several other flags that would be interesting to test are
12205 if (flag_non_call_exceptions)
12206 emit_insn (gen_nops (const1_rtx));
12208 emit_insn (gen_blockage ());
12211 /* First step is to deallocate the stack frame so that we can
12212 pop the registers. Also do it on SEH target for very large
12213 frame as the emitted instructions aren't allowed by the ABI in
12215 if (!m->fs.sp_valid
12217 && (m->fs.sp_offset - frame.reg_save_offset
12218 >= SEH_MAX_FRAME_SIZE)))
12220 pro_epilogue_adjust_stack (stack_pointer_rtx, hard_frame_pointer_rtx,
12221 GEN_INT (m->fs.fp_offset
12222 - frame.reg_save_offset),
12225 else if (m->fs.sp_offset != frame.reg_save_offset)
12227 pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
12228 GEN_INT (m->fs.sp_offset
12229 - frame.reg_save_offset),
12231 m->fs.cfa_reg == stack_pointer_rtx);
12234 ix86_emit_restore_regs_using_pop ();
12237 /* If we used a stack pointer and haven't already got rid of it,
12239 if (m->fs.fp_valid)
12241 /* If the stack pointer is valid and pointing at the frame
12242 pointer store address, then we only need a pop. */
12243 if (m->fs.sp_valid && m->fs.sp_offset == frame.hfp_save_offset)
12244 ix86_emit_restore_reg_using_pop (hard_frame_pointer_rtx);
12245 /* Leave results in shorter dependency chains on CPUs that are
12246 able to grok it fast. */
12247 else if (TARGET_USE_LEAVE
12248 || optimize_bb_for_size_p (EXIT_BLOCK_PTR_FOR_FN (cfun))
12249 || !cfun->machine->use_fast_prologue_epilogue)
12250 ix86_emit_leave ();
12253 pro_epilogue_adjust_stack (stack_pointer_rtx,
12254 hard_frame_pointer_rtx,
12255 const0_rtx, style, !using_drap);
12256 ix86_emit_restore_reg_using_pop (hard_frame_pointer_rtx);
12262 int param_ptr_offset = UNITS_PER_WORD;
12265 gcc_assert (stack_realign_drap);
12267 if (ix86_static_chain_on_stack)
12268 param_ptr_offset += UNITS_PER_WORD;
12269 if (!call_used_regs[REGNO (crtl->drap_reg)])
12270 param_ptr_offset += UNITS_PER_WORD;
12272 insn = emit_insn (gen_rtx_SET
12273 (stack_pointer_rtx,
12274 gen_rtx_PLUS (Pmode,
12276 GEN_INT (-param_ptr_offset))));
12277 m->fs.cfa_reg = stack_pointer_rtx;
12278 m->fs.cfa_offset = param_ptr_offset;
12279 m->fs.sp_offset = param_ptr_offset;
12280 m->fs.realigned = false;
12282 add_reg_note (insn, REG_CFA_DEF_CFA,
12283 gen_rtx_PLUS (Pmode, stack_pointer_rtx,
12284 GEN_INT (param_ptr_offset)));
12285 RTX_FRAME_RELATED_P (insn) = 1;
12287 if (!call_used_regs[REGNO (crtl->drap_reg)])
12288 ix86_emit_restore_reg_using_pop (crtl->drap_reg);
12291 /* At this point the stack pointer must be valid, and we must have
12292 restored all of the registers. We may not have deallocated the
12293 entire stack frame. We've delayed this until now because it may
12294 be possible to merge the local stack deallocation with the
12295 deallocation forced by ix86_static_chain_on_stack. */
12296 gcc_assert (m->fs.sp_valid);
12297 gcc_assert (!m->fs.fp_valid);
12298 gcc_assert (!m->fs.realigned);
12299 if (m->fs.sp_offset != UNITS_PER_WORD)
12301 pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
12302 GEN_INT (m->fs.sp_offset - UNITS_PER_WORD),
12306 ix86_add_queued_cfa_restore_notes (get_last_insn ());
12308 /* Sibcall epilogues don't want a return instruction. */
12311 m->fs = frame_state_save;
12315 if (crtl->args.pops_args && crtl->args.size)
12317 rtx popc = GEN_INT (crtl->args.pops_args);
12319 /* i386 can only pop 64K bytes. If asked to pop more, pop return
12320 address, do explicit add, and jump indirectly to the caller. */
12322 if (crtl->args.pops_args >= 65536)
12324 rtx ecx = gen_rtx_REG (SImode, CX_REG);
12327 /* There is no "pascal" calling convention in any 64bit ABI. */
12328 gcc_assert (!TARGET_64BIT);
12330 insn = emit_insn (gen_pop (ecx));
12331 m->fs.cfa_offset -= UNITS_PER_WORD;
12332 m->fs.sp_offset -= UNITS_PER_WORD;
12334 rtx x = plus_constant (Pmode, stack_pointer_rtx, UNITS_PER_WORD);
12335 x = gen_rtx_SET (stack_pointer_rtx, x);
12336 add_reg_note (insn, REG_CFA_ADJUST_CFA, x);
12337 add_reg_note (insn, REG_CFA_REGISTER, gen_rtx_SET (ecx, pc_rtx));
12338 RTX_FRAME_RELATED_P (insn) = 1;
12340 pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
12342 emit_jump_insn (gen_simple_return_indirect_internal (ecx));
12345 emit_jump_insn (gen_simple_return_pop_internal (popc));
12348 emit_jump_insn (gen_simple_return_internal ());
12350 /* Restore the state back to the state from the prologue,
12351 so that it's correct for the next epilogue. */
12352 m->fs = frame_state_save;
12355 /* Reset from the function's potential modifications. */
12358 ix86_output_function_epilogue (FILE *file ATTRIBUTE_UNUSED, HOST_WIDE_INT)
12360 if (pic_offset_table_rtx
12361 && !ix86_use_pseudo_pic_reg ())
12362 SET_REGNO (pic_offset_table_rtx, REAL_PIC_OFFSET_TABLE_REGNUM);
12364 /* Mach-O doesn't support labels at the end of objects, so if
12365 it looks like we might want one, insert a NOP. */
12367 rtx_insn *insn = get_last_insn ();
12368 rtx_insn *deleted_debug_label = NULL;
12371 && NOTE_KIND (insn) != NOTE_INSN_DELETED_LABEL)
12373 /* Don't insert a nop for NOTE_INSN_DELETED_DEBUG_LABEL
12374 notes only, instead set their CODE_LABEL_NUMBER to -1,
12375 otherwise there would be code generation differences
12376 in between -g and -g0. */
12377 if (NOTE_P (insn) && NOTE_KIND (insn) == NOTE_INSN_DELETED_DEBUG_LABEL)
12378 deleted_debug_label = insn;
12379 insn = PREV_INSN (insn);
12384 && NOTE_KIND (insn) == NOTE_INSN_DELETED_LABEL)))
12385 fputs ("\tnop\n", file);
12386 else if (deleted_debug_label)
12387 for (insn = deleted_debug_label; insn; insn = NEXT_INSN (insn))
12388 if (NOTE_KIND (insn) == NOTE_INSN_DELETED_DEBUG_LABEL)
12389 CODE_LABEL_NUMBER (insn) = -1;
12395 /* Return a scratch register to use in the split stack prologue. The
12396 split stack prologue is used for -fsplit-stack. It is the first
12397 instructions in the function, even before the regular prologue.
12398 The scratch register can be any caller-saved register which is not
12399 used for parameters or for the static chain. */
12401 static unsigned int
12402 split_stack_prologue_scratch_regno (void)
12408 bool is_fastcall, is_thiscall;
12411 is_fastcall = (lookup_attribute ("fastcall",
12412 TYPE_ATTRIBUTES (TREE_TYPE (cfun->decl)))
12414 is_thiscall = (lookup_attribute ("thiscall",
12415 TYPE_ATTRIBUTES (TREE_TYPE (cfun->decl)))
12417 regparm = ix86_function_regparm (TREE_TYPE (cfun->decl), cfun->decl);
12421 if (DECL_STATIC_CHAIN (cfun->decl))
12423 sorry ("-fsplit-stack does not support fastcall with "
12424 "nested function");
12425 return INVALID_REGNUM;
12429 else if (is_thiscall)
12431 if (!DECL_STATIC_CHAIN (cfun->decl))
12435 else if (regparm < 3)
12437 if (!DECL_STATIC_CHAIN (cfun->decl))
12443 sorry ("-fsplit-stack does not support 2 register "
12444 "parameters for a nested function");
12445 return INVALID_REGNUM;
12452 /* FIXME: We could make this work by pushing a register
12453 around the addition and comparison. */
12454 sorry ("-fsplit-stack does not support 3 register parameters");
12455 return INVALID_REGNUM;
12460 /* A SYMBOL_REF for the function which allocates new stackspace for
12463 static GTY(()) rtx split_stack_fn;
12465 /* A SYMBOL_REF for the more stack function when using the large
12468 static GTY(()) rtx split_stack_fn_large;
12470 /* Handle -fsplit-stack. These are the first instructions in the
12471 function, even before the regular prologue. */
12474 ix86_expand_split_stack_prologue (void)
12476 struct ix86_frame frame;
12477 HOST_WIDE_INT allocate;
12478 unsigned HOST_WIDE_INT args_size;
12479 rtx_code_label *label;
12480 rtx limit, current, jump_insn, allocate_rtx, call_insn, call_fusage;
12481 rtx scratch_reg = NULL_RTX;
12482 rtx_code_label *varargs_label = NULL;
12485 gcc_assert (flag_split_stack && reload_completed);
12487 ix86_finalize_stack_realign_flags ();
12488 ix86_compute_frame_layout (&frame);
12489 allocate = frame.stack_pointer_offset - INCOMING_FRAME_SP_OFFSET;
12491 /* This is the label we will branch to if we have enough stack
12492 space. We expect the basic block reordering pass to reverse this
12493 branch if optimizing, so that we branch in the unlikely case. */
12494 label = gen_label_rtx ();
12496 /* We need to compare the stack pointer minus the frame size with
12497 the stack boundary in the TCB. The stack boundary always gives
12498 us SPLIT_STACK_AVAILABLE bytes, so if we need less than that we
12499 can compare directly. Otherwise we need to do an addition. */
12501 limit = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, const0_rtx),
12502 UNSPEC_STACK_CHECK);
12503 limit = gen_rtx_CONST (Pmode, limit);
12504 limit = gen_rtx_MEM (Pmode, limit);
12505 if (allocate < SPLIT_STACK_AVAILABLE)
12506 current = stack_pointer_rtx;
12509 unsigned int scratch_regno;
12512 /* We need a scratch register to hold the stack pointer minus
12513 the required frame size. Since this is the very start of the
12514 function, the scratch register can be any caller-saved
12515 register which is not used for parameters. */
12516 offset = GEN_INT (- allocate);
12517 scratch_regno = split_stack_prologue_scratch_regno ();
12518 if (scratch_regno == INVALID_REGNUM)
12520 scratch_reg = gen_rtx_REG (Pmode, scratch_regno);
12521 if (!TARGET_64BIT || x86_64_immediate_operand (offset, Pmode))
12523 /* We don't use ix86_gen_add3 in this case because it will
12524 want to split to lea, but when not optimizing the insn
12525 will not be split after this point. */
12526 emit_insn (gen_rtx_SET (scratch_reg,
12527 gen_rtx_PLUS (Pmode, stack_pointer_rtx,
12532 emit_move_insn (scratch_reg, offset);
12533 emit_insn (ix86_gen_add3 (scratch_reg, scratch_reg,
12534 stack_pointer_rtx));
12536 current = scratch_reg;
12539 ix86_expand_branch (GEU, current, limit, label);
12540 jump_insn = get_last_insn ();
12541 JUMP_LABEL (jump_insn) = label;
12543 /* Mark the jump as very likely to be taken. */
12544 add_int_reg_note (jump_insn, REG_BR_PROB,
12545 REG_BR_PROB_BASE - REG_BR_PROB_BASE / 100);
12547 if (split_stack_fn == NULL_RTX)
12549 split_stack_fn = gen_rtx_SYMBOL_REF (Pmode, "__morestack");
12550 SYMBOL_REF_FLAGS (split_stack_fn) |= SYMBOL_FLAG_LOCAL;
12552 fn = split_stack_fn;
12554 /* Get more stack space. We pass in the desired stack space and the
12555 size of the arguments to copy to the new stack. In 32-bit mode
12556 we push the parameters; __morestack will return on a new stack
12557 anyhow. In 64-bit mode we pass the parameters in r10 and
12559 allocate_rtx = GEN_INT (allocate);
12560 args_size = crtl->args.size >= 0 ? crtl->args.size : 0;
12561 call_fusage = NULL_RTX;
12566 reg10 = gen_rtx_REG (Pmode, R10_REG);
12567 reg11 = gen_rtx_REG (Pmode, R11_REG);
12569 /* If this function uses a static chain, it will be in %r10.
12570 Preserve it across the call to __morestack. */
12571 if (DECL_STATIC_CHAIN (cfun->decl))
12575 rax = gen_rtx_REG (word_mode, AX_REG);
12576 emit_move_insn (rax, gen_rtx_REG (word_mode, R10_REG));
12577 use_reg (&call_fusage, rax);
12580 if ((ix86_cmodel == CM_LARGE || ix86_cmodel == CM_LARGE_PIC)
12583 HOST_WIDE_INT argval;
12585 gcc_assert (Pmode == DImode);
12586 /* When using the large model we need to load the address
12587 into a register, and we've run out of registers. So we
12588 switch to a different calling convention, and we call a
12589 different function: __morestack_large. We pass the
12590 argument size in the upper 32 bits of r10 and pass the
12591 frame size in the lower 32 bits. */
12592 gcc_assert ((allocate & (HOST_WIDE_INT) 0xffffffff) == allocate);
12593 gcc_assert ((args_size & 0xffffffff) == args_size);
12595 if (split_stack_fn_large == NULL_RTX)
12597 split_stack_fn_large =
12598 gen_rtx_SYMBOL_REF (Pmode, "__morestack_large_model");
12599 SYMBOL_REF_FLAGS (split_stack_fn_large) |= SYMBOL_FLAG_LOCAL;
12601 if (ix86_cmodel == CM_LARGE_PIC)
12603 rtx_code_label *label;
12606 label = gen_label_rtx ();
12607 emit_label (label);
12608 LABEL_PRESERVE_P (label) = 1;
12609 emit_insn (gen_set_rip_rex64 (reg10, label));
12610 emit_insn (gen_set_got_offset_rex64 (reg11, label));
12611 emit_insn (ix86_gen_add3 (reg10, reg10, reg11));
12612 x = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, split_stack_fn_large),
12614 x = gen_rtx_CONST (Pmode, x);
12615 emit_move_insn (reg11, x);
12616 x = gen_rtx_PLUS (Pmode, reg10, reg11);
12617 x = gen_const_mem (Pmode, x);
12618 emit_move_insn (reg11, x);
12621 emit_move_insn (reg11, split_stack_fn_large);
12625 argval = ((args_size << 16) << 16) + allocate;
12626 emit_move_insn (reg10, GEN_INT (argval));
12630 emit_move_insn (reg10, allocate_rtx);
12631 emit_move_insn (reg11, GEN_INT (args_size));
12632 use_reg (&call_fusage, reg11);
12635 use_reg (&call_fusage, reg10);
12639 emit_insn (gen_push (GEN_INT (args_size)));
12640 emit_insn (gen_push (allocate_rtx));
12642 call_insn = ix86_expand_call (NULL_RTX, gen_rtx_MEM (QImode, fn),
12643 GEN_INT (UNITS_PER_WORD), constm1_rtx,
12645 add_function_usage_to (call_insn, call_fusage);
12647 /* In order to make call/return prediction work right, we now need
12648 to execute a return instruction. See
12649 libgcc/config/i386/morestack.S for the details on how this works.
12651 For flow purposes gcc must not see this as a return
12652 instruction--we need control flow to continue at the subsequent
12653 label. Therefore, we use an unspec. */
12654 gcc_assert (crtl->args.pops_args < 65536);
12655 emit_insn (gen_split_stack_return (GEN_INT (crtl->args.pops_args)));
12657 /* If we are in 64-bit mode and this function uses a static chain,
12658 we saved %r10 in %rax before calling _morestack. */
12659 if (TARGET_64BIT && DECL_STATIC_CHAIN (cfun->decl))
12660 emit_move_insn (gen_rtx_REG (word_mode, R10_REG),
12661 gen_rtx_REG (word_mode, AX_REG));
12663 /* If this function calls va_start, we need to store a pointer to
12664 the arguments on the old stack, because they may not have been
12665 all copied to the new stack. At this point the old stack can be
12666 found at the frame pointer value used by __morestack, because
12667 __morestack has set that up before calling back to us. Here we
12668 store that pointer in a scratch register, and in
12669 ix86_expand_prologue we store the scratch register in a stack
12671 if (cfun->machine->split_stack_varargs_pointer != NULL_RTX)
12673 unsigned int scratch_regno;
12677 scratch_regno = split_stack_prologue_scratch_regno ();
12678 scratch_reg = gen_rtx_REG (Pmode, scratch_regno);
12679 frame_reg = gen_rtx_REG (Pmode, BP_REG);
12683 return address within this function
12684 return address of caller of this function
12686 So we add three words to get to the stack arguments.
12690 return address within this function
12691 first argument to __morestack
12692 second argument to __morestack
12693 return address of caller of this function
12695 So we add five words to get to the stack arguments.
12697 words = TARGET_64BIT ? 3 : 5;
12698 emit_insn (gen_rtx_SET (scratch_reg,
12699 gen_rtx_PLUS (Pmode, frame_reg,
12700 GEN_INT (words * UNITS_PER_WORD))));
12702 varargs_label = gen_label_rtx ();
12703 emit_jump_insn (gen_jump (varargs_label));
12704 JUMP_LABEL (get_last_insn ()) = varargs_label;
12709 emit_label (label);
12710 LABEL_NUSES (label) = 1;
12712 /* If this function calls va_start, we now have to set the scratch
12713 register for the case where we do not call __morestack. In this
12714 case we need to set it based on the stack pointer. */
12715 if (cfun->machine->split_stack_varargs_pointer != NULL_RTX)
12717 emit_insn (gen_rtx_SET (scratch_reg,
12718 gen_rtx_PLUS (Pmode, stack_pointer_rtx,
12719 GEN_INT (UNITS_PER_WORD))));
12721 emit_label (varargs_label);
12722 LABEL_NUSES (varargs_label) = 1;
12726 /* We may have to tell the dataflow pass that the split stack prologue
12727 is initializing a scratch register. */
12730 ix86_live_on_entry (bitmap regs)
12732 if (cfun->machine->split_stack_varargs_pointer != NULL_RTX)
12734 gcc_assert (flag_split_stack);
12735 bitmap_set_bit (regs, split_stack_prologue_scratch_regno ());
12739 /* Extract the parts of an RTL expression that is a valid memory address
12740 for an instruction. Return 0 if the structure of the address is
12741 grossly off. Return -1 if the address contains ASHIFT, so it is not
12742 strictly valid, but still used for computing length of lea instruction. */
12745 ix86_decompose_address (rtx addr, struct ix86_address *out)
12747 rtx base = NULL_RTX, index = NULL_RTX, disp = NULL_RTX;
12748 rtx base_reg, index_reg;
12749 HOST_WIDE_INT scale = 1;
12750 rtx scale_rtx = NULL_RTX;
12753 enum ix86_address_seg seg = SEG_DEFAULT;
12755 /* Allow zero-extended SImode addresses,
12756 they will be emitted with addr32 prefix. */
12757 if (TARGET_64BIT && GET_MODE (addr) == DImode)
12759 if (GET_CODE (addr) == ZERO_EXTEND
12760 && GET_MODE (XEXP (addr, 0)) == SImode)
12762 addr = XEXP (addr, 0);
12763 if (CONST_INT_P (addr))
12766 else if (GET_CODE (addr) == AND
12767 && const_32bit_mask (XEXP (addr, 1), DImode))
12769 addr = simplify_gen_subreg (SImode, XEXP (addr, 0), DImode, 0);
12770 if (addr == NULL_RTX)
12773 if (CONST_INT_P (addr))
12778 /* Allow SImode subregs of DImode addresses,
12779 they will be emitted with addr32 prefix. */
12780 if (TARGET_64BIT && GET_MODE (addr) == SImode)
12782 if (GET_CODE (addr) == SUBREG
12783 && GET_MODE (SUBREG_REG (addr)) == DImode)
12785 addr = SUBREG_REG (addr);
12786 if (CONST_INT_P (addr))
12793 else if (GET_CODE (addr) == SUBREG)
12795 if (REG_P (SUBREG_REG (addr)))
12800 else if (GET_CODE (addr) == PLUS)
12802 rtx addends[4], op;
12810 addends[n++] = XEXP (op, 1);
12813 while (GET_CODE (op) == PLUS);
12818 for (i = n; i >= 0; --i)
12821 switch (GET_CODE (op))
12826 index = XEXP (op, 0);
12827 scale_rtx = XEXP (op, 1);
12833 index = XEXP (op, 0);
12834 tmp = XEXP (op, 1);
12835 if (!CONST_INT_P (tmp))
12837 scale = INTVAL (tmp);
12838 if ((unsigned HOST_WIDE_INT) scale > 3)
12840 scale = 1 << scale;
12845 if (GET_CODE (op) != UNSPEC)
12850 if (XINT (op, 1) == UNSPEC_TP
12851 && TARGET_TLS_DIRECT_SEG_REFS
12852 && seg == SEG_DEFAULT)
12853 seg = DEFAULT_TLS_SEG_REG;
12859 if (!REG_P (SUBREG_REG (op)))
12886 else if (GET_CODE (addr) == MULT)
12888 index = XEXP (addr, 0); /* index*scale */
12889 scale_rtx = XEXP (addr, 1);
12891 else if (GET_CODE (addr) == ASHIFT)
12893 /* We're called for lea too, which implements ashift on occasion. */
12894 index = XEXP (addr, 0);
12895 tmp = XEXP (addr, 1);
12896 if (!CONST_INT_P (tmp))
12898 scale = INTVAL (tmp);
12899 if ((unsigned HOST_WIDE_INT) scale > 3)
12901 scale = 1 << scale;
12905 disp = addr; /* displacement */
12911 else if (GET_CODE (index) == SUBREG
12912 && REG_P (SUBREG_REG (index)))
12918 /* Extract the integral value of scale. */
12921 if (!CONST_INT_P (scale_rtx))
12923 scale = INTVAL (scale_rtx);
12926 base_reg = base && GET_CODE (base) == SUBREG ? SUBREG_REG (base) : base;
12927 index_reg = index && GET_CODE (index) == SUBREG ? SUBREG_REG (index) : index;
12929 /* Avoid useless 0 displacement. */
12930 if (disp == const0_rtx && (base || index))
12933 /* Allow arg pointer and stack pointer as index if there is not scaling. */
12934 if (base_reg && index_reg && scale == 1
12935 && (index_reg == arg_pointer_rtx
12936 || index_reg == frame_pointer_rtx
12937 || (REG_P (index_reg) && REGNO (index_reg) == STACK_POINTER_REGNUM)))
12939 std::swap (base, index);
12940 std::swap (base_reg, index_reg);
12943 /* Special case: %ebp cannot be encoded as a base without a displacement.
12947 && (base_reg == hard_frame_pointer_rtx
12948 || base_reg == frame_pointer_rtx
12949 || base_reg == arg_pointer_rtx
12950 || (REG_P (base_reg)
12951 && (REGNO (base_reg) == HARD_FRAME_POINTER_REGNUM
12952 || REGNO (base_reg) == R13_REG))))
12955 /* Special case: on K6, [%esi] makes the instruction vector decoded.
12956 Avoid this by transforming to [%esi+0].
12957 Reload calls address legitimization without cfun defined, so we need
12958 to test cfun for being non-NULL. */
12959 if (TARGET_K6 && cfun && optimize_function_for_speed_p (cfun)
12960 && base_reg && !index_reg && !disp
12961 && REG_P (base_reg) && REGNO (base_reg) == SI_REG)
12964 /* Special case: encode reg+reg instead of reg*2. */
12965 if (!base && index && scale == 2)
12966 base = index, base_reg = index_reg, scale = 1;
12968 /* Special case: scaling cannot be encoded without base or displacement. */
12969 if (!base && !disp && index && scale != 1)
12973 out->index = index;
12975 out->scale = scale;
12981 /* Return cost of the memory address x.
12982 For i386, it is better to use a complex address than let gcc copy
12983 the address into a reg and make a new pseudo. But not if the address
12984 requires to two regs - that would mean more pseudos with longer
12987 ix86_address_cost (rtx x, machine_mode, addr_space_t, bool)
12989 struct ix86_address parts;
12991 int ok = ix86_decompose_address (x, &parts);
12995 if (parts.base && GET_CODE (parts.base) == SUBREG)
12996 parts.base = SUBREG_REG (parts.base);
12997 if (parts.index && GET_CODE (parts.index) == SUBREG)
12998 parts.index = SUBREG_REG (parts.index);
13000 /* Attempt to minimize number of registers in the address by increasing
13001 address cost for each used register. We don't increase address cost
13002 for "pic_offset_table_rtx". When a memopt with "pic_offset_table_rtx"
13003 is not invariant itself it most likely means that base or index is not
13004 invariant. Therefore only "pic_offset_table_rtx" could be hoisted out,
13005 which is not profitable for x86. */
13007 && (!REG_P (parts.base) || REGNO (parts.base) >= FIRST_PSEUDO_REGISTER)
13008 && (current_pass->type == GIMPLE_PASS
13009 || !pic_offset_table_rtx
13010 || !REG_P (parts.base)
13011 || REGNO (pic_offset_table_rtx) != REGNO (parts.base)))
13015 && (!REG_P (parts.index) || REGNO (parts.index) >= FIRST_PSEUDO_REGISTER)
13016 && (current_pass->type == GIMPLE_PASS
13017 || !pic_offset_table_rtx
13018 || !REG_P (parts.index)
13019 || REGNO (pic_offset_table_rtx) != REGNO (parts.index)))
13022 /* AMD-K6 don't like addresses with ModR/M set to 00_xxx_100b,
13023 since it's predecode logic can't detect the length of instructions
13024 and it degenerates to vector decoded. Increase cost of such
13025 addresses here. The penalty is minimally 2 cycles. It may be worthwhile
13026 to split such addresses or even refuse such addresses at all.
13028 Following addressing modes are affected:
13033 The first and last case may be avoidable by explicitly coding the zero in
13034 memory address, but I don't have AMD-K6 machine handy to check this
13038 && ((!parts.disp && parts.base && parts.index && parts.scale != 1)
13039 || (parts.disp && !parts.base && parts.index && parts.scale != 1)
13040 || (!parts.disp && parts.base && parts.index && parts.scale == 1)))
13046 /* Allow {LABEL | SYMBOL}_REF - SYMBOL_REF-FOR-PICBASE for Mach-O as
13047 this is used for to form addresses to local data when -fPIC is in
13051 darwin_local_data_pic (rtx disp)
13053 return (GET_CODE (disp) == UNSPEC
13054 && XINT (disp, 1) == UNSPEC_MACHOPIC_OFFSET);
13057 /* Determine if a given RTX is a valid constant. We already know this
13058 satisfies CONSTANT_P. */
13061 ix86_legitimate_constant_p (machine_mode, rtx x)
13063 /* Pointer bounds constants are not valid. */
13064 if (POINTER_BOUNDS_MODE_P (GET_MODE (x)))
13067 switch (GET_CODE (x))
13072 if (GET_CODE (x) == PLUS)
13074 if (!CONST_INT_P (XEXP (x, 1)))
13079 if (TARGET_MACHO && darwin_local_data_pic (x))
13082 /* Only some unspecs are valid as "constants". */
13083 if (GET_CODE (x) == UNSPEC)
13084 switch (XINT (x, 1))
13087 case UNSPEC_GOTOFF:
13088 case UNSPEC_PLTOFF:
13089 return TARGET_64BIT;
13091 case UNSPEC_NTPOFF:
13092 x = XVECEXP (x, 0, 0);
13093 return (GET_CODE (x) == SYMBOL_REF
13094 && SYMBOL_REF_TLS_MODEL (x) == TLS_MODEL_LOCAL_EXEC);
13095 case UNSPEC_DTPOFF:
13096 x = XVECEXP (x, 0, 0);
13097 return (GET_CODE (x) == SYMBOL_REF
13098 && SYMBOL_REF_TLS_MODEL (x) == TLS_MODEL_LOCAL_DYNAMIC);
13103 /* We must have drilled down to a symbol. */
13104 if (GET_CODE (x) == LABEL_REF)
13106 if (GET_CODE (x) != SYMBOL_REF)
13111 /* TLS symbols are never valid. */
13112 if (SYMBOL_REF_TLS_MODEL (x))
13115 /* DLLIMPORT symbols are never valid. */
13116 if (TARGET_DLLIMPORT_DECL_ATTRIBUTES
13117 && SYMBOL_REF_DLLIMPORT_P (x))
13121 /* mdynamic-no-pic */
13122 if (MACHO_DYNAMIC_NO_PIC_P)
13123 return machopic_symbol_defined_p (x);
13127 case CONST_WIDE_INT:
13128 if (!TARGET_64BIT && !standard_sse_constant_p (x))
13133 if (!standard_sse_constant_p (x))
13140 /* Otherwise we handle everything else in the move patterns. */
13144 /* Determine if it's legal to put X into the constant pool. This
13145 is not possible for the address of thread-local symbols, which
13146 is checked above. */
13149 ix86_cannot_force_const_mem (machine_mode mode, rtx x)
13151 /* We can always put integral constants and vectors in memory. */
13152 switch (GET_CODE (x))
13155 case CONST_WIDE_INT:
13163 return !ix86_legitimate_constant_p (mode, x);
13166 /* Nonzero if the symbol is marked as dllimport, or as stub-variable,
13170 is_imported_p (rtx x)
13172 if (!TARGET_DLLIMPORT_DECL_ATTRIBUTES
13173 || GET_CODE (x) != SYMBOL_REF)
13176 return SYMBOL_REF_DLLIMPORT_P (x) || SYMBOL_REF_STUBVAR_P (x);
13180 /* Nonzero if the constant value X is a legitimate general operand
13181 when generating PIC code. It is given that flag_pic is on and
13182 that X satisfies CONSTANT_P. */
13185 legitimate_pic_operand_p (rtx x)
13189 switch (GET_CODE (x))
13192 inner = XEXP (x, 0);
13193 if (GET_CODE (inner) == PLUS
13194 && CONST_INT_P (XEXP (inner, 1)))
13195 inner = XEXP (inner, 0);
13197 /* Only some unspecs are valid as "constants". */
13198 if (GET_CODE (inner) == UNSPEC)
13199 switch (XINT (inner, 1))
13202 case UNSPEC_GOTOFF:
13203 case UNSPEC_PLTOFF:
13204 return TARGET_64BIT;
13206 x = XVECEXP (inner, 0, 0);
13207 return (GET_CODE (x) == SYMBOL_REF
13208 && SYMBOL_REF_TLS_MODEL (x) == TLS_MODEL_LOCAL_EXEC);
13209 case UNSPEC_MACHOPIC_OFFSET:
13210 return legitimate_pic_address_disp_p (x);
13218 return legitimate_pic_address_disp_p (x);
13225 /* Determine if a given CONST RTX is a valid memory displacement
13229 legitimate_pic_address_disp_p (rtx disp)
13233 /* In 64bit mode we can allow direct addresses of symbols and labels
13234 when they are not dynamic symbols. */
13237 rtx op0 = disp, op1;
13239 switch (GET_CODE (disp))
13245 if (GET_CODE (XEXP (disp, 0)) != PLUS)
13247 op0 = XEXP (XEXP (disp, 0), 0);
13248 op1 = XEXP (XEXP (disp, 0), 1);
13249 if (!CONST_INT_P (op1)
13250 || INTVAL (op1) >= 16*1024*1024
13251 || INTVAL (op1) < -16*1024*1024)
13253 if (GET_CODE (op0) == LABEL_REF)
13255 if (GET_CODE (op0) == CONST
13256 && GET_CODE (XEXP (op0, 0)) == UNSPEC
13257 && XINT (XEXP (op0, 0), 1) == UNSPEC_PCREL)
13259 if (GET_CODE (op0) == UNSPEC
13260 && XINT (op0, 1) == UNSPEC_PCREL)
13262 if (GET_CODE (op0) != SYMBOL_REF)
13267 /* TLS references should always be enclosed in UNSPEC.
13268 The dllimported symbol needs always to be resolved. */
13269 if (SYMBOL_REF_TLS_MODEL (op0)
13270 || (TARGET_DLLIMPORT_DECL_ATTRIBUTES && SYMBOL_REF_DLLIMPORT_P (op0)))
13275 if (is_imported_p (op0))
13278 if (SYMBOL_REF_FAR_ADDR_P (op0)
13279 || !SYMBOL_REF_LOCAL_P (op0))
13282 /* Function-symbols need to be resolved only for
13284 For the small-model we don't need to resolve anything
13286 if ((ix86_cmodel != CM_LARGE_PIC
13287 && SYMBOL_REF_FUNCTION_P (op0))
13288 || ix86_cmodel == CM_SMALL_PIC)
13290 /* Non-external symbols don't need to be resolved for
13291 large, and medium-model. */
13292 if ((ix86_cmodel == CM_LARGE_PIC
13293 || ix86_cmodel == CM_MEDIUM_PIC)
13294 && !SYMBOL_REF_EXTERNAL_P (op0))
13297 else if (!SYMBOL_REF_FAR_ADDR_P (op0)
13298 && (SYMBOL_REF_LOCAL_P (op0)
13299 || (HAVE_LD_PIE_COPYRELOC
13301 && !SYMBOL_REF_WEAK (op0)
13302 && !SYMBOL_REF_FUNCTION_P (op0)))
13303 && ix86_cmodel != CM_LARGE_PIC)
13311 if (GET_CODE (disp) != CONST)
13313 disp = XEXP (disp, 0);
13317 /* We are unsafe to allow PLUS expressions. This limit allowed distance
13318 of GOT tables. We should not need these anyway. */
13319 if (GET_CODE (disp) != UNSPEC
13320 || (XINT (disp, 1) != UNSPEC_GOTPCREL
13321 && XINT (disp, 1) != UNSPEC_GOTOFF
13322 && XINT (disp, 1) != UNSPEC_PCREL
13323 && XINT (disp, 1) != UNSPEC_PLTOFF))
13326 if (GET_CODE (XVECEXP (disp, 0, 0)) != SYMBOL_REF
13327 && GET_CODE (XVECEXP (disp, 0, 0)) != LABEL_REF)
13333 if (GET_CODE (disp) == PLUS)
13335 if (!CONST_INT_P (XEXP (disp, 1)))
13337 disp = XEXP (disp, 0);
13341 if (TARGET_MACHO && darwin_local_data_pic (disp))
13344 if (GET_CODE (disp) != UNSPEC)
13347 switch (XINT (disp, 1))
13352 /* We need to check for both symbols and labels because VxWorks loads
13353 text labels with @GOT rather than @GOTOFF. See gotoff_operand for
13355 return (GET_CODE (XVECEXP (disp, 0, 0)) == SYMBOL_REF
13356 || GET_CODE (XVECEXP (disp, 0, 0)) == LABEL_REF);
13357 case UNSPEC_GOTOFF:
13358 /* Refuse GOTOFF in 64bit mode since it is always 64bit when used.
13359 While ABI specify also 32bit relocation but we don't produce it in
13360 small PIC model at all. */
13361 if ((GET_CODE (XVECEXP (disp, 0, 0)) == SYMBOL_REF
13362 || GET_CODE (XVECEXP (disp, 0, 0)) == LABEL_REF)
13364 return !TARGET_PECOFF && gotoff_operand (XVECEXP (disp, 0, 0), Pmode);
13366 case UNSPEC_GOTTPOFF:
13367 case UNSPEC_GOTNTPOFF:
13368 case UNSPEC_INDNTPOFF:
13371 disp = XVECEXP (disp, 0, 0);
13372 return (GET_CODE (disp) == SYMBOL_REF
13373 && SYMBOL_REF_TLS_MODEL (disp) == TLS_MODEL_INITIAL_EXEC);
13374 case UNSPEC_NTPOFF:
13375 disp = XVECEXP (disp, 0, 0);
13376 return (GET_CODE (disp) == SYMBOL_REF
13377 && SYMBOL_REF_TLS_MODEL (disp) == TLS_MODEL_LOCAL_EXEC);
13378 case UNSPEC_DTPOFF:
13379 disp = XVECEXP (disp, 0, 0);
13380 return (GET_CODE (disp) == SYMBOL_REF
13381 && SYMBOL_REF_TLS_MODEL (disp) == TLS_MODEL_LOCAL_DYNAMIC);
13387 /* Determine if op is suitable RTX for an address register.
13388 Return naked register if a register or a register subreg is
13389 found, otherwise return NULL_RTX. */
13392 ix86_validate_address_register (rtx op)
13394 machine_mode mode = GET_MODE (op);
13396 /* Only SImode or DImode registers can form the address. */
13397 if (mode != SImode && mode != DImode)
13402 else if (GET_CODE (op) == SUBREG)
13404 rtx reg = SUBREG_REG (op);
13409 mode = GET_MODE (reg);
13411 /* Don't allow SUBREGs that span more than a word. It can
13412 lead to spill failures when the register is one word out
13413 of a two word structure. */
13414 if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
13417 /* Allow only SUBREGs of non-eliminable hard registers. */
13418 if (register_no_elim_operand (reg, mode))
13422 /* Op is not a register. */
13426 /* Recognizes RTL expressions that are valid memory addresses for an
13427 instruction. The MODE argument is the machine mode for the MEM
13428 expression that wants to use this address.
13430 It only recognizes address in canonical form. LEGITIMIZE_ADDRESS should
13431 convert common non-canonical forms to canonical form so that they will
13435 ix86_legitimate_address_p (machine_mode, rtx addr, bool strict)
13437 struct ix86_address parts;
13438 rtx base, index, disp;
13439 HOST_WIDE_INT scale;
13440 enum ix86_address_seg seg;
13442 if (ix86_decompose_address (addr, &parts) <= 0)
13443 /* Decomposition failed. */
13447 index = parts.index;
13449 scale = parts.scale;
13452 /* Validate base register. */
13455 rtx reg = ix86_validate_address_register (base);
13457 if (reg == NULL_RTX)
13460 if ((strict && ! REG_OK_FOR_BASE_STRICT_P (reg))
13461 || (! strict && ! REG_OK_FOR_BASE_NONSTRICT_P (reg)))
13462 /* Base is not valid. */
13466 /* Validate index register. */
13469 rtx reg = ix86_validate_address_register (index);
13471 if (reg == NULL_RTX)
13474 if ((strict && ! REG_OK_FOR_INDEX_STRICT_P (reg))
13475 || (! strict && ! REG_OK_FOR_INDEX_NONSTRICT_P (reg)))
13476 /* Index is not valid. */
13480 /* Index and base should have the same mode. */
13482 && GET_MODE (base) != GET_MODE (index))
13485 /* Address override works only on the (%reg) part of %fs:(%reg). */
13486 if (seg != SEG_DEFAULT
13487 && ((base && GET_MODE (base) != word_mode)
13488 || (index && GET_MODE (index) != word_mode)))
13491 /* Validate scale factor. */
13495 /* Scale without index. */
13498 if (scale != 2 && scale != 4 && scale != 8)
13499 /* Scale is not a valid multiplier. */
13503 /* Validate displacement. */
13506 if (GET_CODE (disp) == CONST
13507 && GET_CODE (XEXP (disp, 0)) == UNSPEC
13508 && XINT (XEXP (disp, 0), 1) != UNSPEC_MACHOPIC_OFFSET)
13509 switch (XINT (XEXP (disp, 0), 1))
13511 /* Refuse GOTOFF and GOT in 64bit mode since it is always 64bit when
13512 used. While ABI specify also 32bit relocations, we don't produce
13513 them at all and use IP relative instead. */
13515 case UNSPEC_GOTOFF:
13516 gcc_assert (flag_pic);
13518 goto is_legitimate_pic;
13520 /* 64bit address unspec. */
13523 case UNSPEC_GOTPCREL:
13525 gcc_assert (flag_pic);
13526 goto is_legitimate_pic;
13528 case UNSPEC_GOTTPOFF:
13529 case UNSPEC_GOTNTPOFF:
13530 case UNSPEC_INDNTPOFF:
13531 case UNSPEC_NTPOFF:
13532 case UNSPEC_DTPOFF:
13535 case UNSPEC_STACK_CHECK:
13536 gcc_assert (flag_split_stack);
13540 /* Invalid address unspec. */
13544 else if (SYMBOLIC_CONST (disp)
13548 && MACHOPIC_INDIRECT
13549 && !machopic_operand_p (disp)
13555 if (TARGET_64BIT && (index || base))
13557 /* foo@dtpoff(%rX) is ok. */
13558 if (GET_CODE (disp) != CONST
13559 || GET_CODE (XEXP (disp, 0)) != PLUS
13560 || GET_CODE (XEXP (XEXP (disp, 0), 0)) != UNSPEC
13561 || !CONST_INT_P (XEXP (XEXP (disp, 0), 1))
13562 || (XINT (XEXP (XEXP (disp, 0), 0), 1) != UNSPEC_DTPOFF
13563 && XINT (XEXP (XEXP (disp, 0), 0), 1) != UNSPEC_NTPOFF))
13564 /* Non-constant pic memory reference. */
13567 else if ((!TARGET_MACHO || flag_pic)
13568 && ! legitimate_pic_address_disp_p (disp))
13569 /* Displacement is an invalid pic construct. */
13572 else if (MACHO_DYNAMIC_NO_PIC_P
13573 && !ix86_legitimate_constant_p (Pmode, disp))
13574 /* displacment must be referenced via non_lazy_pointer */
13578 /* This code used to verify that a symbolic pic displacement
13579 includes the pic_offset_table_rtx register.
13581 While this is good idea, unfortunately these constructs may
13582 be created by "adds using lea" optimization for incorrect
13591 This code is nonsensical, but results in addressing
13592 GOT table with pic_offset_table_rtx base. We can't
13593 just refuse it easily, since it gets matched by
13594 "addsi3" pattern, that later gets split to lea in the
13595 case output register differs from input. While this
13596 can be handled by separate addsi pattern for this case
13597 that never results in lea, this seems to be easier and
13598 correct fix for crash to disable this test. */
13600 else if (GET_CODE (disp) != LABEL_REF
13601 && !CONST_INT_P (disp)
13602 && (GET_CODE (disp) != CONST
13603 || !ix86_legitimate_constant_p (Pmode, disp))
13604 && (GET_CODE (disp) != SYMBOL_REF
13605 || !ix86_legitimate_constant_p (Pmode, disp)))
13606 /* Displacement is not constant. */
13608 else if (TARGET_64BIT
13609 && !x86_64_immediate_operand (disp, VOIDmode))
13610 /* Displacement is out of range. */
13612 /* In x32 mode, constant addresses are sign extended to 64bit, so
13613 we have to prevent addresses from 0x80000000 to 0xffffffff. */
13614 else if (TARGET_X32 && !(index || base)
13615 && CONST_INT_P (disp)
13616 && val_signbit_known_set_p (SImode, INTVAL (disp)))
13620 /* Everything looks valid. */
13624 /* Determine if a given RTX is a valid constant address. */
13627 constant_address_p (rtx x)
13629 return CONSTANT_P (x) && ix86_legitimate_address_p (Pmode, x, 1);
13632 /* Return a unique alias set for the GOT. */
13634 static alias_set_type
13635 ix86_GOT_alias_set (void)
13637 static alias_set_type set = -1;
13639 set = new_alias_set ();
13643 /* Return a legitimate reference for ORIG (an address) using the
13644 register REG. If REG is 0, a new pseudo is generated.
13646 There are two types of references that must be handled:
13648 1. Global data references must load the address from the GOT, via
13649 the PIC reg. An insn is emitted to do this load, and the reg is
13652 2. Static data references, constant pool addresses, and code labels
13653 compute the address as an offset from the GOT, whose base is in
13654 the PIC reg. Static data objects have SYMBOL_FLAG_LOCAL set to
13655 differentiate them from global data objects. The returned
13656 address is the PIC reg + an unspec constant.
13658 TARGET_LEGITIMATE_ADDRESS_P rejects symbolic references unless the PIC
13659 reg also appears in the address. */
13662 legitimize_pic_address (rtx orig, rtx reg)
13665 rtx new_rtx = orig;
13668 if (TARGET_MACHO && !TARGET_64BIT)
13671 reg = gen_reg_rtx (Pmode);
13672 /* Use the generic Mach-O PIC machinery. */
13673 return machopic_legitimize_pic_address (orig, GET_MODE (orig), reg);
13677 if (TARGET_64BIT && TARGET_DLLIMPORT_DECL_ATTRIBUTES)
13679 rtx tmp = legitimize_pe_coff_symbol (addr, true);
13684 if (TARGET_64BIT && legitimate_pic_address_disp_p (addr))
13686 else if (TARGET_64BIT && !TARGET_PECOFF
13687 && ix86_cmodel != CM_SMALL_PIC && gotoff_operand (addr, Pmode))
13690 /* This symbol may be referenced via a displacement from the PIC
13691 base address (@GOTOFF). */
13693 if (GET_CODE (addr) == CONST)
13694 addr = XEXP (addr, 0);
13695 if (GET_CODE (addr) == PLUS)
13697 new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, XEXP (addr, 0)),
13699 new_rtx = gen_rtx_PLUS (Pmode, new_rtx, XEXP (addr, 1));
13702 new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr), UNSPEC_GOTOFF);
13703 new_rtx = gen_rtx_CONST (Pmode, new_rtx);
13705 tmpreg = gen_reg_rtx (Pmode);
13708 emit_move_insn (tmpreg, new_rtx);
13712 new_rtx = expand_simple_binop (Pmode, PLUS, reg, pic_offset_table_rtx,
13713 tmpreg, 1, OPTAB_DIRECT);
13717 new_rtx = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, tmpreg);
13719 else if (!TARGET_64BIT && !TARGET_PECOFF && gotoff_operand (addr, Pmode))
13721 /* This symbol may be referenced via a displacement from the PIC
13722 base address (@GOTOFF). */
13724 if (GET_CODE (addr) == CONST)
13725 addr = XEXP (addr, 0);
13726 if (GET_CODE (addr) == PLUS)
13728 new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, XEXP (addr, 0)),
13730 new_rtx = gen_rtx_PLUS (Pmode, new_rtx, XEXP (addr, 1));
13733 new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr), UNSPEC_GOTOFF);
13734 new_rtx = gen_rtx_CONST (Pmode, new_rtx);
13735 new_rtx = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, new_rtx);
13739 emit_move_insn (reg, new_rtx);
13743 else if ((GET_CODE (addr) == SYMBOL_REF && SYMBOL_REF_TLS_MODEL (addr) == 0)
13744 /* We can't use @GOTOFF for text labels on VxWorks;
13745 see gotoff_operand. */
13746 || (TARGET_VXWORKS_RTP && GET_CODE (addr) == LABEL_REF))
13748 rtx tmp = legitimize_pe_coff_symbol (addr, true);
13752 /* For x64 PE-COFF there is no GOT table. So we use address
13754 if (TARGET_64BIT && TARGET_PECOFF)
13756 new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr), UNSPEC_PCREL);
13757 new_rtx = gen_rtx_CONST (Pmode, new_rtx);
13760 reg = gen_reg_rtx (Pmode);
13761 emit_move_insn (reg, new_rtx);
13764 else if (TARGET_64BIT && ix86_cmodel != CM_LARGE_PIC)
13766 new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr), UNSPEC_GOTPCREL);
13767 new_rtx = gen_rtx_CONST (Pmode, new_rtx);
13768 new_rtx = gen_const_mem (Pmode, new_rtx);
13769 set_mem_alias_set (new_rtx, ix86_GOT_alias_set ());
13772 reg = gen_reg_rtx (Pmode);
13773 /* Use directly gen_movsi, otherwise the address is loaded
13774 into register for CSE. We don't want to CSE this addresses,
13775 instead we CSE addresses from the GOT table, so skip this. */
13776 emit_insn (gen_movsi (reg, new_rtx));
13781 /* This symbol must be referenced via a load from the
13782 Global Offset Table (@GOT). */
13784 new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr), UNSPEC_GOT);
13785 new_rtx = gen_rtx_CONST (Pmode, new_rtx);
13787 new_rtx = force_reg (Pmode, new_rtx);
13788 new_rtx = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, new_rtx);
13789 new_rtx = gen_const_mem (Pmode, new_rtx);
13790 set_mem_alias_set (new_rtx, ix86_GOT_alias_set ());
13793 reg = gen_reg_rtx (Pmode);
13794 emit_move_insn (reg, new_rtx);
13800 if (CONST_INT_P (addr)
13801 && !x86_64_immediate_operand (addr, VOIDmode))
13805 emit_move_insn (reg, addr);
13809 new_rtx = force_reg (Pmode, addr);
13811 else if (GET_CODE (addr) == CONST)
13813 addr = XEXP (addr, 0);
13815 /* We must match stuff we generate before. Assume the only
13816 unspecs that can get here are ours. Not that we could do
13817 anything with them anyway.... */
13818 if (GET_CODE (addr) == UNSPEC
13819 || (GET_CODE (addr) == PLUS
13820 && GET_CODE (XEXP (addr, 0)) == UNSPEC))
13822 gcc_assert (GET_CODE (addr) == PLUS);
13824 if (GET_CODE (addr) == PLUS)
13826 rtx op0 = XEXP (addr, 0), op1 = XEXP (addr, 1);
13828 /* Check first to see if this is a constant offset from a @GOTOFF
13829 symbol reference. */
13830 if (!TARGET_PECOFF && gotoff_operand (op0, Pmode)
13831 && CONST_INT_P (op1))
13835 new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, op0),
13837 new_rtx = gen_rtx_PLUS (Pmode, new_rtx, op1);
13838 new_rtx = gen_rtx_CONST (Pmode, new_rtx);
13839 new_rtx = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, new_rtx);
13843 emit_move_insn (reg, new_rtx);
13849 if (INTVAL (op1) < -16*1024*1024
13850 || INTVAL (op1) >= 16*1024*1024)
13852 if (!x86_64_immediate_operand (op1, Pmode))
13853 op1 = force_reg (Pmode, op1);
13854 new_rtx = gen_rtx_PLUS (Pmode, force_reg (Pmode, op0), op1);
13860 rtx base = legitimize_pic_address (op0, reg);
13861 machine_mode mode = GET_MODE (base);
13863 = legitimize_pic_address (op1, base == reg ? NULL_RTX : reg);
13865 if (CONST_INT_P (new_rtx))
13867 if (INTVAL (new_rtx) < -16*1024*1024
13868 || INTVAL (new_rtx) >= 16*1024*1024)
13870 if (!x86_64_immediate_operand (new_rtx, mode))
13871 new_rtx = force_reg (mode, new_rtx);
13873 = gen_rtx_PLUS (mode, force_reg (mode, base), new_rtx);
13876 new_rtx = plus_constant (mode, base, INTVAL (new_rtx));
13880 /* For %rip addressing, we have to use just disp32, not
13883 && (GET_CODE (base) == SYMBOL_REF
13884 || GET_CODE (base) == LABEL_REF))
13885 base = force_reg (mode, base);
13886 if (GET_CODE (new_rtx) == PLUS
13887 && CONSTANT_P (XEXP (new_rtx, 1)))
13889 base = gen_rtx_PLUS (mode, base, XEXP (new_rtx, 0));
13890 new_rtx = XEXP (new_rtx, 1);
13892 new_rtx = gen_rtx_PLUS (mode, base, new_rtx);
13900 /* Load the thread pointer. If TO_REG is true, force it into a register. */
13903 get_thread_pointer (machine_mode tp_mode, bool to_reg)
13905 rtx tp = gen_rtx_UNSPEC (ptr_mode, gen_rtvec (1, const0_rtx), UNSPEC_TP);
13907 if (GET_MODE (tp) != tp_mode)
13909 gcc_assert (GET_MODE (tp) == SImode);
13910 gcc_assert (tp_mode == DImode);
13912 tp = gen_rtx_ZERO_EXTEND (tp_mode, tp);
13916 tp = copy_to_mode_reg (tp_mode, tp);
13921 /* Construct the SYMBOL_REF for the tls_get_addr function. */
13923 static GTY(()) rtx ix86_tls_symbol;
13926 ix86_tls_get_addr (void)
13928 if (!ix86_tls_symbol)
13931 = ((TARGET_ANY_GNU_TLS && !TARGET_64BIT)
13932 ? "___tls_get_addr" : "__tls_get_addr");
13934 ix86_tls_symbol = gen_rtx_SYMBOL_REF (Pmode, sym);
13937 if (ix86_cmodel == CM_LARGE_PIC && !TARGET_PECOFF)
13939 rtx unspec = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, ix86_tls_symbol),
13941 return gen_rtx_PLUS (Pmode, pic_offset_table_rtx,
13942 gen_rtx_CONST (Pmode, unspec));
13945 return ix86_tls_symbol;
13948 /* Construct the SYMBOL_REF for the _TLS_MODULE_BASE_ symbol. */
13950 static GTY(()) rtx ix86_tls_module_base_symbol;
13953 ix86_tls_module_base (void)
13955 if (!ix86_tls_module_base_symbol)
13957 ix86_tls_module_base_symbol
13958 = gen_rtx_SYMBOL_REF (Pmode, "_TLS_MODULE_BASE_");
13960 SYMBOL_REF_FLAGS (ix86_tls_module_base_symbol)
13961 |= TLS_MODEL_GLOBAL_DYNAMIC << SYMBOL_FLAG_TLS_SHIFT;
13964 return ix86_tls_module_base_symbol;
13967 /* A subroutine of ix86_legitimize_address and ix86_expand_move. FOR_MOV is
13968 false if we expect this to be used for a memory address and true if
13969 we expect to load the address into a register. */
13972 legitimize_tls_address (rtx x, enum tls_model model, bool for_mov)
13974 rtx dest, base, off;
13975 rtx pic = NULL_RTX, tp = NULL_RTX;
13976 machine_mode tp_mode = Pmode;
13979 /* Fall back to global dynamic model if tool chain cannot support local
13981 if (TARGET_SUN_TLS && !TARGET_64BIT
13982 && !HAVE_AS_IX86_TLSLDMPLT && !HAVE_AS_IX86_TLSLDM
13983 && model == TLS_MODEL_LOCAL_DYNAMIC)
13984 model = TLS_MODEL_GLOBAL_DYNAMIC;
13988 case TLS_MODEL_GLOBAL_DYNAMIC:
13989 dest = gen_reg_rtx (Pmode);
13993 if (flag_pic && !TARGET_PECOFF)
13994 pic = pic_offset_table_rtx;
13997 pic = gen_reg_rtx (Pmode);
13998 emit_insn (gen_set_got (pic));
14002 if (TARGET_GNU2_TLS)
14005 emit_insn (gen_tls_dynamic_gnu2_64 (dest, x));
14007 emit_insn (gen_tls_dynamic_gnu2_32 (dest, x, pic));
14009 tp = get_thread_pointer (Pmode, true);
14010 dest = force_reg (Pmode, gen_rtx_PLUS (Pmode, tp, dest));
14012 if (GET_MODE (x) != Pmode)
14013 x = gen_rtx_ZERO_EXTEND (Pmode, x);
14015 set_unique_reg_note (get_last_insn (), REG_EQUAL, x);
14019 rtx caddr = ix86_tls_get_addr ();
14023 rtx rax = gen_rtx_REG (Pmode, AX_REG);
14028 (ix86_gen_tls_global_dynamic_64 (rax, x, caddr));
14029 insns = get_insns ();
14032 if (GET_MODE (x) != Pmode)
14033 x = gen_rtx_ZERO_EXTEND (Pmode, x);
14035 RTL_CONST_CALL_P (insns) = 1;
14036 emit_libcall_block (insns, dest, rax, x);
14039 emit_insn (gen_tls_global_dynamic_32 (dest, x, pic, caddr));
14043 case TLS_MODEL_LOCAL_DYNAMIC:
14044 base = gen_reg_rtx (Pmode);
14049 pic = pic_offset_table_rtx;
14052 pic = gen_reg_rtx (Pmode);
14053 emit_insn (gen_set_got (pic));
14057 if (TARGET_GNU2_TLS)
14059 rtx tmp = ix86_tls_module_base ();
14062 emit_insn (gen_tls_dynamic_gnu2_64 (base, tmp));
14064 emit_insn (gen_tls_dynamic_gnu2_32 (base, tmp, pic));
14066 tp = get_thread_pointer (Pmode, true);
14067 set_unique_reg_note (get_last_insn (), REG_EQUAL,
14068 gen_rtx_MINUS (Pmode, tmp, tp));
14072 rtx caddr = ix86_tls_get_addr ();
14076 rtx rax = gen_rtx_REG (Pmode, AX_REG);
14082 (ix86_gen_tls_local_dynamic_base_64 (rax, caddr));
14083 insns = get_insns ();
14086 /* Attach a unique REG_EQUAL, to allow the RTL optimizers to
14087 share the LD_BASE result with other LD model accesses. */
14088 eqv = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, const0_rtx),
14089 UNSPEC_TLS_LD_BASE);
14091 RTL_CONST_CALL_P (insns) = 1;
14092 emit_libcall_block (insns, base, rax, eqv);
14095 emit_insn (gen_tls_local_dynamic_base_32 (base, pic, caddr));
14098 off = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, x), UNSPEC_DTPOFF);
14099 off = gen_rtx_CONST (Pmode, off);
14101 dest = force_reg (Pmode, gen_rtx_PLUS (Pmode, base, off));
14103 if (TARGET_GNU2_TLS)
14105 dest = force_reg (Pmode, gen_rtx_PLUS (Pmode, dest, tp));
14107 if (GET_MODE (x) != Pmode)
14108 x = gen_rtx_ZERO_EXTEND (Pmode, x);
14110 set_unique_reg_note (get_last_insn (), REG_EQUAL, x);
14114 case TLS_MODEL_INITIAL_EXEC:
14117 if (TARGET_SUN_TLS && !TARGET_X32)
14119 /* The Sun linker took the AMD64 TLS spec literally
14120 and can only handle %rax as destination of the
14121 initial executable code sequence. */
14123 dest = gen_reg_rtx (DImode);
14124 emit_insn (gen_tls_initial_exec_64_sun (dest, x));
14128 /* Generate DImode references to avoid %fs:(%reg32)
14129 problems and linker IE->LE relaxation bug. */
14132 type = UNSPEC_GOTNTPOFF;
14136 pic = pic_offset_table_rtx;
14137 type = TARGET_ANY_GNU_TLS ? UNSPEC_GOTNTPOFF : UNSPEC_GOTTPOFF;
14139 else if (!TARGET_ANY_GNU_TLS)
14141 pic = gen_reg_rtx (Pmode);
14142 emit_insn (gen_set_got (pic));
14143 type = UNSPEC_GOTTPOFF;
14148 type = UNSPEC_INDNTPOFF;
14151 off = gen_rtx_UNSPEC (tp_mode, gen_rtvec (1, x), type);
14152 off = gen_rtx_CONST (tp_mode, off);
14154 off = gen_rtx_PLUS (tp_mode, pic, off);
14155 off = gen_const_mem (tp_mode, off);
14156 set_mem_alias_set (off, ix86_GOT_alias_set ());
14158 if (TARGET_64BIT || TARGET_ANY_GNU_TLS)
14160 base = get_thread_pointer (tp_mode,
14161 for_mov || !TARGET_TLS_DIRECT_SEG_REFS);
14162 off = force_reg (tp_mode, off);
14163 return gen_rtx_PLUS (tp_mode, base, off);
14167 base = get_thread_pointer (Pmode, true);
14168 dest = gen_reg_rtx (Pmode);
14169 emit_insn (ix86_gen_sub3 (dest, base, off));
14173 case TLS_MODEL_LOCAL_EXEC:
14174 off = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, x),
14175 (TARGET_64BIT || TARGET_ANY_GNU_TLS)
14176 ? UNSPEC_NTPOFF : UNSPEC_TPOFF);
14177 off = gen_rtx_CONST (Pmode, off);
14179 if (TARGET_64BIT || TARGET_ANY_GNU_TLS)
14181 base = get_thread_pointer (Pmode,
14182 for_mov || !TARGET_TLS_DIRECT_SEG_REFS);
14183 return gen_rtx_PLUS (Pmode, base, off);
14187 base = get_thread_pointer (Pmode, true);
14188 dest = gen_reg_rtx (Pmode);
14189 emit_insn (ix86_gen_sub3 (dest, base, off));
14194 gcc_unreachable ();
14200 /* Create or return the unique __imp_DECL dllimport symbol corresponding
14201 to symbol DECL if BEIMPORT is true. Otherwise create or return the
14202 unique refptr-DECL symbol corresponding to symbol DECL. */
14204 struct dllimport_hasher : ggc_cache_hasher<tree_map *>
14206 static inline hashval_t hash (tree_map *m) { return m->hash; }
14208 equal (tree_map *a, tree_map *b)
14210 return a->base.from == b->base.from;
14214 handle_cache_entry (tree_map *&m)
14216 extern void gt_ggc_mx (tree_map *&);
14217 if (m == HTAB_EMPTY_ENTRY || m == HTAB_DELETED_ENTRY)
14219 else if (ggc_marked_p (m->base.from))
14222 m = static_cast<tree_map *> (HTAB_DELETED_ENTRY);
14226 static GTY((cache)) hash_table<dllimport_hasher> *dllimport_map;
14229 get_dllimport_decl (tree decl, bool beimport)
14231 struct tree_map *h, in;
14233 const char *prefix;
14234 size_t namelen, prefixlen;
14239 if (!dllimport_map)
14240 dllimport_map = hash_table<dllimport_hasher>::create_ggc (512);
14242 in.hash = htab_hash_pointer (decl);
14243 in.base.from = decl;
14244 tree_map **loc = dllimport_map->find_slot_with_hash (&in, in.hash, INSERT);
14249 *loc = h = ggc_alloc<tree_map> ();
14251 h->base.from = decl;
14252 h->to = to = build_decl (DECL_SOURCE_LOCATION (decl),
14253 VAR_DECL, NULL, ptr_type_node);
14254 DECL_ARTIFICIAL (to) = 1;
14255 DECL_IGNORED_P (to) = 1;
14256 DECL_EXTERNAL (to) = 1;
14257 TREE_READONLY (to) = 1;
14259 name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl));
14260 name = targetm.strip_name_encoding (name);
14262 prefix = name[0] == FASTCALL_PREFIX || user_label_prefix[0] == 0
14263 ? "*__imp_" : "*__imp__";
14265 prefix = user_label_prefix[0] == 0 ? "*.refptr." : "*refptr.";
14266 namelen = strlen (name);
14267 prefixlen = strlen (prefix);
14268 imp_name = (char *) alloca (namelen + prefixlen + 1);
14269 memcpy (imp_name, prefix, prefixlen);
14270 memcpy (imp_name + prefixlen, name, namelen + 1);
14272 name = ggc_alloc_string (imp_name, namelen + prefixlen);
14273 rtl = gen_rtx_SYMBOL_REF (Pmode, name);
14274 SET_SYMBOL_REF_DECL (rtl, to);
14275 SYMBOL_REF_FLAGS (rtl) = SYMBOL_FLAG_LOCAL | SYMBOL_FLAG_STUBVAR;
14278 SYMBOL_REF_FLAGS (rtl) |= SYMBOL_FLAG_EXTERNAL;
14279 #ifdef SUB_TARGET_RECORD_STUB
14280 SUB_TARGET_RECORD_STUB (name);
14284 rtl = gen_const_mem (Pmode, rtl);
14285 set_mem_alias_set (rtl, ix86_GOT_alias_set ());
14287 SET_DECL_RTL (to, rtl);
14288 SET_DECL_ASSEMBLER_NAME (to, get_identifier (name));
14293 /* Expand SYMBOL into its corresponding far-addresse symbol.
14294 WANT_REG is true if we require the result be a register. */
14297 legitimize_pe_coff_extern_decl (rtx symbol, bool want_reg)
14302 gcc_assert (SYMBOL_REF_DECL (symbol));
14303 imp_decl = get_dllimport_decl (SYMBOL_REF_DECL (symbol), false);
14305 x = DECL_RTL (imp_decl);
14307 x = force_reg (Pmode, x);
14311 /* Expand SYMBOL into its corresponding dllimport symbol. WANT_REG is
14312 true if we require the result be a register. */
14315 legitimize_dllimport_symbol (rtx symbol, bool want_reg)
14320 gcc_assert (SYMBOL_REF_DECL (symbol));
14321 imp_decl = get_dllimport_decl (SYMBOL_REF_DECL (symbol), true);
14323 x = DECL_RTL (imp_decl);
14325 x = force_reg (Pmode, x);
14329 /* Expand SYMBOL into its corresponding dllimport or refptr symbol. WANT_REG
14330 is true if we require the result be a register. */
14333 legitimize_pe_coff_symbol (rtx addr, bool inreg)
14335 if (!TARGET_PECOFF)
14338 if (TARGET_DLLIMPORT_DECL_ATTRIBUTES)
14340 if (GET_CODE (addr) == SYMBOL_REF && SYMBOL_REF_DLLIMPORT_P (addr))
14341 return legitimize_dllimport_symbol (addr, inreg);
14342 if (GET_CODE (addr) == CONST
14343 && GET_CODE (XEXP (addr, 0)) == PLUS
14344 && GET_CODE (XEXP (XEXP (addr, 0), 0)) == SYMBOL_REF
14345 && SYMBOL_REF_DLLIMPORT_P (XEXP (XEXP (addr, 0), 0)))
14347 rtx t = legitimize_dllimport_symbol (XEXP (XEXP (addr, 0), 0), inreg);
14348 return gen_rtx_PLUS (Pmode, t, XEXP (XEXP (addr, 0), 1));
14352 if (ix86_cmodel != CM_LARGE_PIC && ix86_cmodel != CM_MEDIUM_PIC)
14354 if (GET_CODE (addr) == SYMBOL_REF
14355 && !is_imported_p (addr)
14356 && SYMBOL_REF_EXTERNAL_P (addr)
14357 && SYMBOL_REF_DECL (addr))
14358 return legitimize_pe_coff_extern_decl (addr, inreg);
14360 if (GET_CODE (addr) == CONST
14361 && GET_CODE (XEXP (addr, 0)) == PLUS
14362 && GET_CODE (XEXP (XEXP (addr, 0), 0)) == SYMBOL_REF
14363 && !is_imported_p (XEXP (XEXP (addr, 0), 0))
14364 && SYMBOL_REF_EXTERNAL_P (XEXP (XEXP (addr, 0), 0))
14365 && SYMBOL_REF_DECL (XEXP (XEXP (addr, 0), 0)))
14367 rtx t = legitimize_pe_coff_extern_decl (XEXP (XEXP (addr, 0), 0), inreg);
14368 return gen_rtx_PLUS (Pmode, t, XEXP (XEXP (addr, 0), 1));
14373 /* Try machine-dependent ways of modifying an illegitimate address
14374 to be legitimate. If we find one, return the new, valid address.
14375 This macro is used in only one place: `memory_address' in explow.c.
14377 OLDX is the address as it was before break_out_memory_refs was called.
14378 In some cases it is useful to look at this to decide what needs to be done.
14380 It is always safe for this macro to do nothing. It exists to recognize
14381 opportunities to optimize the output.
14383 For the 80386, we handle X+REG by loading X into a register R and
14384 using R+REG. R will go in a general reg and indexing will be used.
14385 However, if REG is a broken-out memory address or multiplication,
14386 nothing needs to be done because REG can certainly go in a general reg.
14388 When -fpic is used, special handling is needed for symbolic references.
14389 See comments by legitimize_pic_address in i386.c for details. */
14392 ix86_legitimize_address (rtx x, rtx, machine_mode mode)
14394 bool changed = false;
14397 log = GET_CODE (x) == SYMBOL_REF ? SYMBOL_REF_TLS_MODEL (x) : 0;
14399 return legitimize_tls_address (x, (enum tls_model) log, false);
14400 if (GET_CODE (x) == CONST
14401 && GET_CODE (XEXP (x, 0)) == PLUS
14402 && GET_CODE (XEXP (XEXP (x, 0), 0)) == SYMBOL_REF
14403 && (log = SYMBOL_REF_TLS_MODEL (XEXP (XEXP (x, 0), 0))))
14405 rtx t = legitimize_tls_address (XEXP (XEXP (x, 0), 0),
14406 (enum tls_model) log, false);
14407 return gen_rtx_PLUS (Pmode, t, XEXP (XEXP (x, 0), 1));
14410 if (TARGET_DLLIMPORT_DECL_ATTRIBUTES)
14412 rtx tmp = legitimize_pe_coff_symbol (x, true);
14417 if (flag_pic && SYMBOLIC_CONST (x))
14418 return legitimize_pic_address (x, 0);
14421 if (MACHO_DYNAMIC_NO_PIC_P && SYMBOLIC_CONST (x))
14422 return machopic_indirect_data_reference (x, 0);
14425 /* Canonicalize shifts by 0, 1, 2, 3 into multiply */
14426 if (GET_CODE (x) == ASHIFT
14427 && CONST_INT_P (XEXP (x, 1))
14428 && (unsigned HOST_WIDE_INT) INTVAL (XEXP (x, 1)) < 4)
14431 log = INTVAL (XEXP (x, 1));
14432 x = gen_rtx_MULT (Pmode, force_reg (Pmode, XEXP (x, 0)),
14433 GEN_INT (1 << log));
14436 if (GET_CODE (x) == PLUS)
14438 /* Canonicalize shifts by 0, 1, 2, 3 into multiply. */
14440 if (GET_CODE (XEXP (x, 0)) == ASHIFT
14441 && CONST_INT_P (XEXP (XEXP (x, 0), 1))
14442 && (unsigned HOST_WIDE_INT) INTVAL (XEXP (XEXP (x, 0), 1)) < 4)
14445 log = INTVAL (XEXP (XEXP (x, 0), 1));
14446 XEXP (x, 0) = gen_rtx_MULT (Pmode,
14447 force_reg (Pmode, XEXP (XEXP (x, 0), 0)),
14448 GEN_INT (1 << log));
14451 if (GET_CODE (XEXP (x, 1)) == ASHIFT
14452 && CONST_INT_P (XEXP (XEXP (x, 1), 1))
14453 && (unsigned HOST_WIDE_INT) INTVAL (XEXP (XEXP (x, 1), 1)) < 4)
14456 log = INTVAL (XEXP (XEXP (x, 1), 1));
14457 XEXP (x, 1) = gen_rtx_MULT (Pmode,
14458 force_reg (Pmode, XEXP (XEXP (x, 1), 0)),
14459 GEN_INT (1 << log));
14462 /* Put multiply first if it isn't already. */
14463 if (GET_CODE (XEXP (x, 1)) == MULT)
14465 std::swap (XEXP (x, 0), XEXP (x, 1));
14469 /* Canonicalize (plus (mult (reg) (const)) (plus (reg) (const)))
14470 into (plus (plus (mult (reg) (const)) (reg)) (const)). This can be
14471 created by virtual register instantiation, register elimination, and
14472 similar optimizations. */
14473 if (GET_CODE (XEXP (x, 0)) == MULT && GET_CODE (XEXP (x, 1)) == PLUS)
14476 x = gen_rtx_PLUS (Pmode,
14477 gen_rtx_PLUS (Pmode, XEXP (x, 0),
14478 XEXP (XEXP (x, 1), 0)),
14479 XEXP (XEXP (x, 1), 1));
14483 (plus (plus (mult (reg) (const)) (plus (reg) (const))) const)
14484 into (plus (plus (mult (reg) (const)) (reg)) (const)). */
14485 else if (GET_CODE (x) == PLUS && GET_CODE (XEXP (x, 0)) == PLUS
14486 && GET_CODE (XEXP (XEXP (x, 0), 0)) == MULT
14487 && GET_CODE (XEXP (XEXP (x, 0), 1)) == PLUS
14488 && CONSTANT_P (XEXP (x, 1)))
14491 rtx other = NULL_RTX;
14493 if (CONST_INT_P (XEXP (x, 1)))
14495 constant = XEXP (x, 1);
14496 other = XEXP (XEXP (XEXP (x, 0), 1), 1);
14498 else if (CONST_INT_P (XEXP (XEXP (XEXP (x, 0), 1), 1)))
14500 constant = XEXP (XEXP (XEXP (x, 0), 1), 1);
14501 other = XEXP (x, 1);
14509 x = gen_rtx_PLUS (Pmode,
14510 gen_rtx_PLUS (Pmode, XEXP (XEXP (x, 0), 0),
14511 XEXP (XEXP (XEXP (x, 0), 1), 0)),
14512 plus_constant (Pmode, other,
14513 INTVAL (constant)));
14517 if (changed && ix86_legitimate_address_p (mode, x, false))
14520 if (GET_CODE (XEXP (x, 0)) == MULT)
14523 XEXP (x, 0) = copy_addr_to_reg (XEXP (x, 0));
14526 if (GET_CODE (XEXP (x, 1)) == MULT)
14529 XEXP (x, 1) = copy_addr_to_reg (XEXP (x, 1));
14533 && REG_P (XEXP (x, 1))
14534 && REG_P (XEXP (x, 0)))
14537 if (flag_pic && SYMBOLIC_CONST (XEXP (x, 1)))
14540 x = legitimize_pic_address (x, 0);
14543 if (changed && ix86_legitimate_address_p (mode, x, false))
14546 if (REG_P (XEXP (x, 0)))
14548 rtx temp = gen_reg_rtx (Pmode);
14549 rtx val = force_operand (XEXP (x, 1), temp);
14552 val = convert_to_mode (Pmode, val, 1);
14553 emit_move_insn (temp, val);
14556 XEXP (x, 1) = temp;
14560 else if (REG_P (XEXP (x, 1)))
14562 rtx temp = gen_reg_rtx (Pmode);
14563 rtx val = force_operand (XEXP (x, 0), temp);
14566 val = convert_to_mode (Pmode, val, 1);
14567 emit_move_insn (temp, val);
14570 XEXP (x, 0) = temp;
14578 /* Print an integer constant expression in assembler syntax. Addition
14579 and subtraction are the only arithmetic that may appear in these
14580 expressions. FILE is the stdio stream to write to, X is the rtx, and
14581 CODE is the operand print code from the output string. */
14584 output_pic_addr_const (FILE *file, rtx x, int code)
14588 switch (GET_CODE (x))
14591 gcc_assert (flag_pic);
14596 if (TARGET_64BIT || ! TARGET_MACHO_BRANCH_ISLANDS)
14597 output_addr_const (file, x);
14600 const char *name = XSTR (x, 0);
14602 /* Mark the decl as referenced so that cgraph will
14603 output the function. */
14604 if (SYMBOL_REF_DECL (x))
14605 mark_decl_referenced (SYMBOL_REF_DECL (x));
14608 if (MACHOPIC_INDIRECT
14609 && machopic_classify_symbol (x) == MACHOPIC_UNDEFINED_FUNCTION)
14610 name = machopic_indirection_name (x, /*stub_p=*/true);
14612 assemble_name (file, name);
14614 if (!TARGET_MACHO && !(TARGET_64BIT && TARGET_PECOFF)
14615 && code == 'P' && ! SYMBOL_REF_LOCAL_P (x))
14616 fputs ("@PLT", file);
14623 ASM_GENERATE_INTERNAL_LABEL (buf, "L", CODE_LABEL_NUMBER (x));
14624 assemble_name (asm_out_file, buf);
14628 fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (x));
14632 /* This used to output parentheses around the expression,
14633 but that does not work on the 386 (either ATT or BSD assembler). */
14634 output_pic_addr_const (file, XEXP (x, 0), code);
14638 /* We can't handle floating point constants;
14639 TARGET_PRINT_OPERAND must handle them. */
14640 output_operand_lossage ("floating constant misused");
14644 /* Some assemblers need integer constants to appear first. */
14645 if (CONST_INT_P (XEXP (x, 0)))
14647 output_pic_addr_const (file, XEXP (x, 0), code);
14649 output_pic_addr_const (file, XEXP (x, 1), code);
14653 gcc_assert (CONST_INT_P (XEXP (x, 1)));
14654 output_pic_addr_const (file, XEXP (x, 1), code);
14656 output_pic_addr_const (file, XEXP (x, 0), code);
14662 putc (ASSEMBLER_DIALECT == ASM_INTEL ? '(' : '[', file);
14663 output_pic_addr_const (file, XEXP (x, 0), code);
14665 output_pic_addr_const (file, XEXP (x, 1), code);
14667 putc (ASSEMBLER_DIALECT == ASM_INTEL ? ')' : ']', file);
14671 if (XINT (x, 1) == UNSPEC_STACK_CHECK)
14673 bool f = i386_asm_output_addr_const_extra (file, x);
14678 gcc_assert (XVECLEN (x, 0) == 1);
14679 output_pic_addr_const (file, XVECEXP (x, 0, 0), code);
14680 switch (XINT (x, 1))
14683 fputs ("@GOT", file);
14685 case UNSPEC_GOTOFF:
14686 fputs ("@GOTOFF", file);
14688 case UNSPEC_PLTOFF:
14689 fputs ("@PLTOFF", file);
14692 fputs (ASSEMBLER_DIALECT == ASM_ATT ?
14693 "(%rip)" : "[rip]", file);
14695 case UNSPEC_GOTPCREL:
14696 fputs (ASSEMBLER_DIALECT == ASM_ATT ?
14697 "@GOTPCREL(%rip)" : "@GOTPCREL[rip]", file);
14699 case UNSPEC_GOTTPOFF:
14700 /* FIXME: This might be @TPOFF in Sun ld too. */
14701 fputs ("@gottpoff", file);
14704 fputs ("@tpoff", file);
14706 case UNSPEC_NTPOFF:
14708 fputs ("@tpoff", file);
14710 fputs ("@ntpoff", file);
14712 case UNSPEC_DTPOFF:
14713 fputs ("@dtpoff", file);
14715 case UNSPEC_GOTNTPOFF:
14717 fputs (ASSEMBLER_DIALECT == ASM_ATT ?
14718 "@gottpoff(%rip)": "@gottpoff[rip]", file);
14720 fputs ("@gotntpoff", file);
14722 case UNSPEC_INDNTPOFF:
14723 fputs ("@indntpoff", file);
14726 case UNSPEC_MACHOPIC_OFFSET:
14728 machopic_output_function_base_name (file);
14732 output_operand_lossage ("invalid UNSPEC as operand");
14738 output_operand_lossage ("invalid expression as operand");
14742 /* This is called from dwarf2out.c via TARGET_ASM_OUTPUT_DWARF_DTPREL.
14743 We need to emit DTP-relative relocations. */
14745 static void ATTRIBUTE_UNUSED
14746 i386_output_dwarf_dtprel (FILE *file, int size, rtx x)
14748 fputs (ASM_LONG, file);
14749 output_addr_const (file, x);
14750 fputs ("@dtpoff", file);
14756 fputs (", 0", file);
14759 gcc_unreachable ();
14763 /* Return true if X is a representation of the PIC register. This copes
14764 with calls from ix86_find_base_term, where the register might have
14765 been replaced by a cselib value. */
14768 ix86_pic_register_p (rtx x)
14770 if (GET_CODE (x) == VALUE && CSELIB_VAL_PTR (x))
14771 return (pic_offset_table_rtx
14772 && rtx_equal_for_cselib_p (x, pic_offset_table_rtx));
14773 else if (!REG_P (x))
14775 else if (pic_offset_table_rtx)
14777 if (REGNO (x) == REGNO (pic_offset_table_rtx))
14779 if (HARD_REGISTER_P (x)
14780 && !HARD_REGISTER_P (pic_offset_table_rtx)
14781 && ORIGINAL_REGNO (x) == REGNO (pic_offset_table_rtx))
14786 return REGNO (x) == PIC_OFFSET_TABLE_REGNUM;
14789 /* Helper function for ix86_delegitimize_address.
14790 Attempt to delegitimize TLS local-exec accesses. */
14793 ix86_delegitimize_tls_address (rtx orig_x)
14795 rtx x = orig_x, unspec;
14796 struct ix86_address addr;
14798 if (!TARGET_TLS_DIRECT_SEG_REFS)
14802 if (GET_CODE (x) != PLUS || GET_MODE (x) != Pmode)
14804 if (ix86_decompose_address (x, &addr) == 0
14805 || addr.seg != DEFAULT_TLS_SEG_REG
14806 || addr.disp == NULL_RTX
14807 || GET_CODE (addr.disp) != CONST)
14809 unspec = XEXP (addr.disp, 0);
14810 if (GET_CODE (unspec) == PLUS && CONST_INT_P (XEXP (unspec, 1)))
14811 unspec = XEXP (unspec, 0);
14812 if (GET_CODE (unspec) != UNSPEC || XINT (unspec, 1) != UNSPEC_NTPOFF)
14814 x = XVECEXP (unspec, 0, 0);
14815 gcc_assert (GET_CODE (x) == SYMBOL_REF);
14816 if (unspec != XEXP (addr.disp, 0))
14817 x = gen_rtx_PLUS (Pmode, x, XEXP (XEXP (addr.disp, 0), 1));
14820 rtx idx = addr.index;
14821 if (addr.scale != 1)
14822 idx = gen_rtx_MULT (Pmode, idx, GEN_INT (addr.scale));
14823 x = gen_rtx_PLUS (Pmode, idx, x);
14826 x = gen_rtx_PLUS (Pmode, addr.base, x);
14827 if (MEM_P (orig_x))
14828 x = replace_equiv_address_nv (orig_x, x);
14832 /* In the name of slightly smaller debug output, and to cater to
14833 general assembler lossage, recognize PIC+GOTOFF and turn it back
14834 into a direct symbol reference.
14836 On Darwin, this is necessary to avoid a crash, because Darwin
14837 has a different PIC label for each routine but the DWARF debugging
14838 information is not associated with any particular routine, so it's
14839 necessary to remove references to the PIC label from RTL stored by
14840 the DWARF output code. */
14843 ix86_delegitimize_address (rtx x)
14845 rtx orig_x = delegitimize_mem_from_attrs (x);
14846 /* addend is NULL or some rtx if x is something+GOTOFF where
14847 something doesn't include the PIC register. */
14848 rtx addend = NULL_RTX;
14849 /* reg_addend is NULL or a multiple of some register. */
14850 rtx reg_addend = NULL_RTX;
14851 /* const_addend is NULL or a const_int. */
14852 rtx const_addend = NULL_RTX;
14853 /* This is the result, or NULL. */
14854 rtx result = NULL_RTX;
14863 if (GET_CODE (x) == CONST
14864 && GET_CODE (XEXP (x, 0)) == PLUS
14865 && GET_MODE (XEXP (x, 0)) == Pmode
14866 && CONST_INT_P (XEXP (XEXP (x, 0), 1))
14867 && GET_CODE (XEXP (XEXP (x, 0), 0)) == UNSPEC
14868 && XINT (XEXP (XEXP (x, 0), 0), 1) == UNSPEC_PCREL)
14870 rtx x2 = XVECEXP (XEXP (XEXP (x, 0), 0), 0, 0);
14871 x = gen_rtx_PLUS (Pmode, XEXP (XEXP (x, 0), 1), x2);
14872 if (MEM_P (orig_x))
14873 x = replace_equiv_address_nv (orig_x, x);
14877 if (GET_CODE (x) == CONST
14878 && GET_CODE (XEXP (x, 0)) == UNSPEC
14879 && (XINT (XEXP (x, 0), 1) == UNSPEC_GOTPCREL
14880 || XINT (XEXP (x, 0), 1) == UNSPEC_PCREL)
14881 && (MEM_P (orig_x) || XINT (XEXP (x, 0), 1) == UNSPEC_PCREL))
14883 x = XVECEXP (XEXP (x, 0), 0, 0);
14884 if (GET_MODE (orig_x) != GET_MODE (x) && MEM_P (orig_x))
14886 x = simplify_gen_subreg (GET_MODE (orig_x), x,
14894 if (ix86_cmodel != CM_MEDIUM_PIC && ix86_cmodel != CM_LARGE_PIC)
14895 return ix86_delegitimize_tls_address (orig_x);
14897 /* Fall thru into the code shared with -m32 for -mcmodel=large -fpic
14898 and -mcmodel=medium -fpic. */
14901 if (GET_CODE (x) != PLUS
14902 || GET_CODE (XEXP (x, 1)) != CONST)
14903 return ix86_delegitimize_tls_address (orig_x);
14905 if (ix86_pic_register_p (XEXP (x, 0)))
14906 /* %ebx + GOT/GOTOFF */
14908 else if (GET_CODE (XEXP (x, 0)) == PLUS)
14910 /* %ebx + %reg * scale + GOT/GOTOFF */
14911 reg_addend = XEXP (x, 0);
14912 if (ix86_pic_register_p (XEXP (reg_addend, 0)))
14913 reg_addend = XEXP (reg_addend, 1);
14914 else if (ix86_pic_register_p (XEXP (reg_addend, 1)))
14915 reg_addend = XEXP (reg_addend, 0);
14918 reg_addend = NULL_RTX;
14919 addend = XEXP (x, 0);
14923 addend = XEXP (x, 0);
14925 x = XEXP (XEXP (x, 1), 0);
14926 if (GET_CODE (x) == PLUS
14927 && CONST_INT_P (XEXP (x, 1)))
14929 const_addend = XEXP (x, 1);
14933 if (GET_CODE (x) == UNSPEC
14934 && ((XINT (x, 1) == UNSPEC_GOT && MEM_P (orig_x) && !addend)
14935 || (XINT (x, 1) == UNSPEC_GOTOFF && !MEM_P (orig_x))
14936 || (XINT (x, 1) == UNSPEC_PLTOFF && ix86_cmodel == CM_LARGE_PIC
14937 && !MEM_P (orig_x) && !addend)))
14938 result = XVECEXP (x, 0, 0);
14940 if (!TARGET_64BIT && TARGET_MACHO && darwin_local_data_pic (x)
14941 && !MEM_P (orig_x))
14942 result = XVECEXP (x, 0, 0);
14945 return ix86_delegitimize_tls_address (orig_x);
14948 result = gen_rtx_CONST (Pmode, gen_rtx_PLUS (Pmode, result, const_addend));
14950 result = gen_rtx_PLUS (Pmode, reg_addend, result);
14953 /* If the rest of original X doesn't involve the PIC register, add
14954 addend and subtract pic_offset_table_rtx. This can happen e.g.
14956 leal (%ebx, %ecx, 4), %ecx
14958 movl foo@GOTOFF(%ecx), %edx
14959 in which case we return (%ecx - %ebx) + foo
14960 or (%ecx - _GLOBAL_OFFSET_TABLE_) + foo if pseudo_pic_reg
14961 and reload has completed. */
14962 if (pic_offset_table_rtx
14963 && (!reload_completed || !ix86_use_pseudo_pic_reg ()))
14964 result = gen_rtx_PLUS (Pmode, gen_rtx_MINUS (Pmode, copy_rtx (addend),
14965 pic_offset_table_rtx),
14967 else if (pic_offset_table_rtx && !TARGET_MACHO && !TARGET_VXWORKS_RTP)
14969 rtx tmp = gen_rtx_SYMBOL_REF (Pmode, GOT_SYMBOL_NAME);
14970 tmp = gen_rtx_MINUS (Pmode, copy_rtx (addend), tmp);
14971 result = gen_rtx_PLUS (Pmode, tmp, result);
14976 if (GET_MODE (orig_x) != Pmode && MEM_P (orig_x))
14978 result = simplify_gen_subreg (GET_MODE (orig_x), result, Pmode, 0);
14979 if (result == NULL_RTX)
14985 /* If X is a machine specific address (i.e. a symbol or label being
14986 referenced as a displacement from the GOT implemented using an
14987 UNSPEC), then return the base term. Otherwise return X. */
14990 ix86_find_base_term (rtx x)
14996 if (GET_CODE (x) != CONST)
14998 term = XEXP (x, 0);
14999 if (GET_CODE (term) == PLUS
15000 && CONST_INT_P (XEXP (term, 1)))
15001 term = XEXP (term, 0);
15002 if (GET_CODE (term) != UNSPEC
15003 || (XINT (term, 1) != UNSPEC_GOTPCREL
15004 && XINT (term, 1) != UNSPEC_PCREL))
15007 return XVECEXP (term, 0, 0);
15010 return ix86_delegitimize_address (x);
15014 put_condition_code (enum rtx_code code, machine_mode mode, bool reverse,
15015 bool fp, FILE *file)
15017 const char *suffix;
15019 if (mode == CCFPmode || mode == CCFPUmode)
15021 code = ix86_fp_compare_code_to_integer (code);
15025 code = reverse_condition (code);
15076 gcc_assert (mode == CCmode || mode == CCNOmode || mode == CCGCmode);
15080 /* ??? Use "nbe" instead of "a" for fcmov lossage on some assemblers.
15081 Those same assemblers have the same but opposite lossage on cmov. */
15082 if (mode == CCmode)
15083 suffix = fp ? "nbe" : "a";
15085 gcc_unreachable ();
15101 gcc_unreachable ();
15105 if (mode == CCmode)
15107 else if (mode == CCCmode)
15108 suffix = fp ? "b" : "c";
15110 gcc_unreachable ();
15126 gcc_unreachable ();
15130 if (mode == CCmode)
15132 else if (mode == CCCmode)
15133 suffix = fp ? "nb" : "nc";
15135 gcc_unreachable ();
15138 gcc_assert (mode == CCmode || mode == CCGCmode || mode == CCNOmode);
15142 if (mode == CCmode)
15145 gcc_unreachable ();
15148 suffix = fp ? "u" : "p";
15151 suffix = fp ? "nu" : "np";
15154 gcc_unreachable ();
15156 fputs (suffix, file);
15159 /* Print the name of register X to FILE based on its machine mode and number.
15160 If CODE is 'w', pretend the mode is HImode.
15161 If CODE is 'b', pretend the mode is QImode.
15162 If CODE is 'k', pretend the mode is SImode.
15163 If CODE is 'q', pretend the mode is DImode.
15164 If CODE is 'x', pretend the mode is V4SFmode.
15165 If CODE is 't', pretend the mode is V8SFmode.
15166 If CODE is 'g', pretend the mode is V16SFmode.
15167 If CODE is 'h', pretend the reg is the 'high' byte register.
15168 If CODE is 'y', print "st(0)" instead of "st", if the reg is stack op.
15169 If CODE is 'd', duplicate the operand for AVX instruction.
15173 print_reg (rtx x, int code, FILE *file)
15177 unsigned int regno;
15180 if (ASSEMBLER_DIALECT == ASM_ATT)
15185 gcc_assert (TARGET_64BIT);
15186 fputs ("rip", file);
15190 if (code == 'y' && STACK_TOP_P (x))
15192 fputs ("st(0)", file);
15198 else if (code == 'b')
15200 else if (code == 'k')
15202 else if (code == 'q')
15204 else if (code == 'h')
15206 else if (code == 'x')
15208 else if (code == 't')
15210 else if (code == 'g')
15213 msize = GET_MODE_SIZE (GET_MODE (x));
15215 regno = true_regnum (x);
15217 gcc_assert (regno != ARG_POINTER_REGNUM
15218 && regno != FRAME_POINTER_REGNUM
15219 && regno != FLAGS_REG
15220 && regno != FPSR_REG
15221 && regno != FPCR_REG);
15223 duplicated = code == 'd' && TARGET_AVX;
15229 if (LEGACY_INT_REGNO_P (regno))
15230 putc (msize == 8 && TARGET_64BIT ? 'r' : 'e', file);
15235 reg = hi_reg_name[regno];
15238 if (regno >= ARRAY_SIZE (qi_reg_name))
15240 reg = qi_reg_name[regno];
15243 if (regno >= ARRAY_SIZE (qi_high_reg_name))
15245 reg = qi_high_reg_name[regno];
15249 if (SSE_REGNO_P (regno))
15251 gcc_assert (!duplicated);
15252 putc (msize == 32 ? 'y' : 'z', file);
15253 reg = hi_reg_name[regno] + 1;
15258 gcc_unreachable ();
15263 /* Irritatingly, AMD extended registers use
15264 different naming convention: "r%d[bwd]" */
15265 if (REX_INT_REGNO_P (regno))
15267 gcc_assert (TARGET_64BIT);
15271 error ("extended registers have no high halves");
15286 error ("unsupported operand size for extended register");
15294 if (ASSEMBLER_DIALECT == ASM_ATT)
15295 fprintf (file, ", %%%s", reg);
15297 fprintf (file, ", %s", reg);
15301 /* Meaning of CODE:
15302 L,W,B,Q,S,T -- print the opcode suffix for specified size of operand.
15303 C -- print opcode suffix for set/cmov insn.
15304 c -- like C, but print reversed condition
15305 F,f -- likewise, but for floating-point.
15306 O -- if HAVE_AS_IX86_CMOV_SUN_SYNTAX, expand to "w.", "l." or "q.",
15308 R -- print embeded rounding and sae.
15309 r -- print only sae.
15310 z -- print the opcode suffix for the size of the current operand.
15311 Z -- likewise, with special suffixes for x87 instructions.
15312 * -- print a star (in certain assembler syntax)
15313 A -- print an absolute memory reference.
15314 E -- print address with DImode register names if TARGET_64BIT.
15315 w -- print the operand as if it's a "word" (HImode) even if it isn't.
15316 s -- print a shift double count, followed by the assemblers argument
15318 b -- print the QImode name of the register for the indicated operand.
15319 %b0 would print %al if operands[0] is reg 0.
15320 w -- likewise, print the HImode name of the register.
15321 k -- likewise, print the SImode name of the register.
15322 q -- likewise, print the DImode name of the register.
15323 x -- likewise, print the V4SFmode name of the register.
15324 t -- likewise, print the V8SFmode name of the register.
15325 g -- likewise, print the V16SFmode name of the register.
15326 h -- print the QImode name for a "high" register, either ah, bh, ch or dh.
15327 y -- print "st(0)" instead of "st" as a register.
15328 d -- print duplicated register operand for AVX instruction.
15329 D -- print condition for SSE cmp instruction.
15330 P -- if PIC, print an @PLT suffix.
15331 p -- print raw symbol name.
15332 X -- don't print any sort of PIC '@' suffix for a symbol.
15333 & -- print some in-use local-dynamic symbol name.
15334 H -- print a memory address offset by 8; used for sse high-parts
15335 Y -- print condition for XOP pcom* instruction.
15336 + -- print a branch hint as 'cs' or 'ds' prefix
15337 ; -- print a semicolon (after prefixes due to bug in older gas).
15338 ~ -- print "i" if TARGET_AVX2, "f" otherwise.
15339 @ -- print a segment register of thread base pointer load
15340 ^ -- print addr32 prefix if TARGET_64BIT and Pmode != word_mode
15341 ! -- print MPX prefix for jxx/call/ret instructions if required.
15345 ix86_print_operand (FILE *file, rtx x, int code)
15352 switch (ASSEMBLER_DIALECT)
15359 /* Intel syntax. For absolute addresses, registers should not
15360 be surrounded by braces. */
15364 ix86_print_operand (file, x, 0);
15371 gcc_unreachable ();
15374 ix86_print_operand (file, x, 0);
15378 /* Wrap address in an UNSPEC to declare special handling. */
15380 x = gen_rtx_UNSPEC (DImode, gen_rtvec (1, x), UNSPEC_LEA_ADDR);
15382 output_address (x);
15386 if (ASSEMBLER_DIALECT == ASM_ATT)
15391 if (ASSEMBLER_DIALECT == ASM_ATT)
15396 if (ASSEMBLER_DIALECT == ASM_ATT)
15401 if (ASSEMBLER_DIALECT == ASM_ATT)
15406 if (ASSEMBLER_DIALECT == ASM_ATT)
15411 if (ASSEMBLER_DIALECT == ASM_ATT)
15416 #ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
15417 if (ASSEMBLER_DIALECT != ASM_ATT)
15420 switch (GET_MODE_SIZE (GET_MODE (x)))
15435 output_operand_lossage
15436 ("invalid operand size for operand code 'O'");
15445 if (GET_MODE_CLASS (GET_MODE (x)) == MODE_INT)
15447 /* Opcodes don't get size suffixes if using Intel opcodes. */
15448 if (ASSEMBLER_DIALECT == ASM_INTEL)
15451 switch (GET_MODE_SIZE (GET_MODE (x)))
15470 output_operand_lossage
15471 ("invalid operand size for operand code 'z'");
15476 if (GET_MODE_CLASS (GET_MODE (x)) == MODE_FLOAT)
15478 (0, "non-integer operand used with operand code 'z'");
15482 /* 387 opcodes don't get size suffixes if using Intel opcodes. */
15483 if (ASSEMBLER_DIALECT == ASM_INTEL)
15486 if (GET_MODE_CLASS (GET_MODE (x)) == MODE_INT)
15488 switch (GET_MODE_SIZE (GET_MODE (x)))
15491 #ifdef HAVE_AS_IX86_FILDS
15501 #ifdef HAVE_AS_IX86_FILDQ
15504 fputs ("ll", file);
15512 else if (GET_MODE_CLASS (GET_MODE (x)) == MODE_FLOAT)
15514 /* 387 opcodes don't get size suffixes
15515 if the operands are registers. */
15516 if (STACK_REG_P (x))
15519 switch (GET_MODE_SIZE (GET_MODE (x)))
15540 output_operand_lossage
15541 ("invalid operand type used with operand code 'Z'");
15545 output_operand_lossage
15546 ("invalid operand size for operand code 'Z'");
15565 if (CONST_INT_P (x) || ! SHIFT_DOUBLE_OMITS_COUNT)
15567 ix86_print_operand (file, x, 0);
15568 fputs (", ", file);
15573 switch (GET_CODE (x))
15576 fputs ("neq", file);
15579 fputs ("eq", file);
15583 fputs (INTEGRAL_MODE_P (GET_MODE (x)) ? "ge" : "unlt", file);
15587 fputs (INTEGRAL_MODE_P (GET_MODE (x)) ? "gt" : "unle", file);
15591 fputs ("le", file);
15595 fputs ("lt", file);
15598 fputs ("unord", file);
15601 fputs ("ord", file);
15604 fputs ("ueq", file);
15607 fputs ("nlt", file);
15610 fputs ("nle", file);
15613 fputs ("ule", file);
15616 fputs ("ult", file);
15619 fputs ("une", file);
15622 output_operand_lossage ("operand is not a condition code, "
15623 "invalid operand code 'Y'");
15629 /* Little bit of braindamage here. The SSE compare instructions
15630 does use completely different names for the comparisons that the
15631 fp conditional moves. */
15632 switch (GET_CODE (x))
15637 fputs ("eq_us", file);
15641 fputs ("eq", file);
15646 fputs ("nge", file);
15650 fputs ("lt", file);
15655 fputs ("ngt", file);
15659 fputs ("le", file);
15662 fputs ("unord", file);
15667 fputs ("neq_oq", file);
15671 fputs ("neq", file);
15676 fputs ("ge", file);
15680 fputs ("nlt", file);
15685 fputs ("gt", file);
15689 fputs ("nle", file);
15692 fputs ("ord", file);
15695 output_operand_lossage ("operand is not a condition code, "
15696 "invalid operand code 'D'");
15703 #ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
15704 if (ASSEMBLER_DIALECT == ASM_ATT)
15710 if (!COMPARISON_P (x))
15712 output_operand_lossage ("operand is not a condition code, "
15713 "invalid operand code '%c'", code);
15716 put_condition_code (GET_CODE (x), GET_MODE (XEXP (x, 0)),
15717 code == 'c' || code == 'f',
15718 code == 'F' || code == 'f',
15723 if (!offsettable_memref_p (x))
15725 output_operand_lossage ("operand is not an offsettable memory "
15726 "reference, invalid operand code 'H'");
15729 /* It doesn't actually matter what mode we use here, as we're
15730 only going to use this for printing. */
15731 x = adjust_address_nv (x, DImode, 8);
15732 /* Output 'qword ptr' for intel assembler dialect. */
15733 if (ASSEMBLER_DIALECT == ASM_INTEL)
15738 gcc_assert (CONST_INT_P (x));
15740 if (INTVAL (x) & IX86_HLE_ACQUIRE)
15741 #ifdef HAVE_AS_IX86_HLE
15742 fputs ("xacquire ", file);
15744 fputs ("\n" ASM_BYTE "0xf2\n\t", file);
15746 else if (INTVAL (x) & IX86_HLE_RELEASE)
15747 #ifdef HAVE_AS_IX86_HLE
15748 fputs ("xrelease ", file);
15750 fputs ("\n" ASM_BYTE "0xf3\n\t", file);
15752 /* We do not want to print value of the operand. */
15756 if (x == const0_rtx || x == CONST0_RTX (GET_MODE (x)))
15757 fputs ("{z}", file);
15761 gcc_assert (CONST_INT_P (x));
15762 gcc_assert (INTVAL (x) == ROUND_SAE);
15764 if (ASSEMBLER_DIALECT == ASM_INTEL)
15765 fputs (", ", file);
15767 fputs ("{sae}", file);
15769 if (ASSEMBLER_DIALECT == ASM_ATT)
15770 fputs (", ", file);
15775 gcc_assert (CONST_INT_P (x));
15777 if (ASSEMBLER_DIALECT == ASM_INTEL)
15778 fputs (", ", file);
15780 switch (INTVAL (x))
15782 case ROUND_NEAREST_INT | ROUND_SAE:
15783 fputs ("{rn-sae}", file);
15785 case ROUND_NEG_INF | ROUND_SAE:
15786 fputs ("{rd-sae}", file);
15788 case ROUND_POS_INF | ROUND_SAE:
15789 fputs ("{ru-sae}", file);
15791 case ROUND_ZERO | ROUND_SAE:
15792 fputs ("{rz-sae}", file);
15795 gcc_unreachable ();
15798 if (ASSEMBLER_DIALECT == ASM_ATT)
15799 fputs (", ", file);
15804 if (ASSEMBLER_DIALECT == ASM_ATT)
15810 const char *name = get_some_local_dynamic_name ();
15812 output_operand_lossage ("'%%&' used without any "
15813 "local dynamic TLS references");
15815 assemble_name (file, name);
15824 || optimize_function_for_size_p (cfun)
15825 || !TARGET_BRANCH_PREDICTION_HINTS)
15828 x = find_reg_note (current_output_insn, REG_BR_PROB, 0);
15831 int pred_val = XINT (x, 0);
15833 if (pred_val < REG_BR_PROB_BASE * 45 / 100
15834 || pred_val > REG_BR_PROB_BASE * 55 / 100)
15836 bool taken = pred_val > REG_BR_PROB_BASE / 2;
15838 = final_forward_branch_p (current_output_insn) == 0;
15840 /* Emit hints only in the case default branch prediction
15841 heuristics would fail. */
15842 if (taken != cputaken)
15844 /* We use 3e (DS) prefix for taken branches and
15845 2e (CS) prefix for not taken branches. */
15847 fputs ("ds ; ", file);
15849 fputs ("cs ; ", file);
15857 #ifndef HAVE_AS_IX86_REP_LOCK_PREFIX
15863 if (ASSEMBLER_DIALECT == ASM_ATT)
15866 /* The kernel uses a different segment register for performance
15867 reasons; a system call would not have to trash the userspace
15868 segment register, which would be expensive. */
15869 if (TARGET_64BIT && ix86_cmodel != CM_KERNEL)
15870 fputs ("fs", file);
15872 fputs ("gs", file);
15876 putc (TARGET_AVX2 ? 'i' : 'f', file);
15880 if (TARGET_64BIT && Pmode != word_mode)
15881 fputs ("addr32 ", file);
15885 if (ix86_bnd_prefixed_insn_p (current_output_insn))
15886 fputs ("bnd ", file);
15890 output_operand_lossage ("invalid operand code '%c'", code);
15895 print_reg (x, code, file);
15897 else if (MEM_P (x))
15899 /* No `byte ptr' prefix for call instructions or BLKmode operands. */
15900 if (ASSEMBLER_DIALECT == ASM_INTEL && code != 'X' && code != 'P'
15901 && GET_MODE (x) != BLKmode)
15904 switch (GET_MODE_SIZE (GET_MODE (x)))
15906 case 1: size = "BYTE"; break;
15907 case 2: size = "WORD"; break;
15908 case 4: size = "DWORD"; break;
15909 case 8: size = "QWORD"; break;
15910 case 12: size = "TBYTE"; break;
15912 if (GET_MODE (x) == XFmode)
15917 case 32: size = "YMMWORD"; break;
15918 case 64: size = "ZMMWORD"; break;
15920 gcc_unreachable ();
15923 /* Check for explicit size override (codes 'b', 'w', 'k',
15927 else if (code == 'w')
15929 else if (code == 'k')
15931 else if (code == 'q')
15933 else if (code == 'x')
15936 fputs (size, file);
15937 fputs (" PTR ", file);
15941 /* Avoid (%rip) for call operands. */
15942 if (CONSTANT_ADDRESS_P (x) && code == 'P'
15943 && !CONST_INT_P (x))
15944 output_addr_const (file, x);
15945 else if (this_is_asm_operands && ! address_operand (x, VOIDmode))
15946 output_operand_lossage ("invalid constraints for operand");
15948 output_address (x);
15951 else if (CONST_DOUBLE_P (x) && GET_MODE (x) == SFmode)
15956 REAL_VALUE_FROM_CONST_DOUBLE (r, x);
15957 REAL_VALUE_TO_TARGET_SINGLE (r, l);
15959 if (ASSEMBLER_DIALECT == ASM_ATT)
15961 /* Sign extend 32bit SFmode immediate to 8 bytes. */
15963 fprintf (file, "0x%08" HOST_LONG_LONG_FORMAT "x",
15964 (unsigned long long) (int) l);
15966 fprintf (file, "0x%08x", (unsigned int) l);
15969 else if (CONST_DOUBLE_P (x) && GET_MODE (x) == DFmode)
15974 REAL_VALUE_FROM_CONST_DOUBLE (r, x);
15975 REAL_VALUE_TO_TARGET_DOUBLE (r, l);
15977 if (ASSEMBLER_DIALECT == ASM_ATT)
15979 fprintf (file, "0x%lx%08lx", l[1] & 0xffffffff, l[0] & 0xffffffff);
15982 /* These float cases don't actually occur as immediate operands. */
15983 else if (CONST_DOUBLE_P (x) && GET_MODE (x) == XFmode)
15987 real_to_decimal (dstr, CONST_DOUBLE_REAL_VALUE (x), sizeof (dstr), 0, 1);
15988 fputs (dstr, file);
15993 /* We have patterns that allow zero sets of memory, for instance.
15994 In 64-bit mode, we should probably support all 8-byte vectors,
15995 since we can in fact encode that into an immediate. */
15996 if (GET_CODE (x) == CONST_VECTOR)
15998 gcc_assert (x == CONST0_RTX (GET_MODE (x)));
16002 if (code != 'P' && code != 'p')
16004 if (CONST_INT_P (x))
16006 if (ASSEMBLER_DIALECT == ASM_ATT)
16009 else if (GET_CODE (x) == CONST || GET_CODE (x) == SYMBOL_REF
16010 || GET_CODE (x) == LABEL_REF)
16012 if (ASSEMBLER_DIALECT == ASM_ATT)
16015 fputs ("OFFSET FLAT:", file);
16018 if (CONST_INT_P (x))
16019 fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (x));
16020 else if (flag_pic || MACHOPIC_INDIRECT)
16021 output_pic_addr_const (file, x, code);
16023 output_addr_const (file, x);
16028 ix86_print_operand_punct_valid_p (unsigned char code)
16030 return (code == '@' || code == '*' || code == '+' || code == '&'
16031 || code == ';' || code == '~' || code == '^' || code == '!');
16034 /* Print a memory operand whose address is ADDR. */
16037 ix86_print_operand_address (FILE *file, rtx addr)
16039 struct ix86_address parts;
16040 rtx base, index, disp;
16046 if (GET_CODE (addr) == UNSPEC && XINT (addr, 1) == UNSPEC_VSIBADDR)
16048 ok = ix86_decompose_address (XVECEXP (addr, 0, 0), &parts);
16049 gcc_assert (parts.index == NULL_RTX);
16050 parts.index = XVECEXP (addr, 0, 1);
16051 parts.scale = INTVAL (XVECEXP (addr, 0, 2));
16052 addr = XVECEXP (addr, 0, 0);
16055 else if (GET_CODE (addr) == UNSPEC && XINT (addr, 1) == UNSPEC_LEA_ADDR)
16057 gcc_assert (TARGET_64BIT);
16058 ok = ix86_decompose_address (XVECEXP (addr, 0, 0), &parts);
16061 else if (GET_CODE (addr) == UNSPEC && XINT (addr, 1) == UNSPEC_BNDMK_ADDR)
16063 ok = ix86_decompose_address (XVECEXP (addr, 0, 1), &parts);
16064 gcc_assert (parts.base == NULL_RTX || parts.index == NULL_RTX);
16065 if (parts.base != NULL_RTX)
16067 parts.index = parts.base;
16070 parts.base = XVECEXP (addr, 0, 0);
16071 addr = XVECEXP (addr, 0, 0);
16073 else if (GET_CODE (addr) == UNSPEC && XINT (addr, 1) == UNSPEC_BNDLDX_ADDR)
16075 ok = ix86_decompose_address (XVECEXP (addr, 0, 0), &parts);
16076 gcc_assert (parts.index == NULL_RTX);
16077 parts.index = XVECEXP (addr, 0, 1);
16078 addr = XVECEXP (addr, 0, 0);
16081 ok = ix86_decompose_address (addr, &parts);
16086 index = parts.index;
16088 scale = parts.scale;
16096 if (ASSEMBLER_DIALECT == ASM_ATT)
16098 fputs ((parts.seg == SEG_FS ? "fs:" : "gs:"), file);
16101 gcc_unreachable ();
16104 /* Use one byte shorter RIP relative addressing for 64bit mode. */
16105 if (TARGET_64BIT && !base && !index)
16109 if (GET_CODE (disp) == CONST
16110 && GET_CODE (XEXP (disp, 0)) == PLUS
16111 && CONST_INT_P (XEXP (XEXP (disp, 0), 1)))
16112 symbol = XEXP (XEXP (disp, 0), 0);
16114 if (GET_CODE (symbol) == LABEL_REF
16115 || (GET_CODE (symbol) == SYMBOL_REF
16116 && SYMBOL_REF_TLS_MODEL (symbol) == 0))
16119 if (!base && !index)
16121 /* Displacement only requires special attention. */
16123 if (CONST_INT_P (disp))
16125 if (ASSEMBLER_DIALECT == ASM_INTEL && parts.seg == SEG_DEFAULT)
16126 fputs ("ds:", file);
16127 fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (disp));
16130 output_pic_addr_const (file, disp, 0);
16132 output_addr_const (file, disp);
16136 /* Print SImode register names to force addr32 prefix. */
16137 if (SImode_address_operand (addr, VOIDmode))
16139 #ifdef ENABLE_CHECKING
16140 gcc_assert (TARGET_64BIT);
16141 switch (GET_CODE (addr))
16144 gcc_assert (GET_MODE (addr) == SImode);
16145 gcc_assert (GET_MODE (SUBREG_REG (addr)) == DImode);
16149 gcc_assert (GET_MODE (addr) == DImode);
16152 gcc_unreachable ();
16155 gcc_assert (!code);
16161 && CONST_INT_P (disp)
16162 && INTVAL (disp) < -16*1024*1024)
16164 /* X32 runs in 64-bit mode, where displacement, DISP, in
16165 address DISP(%r64), is encoded as 32-bit immediate sign-
16166 extended from 32-bit to 64-bit. For -0x40000300(%r64),
16167 address is %r64 + 0xffffffffbffffd00. When %r64 <
16168 0x40000300, like 0x37ffe064, address is 0xfffffffff7ffdd64,
16169 which is invalid for x32. The correct address is %r64
16170 - 0x40000300 == 0xf7ffdd64. To properly encode
16171 -0x40000300(%r64) for x32, we zero-extend negative
16172 displacement by forcing addr32 prefix which truncates
16173 0xfffffffff7ffdd64 to 0xf7ffdd64. In theory, we should
16174 zero-extend all negative displacements, including -1(%rsp).
16175 However, for small negative displacements, sign-extension
16176 won't cause overflow. We only zero-extend negative
16177 displacements if they < -16*1024*1024, which is also used
16178 to check legitimate address displacements for PIC. */
16182 if (ASSEMBLER_DIALECT == ASM_ATT)
16187 output_pic_addr_const (file, disp, 0);
16188 else if (GET_CODE (disp) == LABEL_REF)
16189 output_asm_label (disp);
16191 output_addr_const (file, disp);
16196 print_reg (base, code, file);
16200 print_reg (index, vsib ? 0 : code, file);
16201 if (scale != 1 || vsib)
16202 fprintf (file, ",%d", scale);
16208 rtx offset = NULL_RTX;
16212 /* Pull out the offset of a symbol; print any symbol itself. */
16213 if (GET_CODE (disp) == CONST
16214 && GET_CODE (XEXP (disp, 0)) == PLUS
16215 && CONST_INT_P (XEXP (XEXP (disp, 0), 1)))
16217 offset = XEXP (XEXP (disp, 0), 1);
16218 disp = gen_rtx_CONST (VOIDmode,
16219 XEXP (XEXP (disp, 0), 0));
16223 output_pic_addr_const (file, disp, 0);
16224 else if (GET_CODE (disp) == LABEL_REF)
16225 output_asm_label (disp);
16226 else if (CONST_INT_P (disp))
16229 output_addr_const (file, disp);
16235 print_reg (base, code, file);
16238 if (INTVAL (offset) >= 0)
16240 fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (offset));
16244 fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (offset));
16251 print_reg (index, vsib ? 0 : code, file);
16252 if (scale != 1 || vsib)
16253 fprintf (file, "*%d", scale);
16260 /* Implementation of TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA. */
16263 i386_asm_output_addr_const_extra (FILE *file, rtx x)
16267 if (GET_CODE (x) != UNSPEC)
16270 op = XVECEXP (x, 0, 0);
16271 switch (XINT (x, 1))
16273 case UNSPEC_GOTTPOFF:
16274 output_addr_const (file, op);
16275 /* FIXME: This might be @TPOFF in Sun ld. */
16276 fputs ("@gottpoff", file);
16279 output_addr_const (file, op);
16280 fputs ("@tpoff", file);
16282 case UNSPEC_NTPOFF:
16283 output_addr_const (file, op);
16285 fputs ("@tpoff", file);
16287 fputs ("@ntpoff", file);
16289 case UNSPEC_DTPOFF:
16290 output_addr_const (file, op);
16291 fputs ("@dtpoff", file);
16293 case UNSPEC_GOTNTPOFF:
16294 output_addr_const (file, op);
16296 fputs (ASSEMBLER_DIALECT == ASM_ATT ?
16297 "@gottpoff(%rip)" : "@gottpoff[rip]", file);
16299 fputs ("@gotntpoff", file);
16301 case UNSPEC_INDNTPOFF:
16302 output_addr_const (file, op);
16303 fputs ("@indntpoff", file);
16306 case UNSPEC_MACHOPIC_OFFSET:
16307 output_addr_const (file, op);
16309 machopic_output_function_base_name (file);
16313 case UNSPEC_STACK_CHECK:
16317 gcc_assert (flag_split_stack);
16319 #ifdef TARGET_THREAD_SPLIT_STACK_OFFSET
16320 offset = TARGET_THREAD_SPLIT_STACK_OFFSET;
16322 gcc_unreachable ();
16325 fprintf (file, "%s:%d", TARGET_64BIT ? "%fs" : "%gs", offset);
16336 /* Split one or more double-mode RTL references into pairs of half-mode
16337 references. The RTL can be REG, offsettable MEM, integer constant, or
16338 CONST_DOUBLE. "operands" is a pointer to an array of double-mode RTLs to
16339 split and "num" is its length. lo_half and hi_half are output arrays
16340 that parallel "operands". */
16343 split_double_mode (machine_mode mode, rtx operands[],
16344 int num, rtx lo_half[], rtx hi_half[])
16346 machine_mode half_mode;
16352 half_mode = DImode;
16355 half_mode = SImode;
16358 gcc_unreachable ();
16361 byte = GET_MODE_SIZE (half_mode);
16365 rtx op = operands[num];
16367 /* simplify_subreg refuse to split volatile memory addresses,
16368 but we still have to handle it. */
16371 lo_half[num] = adjust_address (op, half_mode, 0);
16372 hi_half[num] = adjust_address (op, half_mode, byte);
16376 lo_half[num] = simplify_gen_subreg (half_mode, op,
16377 GET_MODE (op) == VOIDmode
16378 ? mode : GET_MODE (op), 0);
16379 hi_half[num] = simplify_gen_subreg (half_mode, op,
16380 GET_MODE (op) == VOIDmode
16381 ? mode : GET_MODE (op), byte);
16386 /* Output code to perform a 387 binary operation in INSN, one of PLUS,
16387 MINUS, MULT or DIV. OPERANDS are the insn operands, where operands[3]
16388 is the expression of the binary operation. The output may either be
16389 emitted here, or returned to the caller, like all output_* functions.
16391 There is no guarantee that the operands are the same mode, as they
16392 might be within FLOAT or FLOAT_EXTEND expressions. */
16394 #ifndef SYSV386_COMPAT
16395 /* Set to 1 for compatibility with brain-damaged assemblers. No-one
16396 wants to fix the assemblers because that causes incompatibility
16397 with gcc. No-one wants to fix gcc because that causes
16398 incompatibility with assemblers... You can use the option of
16399 -DSYSV386_COMPAT=0 if you recompile both gcc and gas this way. */
16400 #define SYSV386_COMPAT 1
16404 output_387_binary_op (rtx insn, rtx *operands)
16406 static char buf[40];
16409 int is_sse = SSE_REG_P (operands[0]) || SSE_REG_P (operands[1]) || SSE_REG_P (operands[2]);
16411 #ifdef ENABLE_CHECKING
16412 /* Even if we do not want to check the inputs, this documents input
16413 constraints. Which helps in understanding the following code. */
16414 if (STACK_REG_P (operands[0])
16415 && ((REG_P (operands[1])
16416 && REGNO (operands[0]) == REGNO (operands[1])
16417 && (STACK_REG_P (operands[2]) || MEM_P (operands[2])))
16418 || (REG_P (operands[2])
16419 && REGNO (operands[0]) == REGNO (operands[2])
16420 && (STACK_REG_P (operands[1]) || MEM_P (operands[1]))))
16421 && (STACK_TOP_P (operands[1]) || STACK_TOP_P (operands[2])))
16424 gcc_assert (is_sse);
16427 switch (GET_CODE (operands[3]))
16430 if (GET_MODE_CLASS (GET_MODE (operands[1])) == MODE_INT
16431 || GET_MODE_CLASS (GET_MODE (operands[2])) == MODE_INT)
16439 if (GET_MODE_CLASS (GET_MODE (operands[1])) == MODE_INT
16440 || GET_MODE_CLASS (GET_MODE (operands[2])) == MODE_INT)
16448 if (GET_MODE_CLASS (GET_MODE (operands[1])) == MODE_INT
16449 || GET_MODE_CLASS (GET_MODE (operands[2])) == MODE_INT)
16457 if (GET_MODE_CLASS (GET_MODE (operands[1])) == MODE_INT
16458 || GET_MODE_CLASS (GET_MODE (operands[2])) == MODE_INT)
16466 gcc_unreachable ();
16473 strcpy (buf, ssep);
16474 if (GET_MODE (operands[0]) == SFmode)
16475 strcat (buf, "ss\t{%2, %1, %0|%0, %1, %2}");
16477 strcat (buf, "sd\t{%2, %1, %0|%0, %1, %2}");
16481 strcpy (buf, ssep + 1);
16482 if (GET_MODE (operands[0]) == SFmode)
16483 strcat (buf, "ss\t{%2, %0|%0, %2}");
16485 strcat (buf, "sd\t{%2, %0|%0, %2}");
16491 switch (GET_CODE (operands[3]))
16495 if (REG_P (operands[2]) && REGNO (operands[0]) == REGNO (operands[2]))
16496 std::swap (operands[1], operands[2]);
16498 /* know operands[0] == operands[1]. */
16500 if (MEM_P (operands[2]))
16506 if (find_regno_note (insn, REG_DEAD, REGNO (operands[2])))
16508 if (STACK_TOP_P (operands[0]))
16509 /* How is it that we are storing to a dead operand[2]?
16510 Well, presumably operands[1] is dead too. We can't
16511 store the result to st(0) as st(0) gets popped on this
16512 instruction. Instead store to operands[2] (which I
16513 think has to be st(1)). st(1) will be popped later.
16514 gcc <= 2.8.1 didn't have this check and generated
16515 assembly code that the Unixware assembler rejected. */
16516 p = "p\t{%0, %2|%2, %0}"; /* st(1) = st(0) op st(1); pop */
16518 p = "p\t{%2, %0|%0, %2}"; /* st(r1) = st(r1) op st(0); pop */
16522 if (STACK_TOP_P (operands[0]))
16523 p = "\t{%y2, %0|%0, %y2}"; /* st(0) = st(0) op st(r2) */
16525 p = "\t{%2, %0|%0, %2}"; /* st(r1) = st(r1) op st(0) */
16530 if (MEM_P (operands[1]))
16536 if (MEM_P (operands[2]))
16542 if (find_regno_note (insn, REG_DEAD, REGNO (operands[2])))
16545 /* The SystemV/386 SVR3.2 assembler, and probably all AT&T
16546 derived assemblers, confusingly reverse the direction of
16547 the operation for fsub{r} and fdiv{r} when the
16548 destination register is not st(0). The Intel assembler
16549 doesn't have this brain damage. Read !SYSV386_COMPAT to
16550 figure out what the hardware really does. */
16551 if (STACK_TOP_P (operands[0]))
16552 p = "{p\t%0, %2|rp\t%2, %0}";
16554 p = "{rp\t%2, %0|p\t%0, %2}";
16556 if (STACK_TOP_P (operands[0]))
16557 /* As above for fmul/fadd, we can't store to st(0). */
16558 p = "rp\t{%0, %2|%2, %0}"; /* st(1) = st(0) op st(1); pop */
16560 p = "p\t{%2, %0|%0, %2}"; /* st(r1) = st(r1) op st(0); pop */
16565 if (find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
16568 if (STACK_TOP_P (operands[0]))
16569 p = "{rp\t%0, %1|p\t%1, %0}";
16571 p = "{p\t%1, %0|rp\t%0, %1}";
16573 if (STACK_TOP_P (operands[0]))
16574 p = "p\t{%0, %1|%1, %0}"; /* st(1) = st(1) op st(0); pop */
16576 p = "rp\t{%1, %0|%0, %1}"; /* st(r2) = st(0) op st(r2); pop */
16581 if (STACK_TOP_P (operands[0]))
16583 if (STACK_TOP_P (operands[1]))
16584 p = "\t{%y2, %0|%0, %y2}"; /* st(0) = st(0) op st(r2) */
16586 p = "r\t{%y1, %0|%0, %y1}"; /* st(0) = st(r1) op st(0) */
16589 else if (STACK_TOP_P (operands[1]))
16592 p = "{\t%1, %0|r\t%0, %1}";
16594 p = "r\t{%1, %0|%0, %1}"; /* st(r2) = st(0) op st(r2) */
16600 p = "{r\t%2, %0|\t%0, %2}";
16602 p = "\t{%2, %0|%0, %2}"; /* st(r1) = st(r1) op st(0) */
16608 gcc_unreachable ();
16615 /* Check if a 256bit AVX register is referenced inside of EXP. */
16618 ix86_check_avx256_register (const_rtx exp)
16620 if (GET_CODE (exp) == SUBREG)
16621 exp = SUBREG_REG (exp);
16623 return (REG_P (exp)
16624 && VALID_AVX256_REG_OR_OI_MODE (GET_MODE (exp)));
16627 /* Return needed mode for entity in optimize_mode_switching pass. */
16630 ix86_avx_u128_mode_needed (rtx_insn *insn)
16636 /* Needed mode is set to AVX_U128_CLEAN if there are
16637 no 256bit modes used in function arguments. */
16638 for (link = CALL_INSN_FUNCTION_USAGE (insn);
16640 link = XEXP (link, 1))
16642 if (GET_CODE (XEXP (link, 0)) == USE)
16644 rtx arg = XEXP (XEXP (link, 0), 0);
16646 if (ix86_check_avx256_register (arg))
16647 return AVX_U128_DIRTY;
16651 return AVX_U128_CLEAN;
16654 /* Require DIRTY mode if a 256bit AVX register is referenced. Hardware
16655 changes state only when a 256bit register is written to, but we need
16656 to prevent the compiler from moving optimal insertion point above
16657 eventual read from 256bit register. */
16658 subrtx_iterator::array_type array;
16659 FOR_EACH_SUBRTX (iter, array, PATTERN (insn), NONCONST)
16660 if (ix86_check_avx256_register (*iter))
16661 return AVX_U128_DIRTY;
16663 return AVX_U128_ANY;
16666 /* Return mode that i387 must be switched into
16667 prior to the execution of insn. */
16670 ix86_i387_mode_needed (int entity, rtx_insn *insn)
16672 enum attr_i387_cw mode;
16674 /* The mode UNINITIALIZED is used to store control word after a
16675 function call or ASM pattern. The mode ANY specify that function
16676 has no requirements on the control word and make no changes in the
16677 bits we are interested in. */
16680 || (NONJUMP_INSN_P (insn)
16681 && (asm_noperands (PATTERN (insn)) >= 0
16682 || GET_CODE (PATTERN (insn)) == ASM_INPUT)))
16683 return I387_CW_UNINITIALIZED;
16685 if (recog_memoized (insn) < 0)
16686 return I387_CW_ANY;
16688 mode = get_attr_i387_cw (insn);
16693 if (mode == I387_CW_TRUNC)
16698 if (mode == I387_CW_FLOOR)
16703 if (mode == I387_CW_CEIL)
16708 if (mode == I387_CW_MASK_PM)
16713 gcc_unreachable ();
16716 return I387_CW_ANY;
16719 /* Return mode that entity must be switched into
16720 prior to the execution of insn. */
16723 ix86_mode_needed (int entity, rtx_insn *insn)
16728 return ix86_avx_u128_mode_needed (insn);
16733 return ix86_i387_mode_needed (entity, insn);
16735 gcc_unreachable ();
16740 /* Check if a 256bit AVX register is referenced in stores. */
16743 ix86_check_avx256_stores (rtx dest, const_rtx, void *data)
16745 if (ix86_check_avx256_register (dest))
16747 bool *used = (bool *) data;
16752 /* Calculate mode of upper 128bit AVX registers after the insn. */
16755 ix86_avx_u128_mode_after (int mode, rtx_insn *insn)
16757 rtx pat = PATTERN (insn);
16759 if (vzeroupper_operation (pat, VOIDmode)
16760 || vzeroall_operation (pat, VOIDmode))
16761 return AVX_U128_CLEAN;
16763 /* We know that state is clean after CALL insn if there are no
16764 256bit registers used in the function return register. */
16767 bool avx_reg256_found = false;
16768 note_stores (pat, ix86_check_avx256_stores, &avx_reg256_found);
16770 return avx_reg256_found ? AVX_U128_DIRTY : AVX_U128_CLEAN;
16773 /* Otherwise, return current mode. Remember that if insn
16774 references AVX 256bit registers, the mode was already changed
16775 to DIRTY from MODE_NEEDED. */
16779 /* Return the mode that an insn results in. */
16782 ix86_mode_after (int entity, int mode, rtx_insn *insn)
16787 return ix86_avx_u128_mode_after (mode, insn);
16794 gcc_unreachable ();
16799 ix86_avx_u128_mode_entry (void)
16803 /* Entry mode is set to AVX_U128_DIRTY if there are
16804 256bit modes used in function arguments. */
16805 for (arg = DECL_ARGUMENTS (current_function_decl); arg;
16806 arg = TREE_CHAIN (arg))
16808 rtx incoming = DECL_INCOMING_RTL (arg);
16810 if (incoming && ix86_check_avx256_register (incoming))
16811 return AVX_U128_DIRTY;
16814 return AVX_U128_CLEAN;
16817 /* Return a mode that ENTITY is assumed to be
16818 switched to at function entry. */
16821 ix86_mode_entry (int entity)
16826 return ix86_avx_u128_mode_entry ();
16831 return I387_CW_ANY;
16833 gcc_unreachable ();
16838 ix86_avx_u128_mode_exit (void)
16840 rtx reg = crtl->return_rtx;
16842 /* Exit mode is set to AVX_U128_DIRTY if there are
16843 256bit modes used in the function return register. */
16844 if (reg && ix86_check_avx256_register (reg))
16845 return AVX_U128_DIRTY;
16847 return AVX_U128_CLEAN;
16850 /* Return a mode that ENTITY is assumed to be
16851 switched to at function exit. */
16854 ix86_mode_exit (int entity)
16859 return ix86_avx_u128_mode_exit ();
16864 return I387_CW_ANY;
16866 gcc_unreachable ();
16871 ix86_mode_priority (int, int n)
16876 /* Output code to initialize control word copies used by trunc?f?i and
16877 rounding patterns. CURRENT_MODE is set to current control word,
16878 while NEW_MODE is set to new control word. */
16881 emit_i387_cw_initialization (int mode)
16883 rtx stored_mode = assign_386_stack_local (HImode, SLOT_CW_STORED);
16886 enum ix86_stack_slot slot;
16888 rtx reg = gen_reg_rtx (HImode);
16890 emit_insn (gen_x86_fnstcw_1 (stored_mode));
16891 emit_move_insn (reg, copy_rtx (stored_mode));
16893 if (TARGET_64BIT || TARGET_PARTIAL_REG_STALL
16894 || optimize_insn_for_size_p ())
16898 case I387_CW_TRUNC:
16899 /* round toward zero (truncate) */
16900 emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0c00)));
16901 slot = SLOT_CW_TRUNC;
16904 case I387_CW_FLOOR:
16905 /* round down toward -oo */
16906 emit_insn (gen_andhi3 (reg, reg, GEN_INT (~0x0c00)));
16907 emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0400)));
16908 slot = SLOT_CW_FLOOR;
16912 /* round up toward +oo */
16913 emit_insn (gen_andhi3 (reg, reg, GEN_INT (~0x0c00)));
16914 emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0800)));
16915 slot = SLOT_CW_CEIL;
16918 case I387_CW_MASK_PM:
16919 /* mask precision exception for nearbyint() */
16920 emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0020)));
16921 slot = SLOT_CW_MASK_PM;
16925 gcc_unreachable ();
16932 case I387_CW_TRUNC:
16933 /* round toward zero (truncate) */
16934 emit_insn (gen_movsi_insv_1 (reg, GEN_INT (0xc)));
16935 slot = SLOT_CW_TRUNC;
16938 case I387_CW_FLOOR:
16939 /* round down toward -oo */
16940 emit_insn (gen_movsi_insv_1 (reg, GEN_INT (0x4)));
16941 slot = SLOT_CW_FLOOR;
16945 /* round up toward +oo */
16946 emit_insn (gen_movsi_insv_1 (reg, GEN_INT (0x8)));
16947 slot = SLOT_CW_CEIL;
16950 case I387_CW_MASK_PM:
16951 /* mask precision exception for nearbyint() */
16952 emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0020)));
16953 slot = SLOT_CW_MASK_PM;
16957 gcc_unreachable ();
16961 gcc_assert (slot < MAX_386_STACK_LOCALS);
16963 new_mode = assign_386_stack_local (HImode, slot);
16964 emit_move_insn (new_mode, reg);
16967 /* Emit vzeroupper. */
16970 ix86_avx_emit_vzeroupper (HARD_REG_SET regs_live)
16974 /* Cancel automatic vzeroupper insertion if there are
16975 live call-saved SSE registers at the insertion point. */
16977 for (i = FIRST_SSE_REG; i <= LAST_SSE_REG; i++)
16978 if (TEST_HARD_REG_BIT (regs_live, i) && !call_used_regs[i])
16982 for (i = FIRST_REX_SSE_REG; i <= LAST_REX_SSE_REG; i++)
16983 if (TEST_HARD_REG_BIT (regs_live, i) && !call_used_regs[i])
16986 emit_insn (gen_avx_vzeroupper ());
16989 /* Generate one or more insns to set ENTITY to MODE. */
16991 /* Generate one or more insns to set ENTITY to MODE. HARD_REG_LIVE
16992 is the set of hard registers live at the point where the insn(s)
16993 are to be inserted. */
16996 ix86_emit_mode_set (int entity, int mode, int prev_mode ATTRIBUTE_UNUSED,
16997 HARD_REG_SET regs_live)
17002 if (mode == AVX_U128_CLEAN)
17003 ix86_avx_emit_vzeroupper (regs_live);
17009 if (mode != I387_CW_ANY
17010 && mode != I387_CW_UNINITIALIZED)
17011 emit_i387_cw_initialization (mode);
17014 gcc_unreachable ();
17018 /* Output code for INSN to convert a float to a signed int. OPERANDS
17019 are the insn operands. The output may be [HSD]Imode and the input
17020 operand may be [SDX]Fmode. */
17023 output_fix_trunc (rtx_insn *insn, rtx *operands, bool fisttp)
17025 int stack_top_dies = find_regno_note (insn, REG_DEAD, FIRST_STACK_REG) != 0;
17026 int dimode_p = GET_MODE (operands[0]) == DImode;
17027 int round_mode = get_attr_i387_cw (insn);
17029 /* Jump through a hoop or two for DImode, since the hardware has no
17030 non-popping instruction. We used to do this a different way, but
17031 that was somewhat fragile and broke with post-reload splitters. */
17032 if ((dimode_p || fisttp) && !stack_top_dies)
17033 output_asm_insn ("fld\t%y1", operands);
17035 gcc_assert (STACK_TOP_P (operands[1]));
17036 gcc_assert (MEM_P (operands[0]));
17037 gcc_assert (GET_MODE (operands[1]) != TFmode);
17040 output_asm_insn ("fisttp%Z0\t%0", operands);
17043 if (round_mode != I387_CW_ANY)
17044 output_asm_insn ("fldcw\t%3", operands);
17045 if (stack_top_dies || dimode_p)
17046 output_asm_insn ("fistp%Z0\t%0", operands);
17048 output_asm_insn ("fist%Z0\t%0", operands);
17049 if (round_mode != I387_CW_ANY)
17050 output_asm_insn ("fldcw\t%2", operands);
17056 /* Output code for x87 ffreep insn. The OPNO argument, which may only
17057 have the values zero or one, indicates the ffreep insn's operand
17058 from the OPERANDS array. */
17060 static const char *
17061 output_387_ffreep (rtx *operands ATTRIBUTE_UNUSED, int opno)
17063 if (TARGET_USE_FFREEP)
17064 #ifdef HAVE_AS_IX86_FFREEP
17065 return opno ? "ffreep\t%y1" : "ffreep\t%y0";
17068 static char retval[32];
17069 int regno = REGNO (operands[opno]);
17071 gcc_assert (STACK_REGNO_P (regno));
17073 regno -= FIRST_STACK_REG;
17075 snprintf (retval, sizeof (retval), ASM_SHORT "0xc%ddf", regno);
17080 return opno ? "fstp\t%y1" : "fstp\t%y0";
17084 /* Output code for INSN to compare OPERANDS. EFLAGS_P is 1 when fcomi
17085 should be used. UNORDERED_P is true when fucom should be used. */
17088 output_fp_compare (rtx insn, rtx *operands, bool eflags_p, bool unordered_p)
17090 int stack_top_dies;
17091 rtx cmp_op0, cmp_op1;
17092 int is_sse = SSE_REG_P (operands[0]) || SSE_REG_P (operands[1]);
17096 cmp_op0 = operands[0];
17097 cmp_op1 = operands[1];
17101 cmp_op0 = operands[1];
17102 cmp_op1 = operands[2];
17107 if (GET_MODE (operands[0]) == SFmode)
17109 return "%vucomiss\t{%1, %0|%0, %1}";
17111 return "%vcomiss\t{%1, %0|%0, %1}";
17114 return "%vucomisd\t{%1, %0|%0, %1}";
17116 return "%vcomisd\t{%1, %0|%0, %1}";
17119 gcc_assert (STACK_TOP_P (cmp_op0));
17121 stack_top_dies = find_regno_note (insn, REG_DEAD, FIRST_STACK_REG) != 0;
17123 if (cmp_op1 == CONST0_RTX (GET_MODE (cmp_op1)))
17125 if (stack_top_dies)
17127 output_asm_insn ("ftst\n\tfnstsw\t%0", operands);
17128 return output_387_ffreep (operands, 1);
17131 return "ftst\n\tfnstsw\t%0";
17134 if (STACK_REG_P (cmp_op1)
17136 && find_regno_note (insn, REG_DEAD, REGNO (cmp_op1))
17137 && REGNO (cmp_op1) != FIRST_STACK_REG)
17139 /* If both the top of the 387 stack dies, and the other operand
17140 is also a stack register that dies, then this must be a
17141 `fcompp' float compare */
17145 /* There is no double popping fcomi variant. Fortunately,
17146 eflags is immune from the fstp's cc clobbering. */
17148 output_asm_insn ("fucomip\t{%y1, %0|%0, %y1}", operands);
17150 output_asm_insn ("fcomip\t{%y1, %0|%0, %y1}", operands);
17151 return output_387_ffreep (operands, 0);
17156 return "fucompp\n\tfnstsw\t%0";
17158 return "fcompp\n\tfnstsw\t%0";
17163 /* Encoded here as eflags_p | intmode | unordered_p | stack_top_dies. */
17165 static const char * const alt[16] =
17167 "fcom%Z2\t%y2\n\tfnstsw\t%0",
17168 "fcomp%Z2\t%y2\n\tfnstsw\t%0",
17169 "fucom%Z2\t%y2\n\tfnstsw\t%0",
17170 "fucomp%Z2\t%y2\n\tfnstsw\t%0",
17172 "ficom%Z2\t%y2\n\tfnstsw\t%0",
17173 "ficomp%Z2\t%y2\n\tfnstsw\t%0",
17177 "fcomi\t{%y1, %0|%0, %y1}",
17178 "fcomip\t{%y1, %0|%0, %y1}",
17179 "fucomi\t{%y1, %0|%0, %y1}",
17180 "fucomip\t{%y1, %0|%0, %y1}",
17191 mask = eflags_p << 3;
17192 mask |= (GET_MODE_CLASS (GET_MODE (cmp_op1)) == MODE_INT) << 2;
17193 mask |= unordered_p << 1;
17194 mask |= stack_top_dies;
17196 gcc_assert (mask < 16);
17205 ix86_output_addr_vec_elt (FILE *file, int value)
17207 const char *directive = ASM_LONG;
17211 directive = ASM_QUAD;
17213 gcc_assert (!TARGET_64BIT);
17216 fprintf (file, "%s%s%d\n", directive, LPREFIX, value);
17220 ix86_output_addr_diff_elt (FILE *file, int value, int rel)
17222 const char *directive = ASM_LONG;
17225 if (TARGET_64BIT && CASE_VECTOR_MODE == DImode)
17226 directive = ASM_QUAD;
17228 gcc_assert (!TARGET_64BIT);
17230 /* We can't use @GOTOFF for text labels on VxWorks; see gotoff_operand. */
17231 if (TARGET_64BIT || TARGET_VXWORKS_RTP)
17232 fprintf (file, "%s%s%d-%s%d\n",
17233 directive, LPREFIX, value, LPREFIX, rel);
17234 else if (HAVE_AS_GOTOFF_IN_DATA)
17235 fprintf (file, ASM_LONG "%s%d@GOTOFF\n", LPREFIX, value);
17237 else if (TARGET_MACHO)
17239 fprintf (file, ASM_LONG "%s%d-", LPREFIX, value);
17240 machopic_output_function_base_name (file);
17245 asm_fprintf (file, ASM_LONG "%U%s+[.-%s%d]\n",
17246 GOT_SYMBOL_NAME, LPREFIX, value);
17249 /* Generate either "mov $0, reg" or "xor reg, reg", as appropriate
17253 ix86_expand_clear (rtx dest)
17257 /* We play register width games, which are only valid after reload. */
17258 gcc_assert (reload_completed);
17260 /* Avoid HImode and its attendant prefix byte. */
17261 if (GET_MODE_SIZE (GET_MODE (dest)) < 4)
17262 dest = gen_rtx_REG (SImode, REGNO (dest));
17263 tmp = gen_rtx_SET (dest, const0_rtx);
17265 if (!TARGET_USE_MOV0 || optimize_insn_for_size_p ())
17267 rtx clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
17268 tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, tmp, clob));
17274 /* X is an unchanging MEM. If it is a constant pool reference, return
17275 the constant pool rtx, else NULL. */
17278 maybe_get_pool_constant (rtx x)
17280 x = ix86_delegitimize_address (XEXP (x, 0));
17282 if (GET_CODE (x) == SYMBOL_REF && CONSTANT_POOL_ADDRESS_P (x))
17283 return get_pool_constant (x);
17289 ix86_expand_move (machine_mode mode, rtx operands[])
17292 enum tls_model model;
17297 if (GET_CODE (op1) == SYMBOL_REF)
17301 model = SYMBOL_REF_TLS_MODEL (op1);
17304 op1 = legitimize_tls_address (op1, model, true);
17305 op1 = force_operand (op1, op0);
17308 op1 = convert_to_mode (mode, op1, 1);
17310 else if ((tmp = legitimize_pe_coff_symbol (op1, false)) != NULL_RTX)
17313 else if (GET_CODE (op1) == CONST
17314 && GET_CODE (XEXP (op1, 0)) == PLUS
17315 && GET_CODE (XEXP (XEXP (op1, 0), 0)) == SYMBOL_REF)
17317 rtx addend = XEXP (XEXP (op1, 0), 1);
17318 rtx symbol = XEXP (XEXP (op1, 0), 0);
17321 model = SYMBOL_REF_TLS_MODEL (symbol);
17323 tmp = legitimize_tls_address (symbol, model, true);
17325 tmp = legitimize_pe_coff_symbol (symbol, true);
17329 tmp = force_operand (tmp, NULL);
17330 tmp = expand_simple_binop (Pmode, PLUS, tmp, addend,
17331 op0, 1, OPTAB_DIRECT);
17334 op1 = convert_to_mode (mode, tmp, 1);
17338 if ((flag_pic || MACHOPIC_INDIRECT)
17339 && symbolic_operand (op1, mode))
17341 if (TARGET_MACHO && !TARGET_64BIT)
17344 /* dynamic-no-pic */
17345 if (MACHOPIC_INDIRECT)
17347 rtx temp = (op0 && REG_P (op0) && mode == Pmode)
17348 ? op0 : gen_reg_rtx (Pmode);
17349 op1 = machopic_indirect_data_reference (op1, temp);
17351 op1 = machopic_legitimize_pic_address (op1, mode,
17352 temp == op1 ? 0 : temp);
17354 if (op0 != op1 && GET_CODE (op0) != MEM)
17356 rtx insn = gen_rtx_SET (op0, op1);
17360 if (GET_CODE (op0) == MEM)
17361 op1 = force_reg (Pmode, op1);
17365 if (GET_CODE (temp) != REG)
17366 temp = gen_reg_rtx (Pmode);
17367 temp = legitimize_pic_address (op1, temp);
17372 /* dynamic-no-pic */
17378 op1 = force_reg (mode, op1);
17379 else if (!(TARGET_64BIT && x86_64_movabs_operand (op1, DImode)))
17381 rtx reg = can_create_pseudo_p () ? NULL_RTX : op0;
17382 op1 = legitimize_pic_address (op1, reg);
17385 op1 = convert_to_mode (mode, op1, 1);
17392 && (PUSH_ROUNDING (GET_MODE_SIZE (mode)) != GET_MODE_SIZE (mode)
17393 || !push_operand (op0, mode))
17395 op1 = force_reg (mode, op1);
17397 if (push_operand (op0, mode)
17398 && ! general_no_elim_operand (op1, mode))
17399 op1 = copy_to_mode_reg (mode, op1);
17401 /* Force large constants in 64bit compilation into register
17402 to get them CSEed. */
17403 if (can_create_pseudo_p ()
17404 && (mode == DImode) && TARGET_64BIT
17405 && immediate_operand (op1, mode)
17406 && !x86_64_zext_immediate_operand (op1, VOIDmode)
17407 && !register_operand (op0, mode)
17409 op1 = copy_to_mode_reg (mode, op1);
17411 if (can_create_pseudo_p ()
17412 && CONST_DOUBLE_P (op1))
17414 /* If we are loading a floating point constant to a register,
17415 force the value to memory now, since we'll get better code
17416 out the back end. */
17418 op1 = validize_mem (force_const_mem (mode, op1));
17419 if (!register_operand (op0, mode))
17421 rtx temp = gen_reg_rtx (mode);
17422 emit_insn (gen_rtx_SET (temp, op1));
17423 emit_move_insn (op0, temp);
17429 emit_insn (gen_rtx_SET (op0, op1));
17433 ix86_expand_vector_move (machine_mode mode, rtx operands[])
17435 rtx op0 = operands[0], op1 = operands[1];
17436 unsigned int align = GET_MODE_ALIGNMENT (mode);
17438 if (push_operand (op0, VOIDmode))
17439 op0 = emit_move_resolve_push (mode, op0);
17441 /* Force constants other than zero into memory. We do not know how
17442 the instructions used to build constants modify the upper 64 bits
17443 of the register, once we have that information we may be able
17444 to handle some of them more efficiently. */
17445 if (can_create_pseudo_p ()
17446 && register_operand (op0, mode)
17447 && (CONSTANT_P (op1)
17448 || (GET_CODE (op1) == SUBREG
17449 && CONSTANT_P (SUBREG_REG (op1))))
17450 && !standard_sse_constant_p (op1))
17451 op1 = validize_mem (force_const_mem (mode, op1));
17453 /* We need to check memory alignment for SSE mode since attribute
17454 can make operands unaligned. */
17455 if (can_create_pseudo_p ()
17456 && SSE_REG_MODE_P (mode)
17457 && ((MEM_P (op0) && (MEM_ALIGN (op0) < align))
17458 || (MEM_P (op1) && (MEM_ALIGN (op1) < align))))
17462 /* ix86_expand_vector_move_misalign() does not like constants ... */
17463 if (CONSTANT_P (op1)
17464 || (GET_CODE (op1) == SUBREG
17465 && CONSTANT_P (SUBREG_REG (op1))))
17466 op1 = validize_mem (force_const_mem (mode, op1));
17468 /* ... nor both arguments in memory. */
17469 if (!register_operand (op0, mode)
17470 && !register_operand (op1, mode))
17471 op1 = force_reg (mode, op1);
17473 tmp[0] = op0; tmp[1] = op1;
17474 ix86_expand_vector_move_misalign (mode, tmp);
17478 /* Make operand1 a register if it isn't already. */
17479 if (can_create_pseudo_p ()
17480 && !register_operand (op0, mode)
17481 && !register_operand (op1, mode))
17483 emit_move_insn (op0, force_reg (GET_MODE (op0), op1));
17487 emit_insn (gen_rtx_SET (op0, op1));
17490 /* Split 32-byte AVX unaligned load and store if needed. */
17493 ix86_avx256_split_vector_move_misalign (rtx op0, rtx op1)
17496 rtx (*extract) (rtx, rtx, rtx);
17497 rtx (*load_unaligned) (rtx, rtx);
17498 rtx (*store_unaligned) (rtx, rtx);
17501 switch (GET_MODE (op0))
17504 gcc_unreachable ();
17506 extract = gen_avx_vextractf128v32qi;
17507 load_unaligned = gen_avx_loaddquv32qi;
17508 store_unaligned = gen_avx_storedquv32qi;
17512 extract = gen_avx_vextractf128v8sf;
17513 load_unaligned = gen_avx_loadups256;
17514 store_unaligned = gen_avx_storeups256;
17518 extract = gen_avx_vextractf128v4df;
17519 load_unaligned = gen_avx_loadupd256;
17520 store_unaligned = gen_avx_storeupd256;
17527 if (TARGET_AVX256_SPLIT_UNALIGNED_LOAD
17528 && optimize_insn_for_speed_p ())
17530 rtx r = gen_reg_rtx (mode);
17531 m = adjust_address (op1, mode, 0);
17532 emit_move_insn (r, m);
17533 m = adjust_address (op1, mode, 16);
17534 r = gen_rtx_VEC_CONCAT (GET_MODE (op0), r, m);
17535 emit_move_insn (op0, r);
17537 /* Normal *mov<mode>_internal pattern will handle
17538 unaligned loads just fine if misaligned_operand
17539 is true, and without the UNSPEC it can be combined
17540 with arithmetic instructions. */
17541 else if (misaligned_operand (op1, GET_MODE (op1)))
17542 emit_insn (gen_rtx_SET (op0, op1));
17544 emit_insn (load_unaligned (op0, op1));
17546 else if (MEM_P (op0))
17548 if (TARGET_AVX256_SPLIT_UNALIGNED_STORE
17549 && optimize_insn_for_speed_p ())
17551 m = adjust_address (op0, mode, 0);
17552 emit_insn (extract (m, op1, const0_rtx));
17553 m = adjust_address (op0, mode, 16);
17554 emit_insn (extract (m, op1, const1_rtx));
17557 emit_insn (store_unaligned (op0, op1));
17560 gcc_unreachable ();
17563 /* Implement the movmisalign patterns for SSE. Non-SSE modes go
17564 straight to ix86_expand_vector_move. */
17565 /* Code generation for scalar reg-reg moves of single and double precision data:
17566 if (x86_sse_partial_reg_dependency == true | x86_sse_split_regs == true)
17570 if (x86_sse_partial_reg_dependency == true)
17575 Code generation for scalar loads of double precision data:
17576 if (x86_sse_split_regs == true)
17577 movlpd mem, reg (gas syntax)
17581 Code generation for unaligned packed loads of single precision data
17582 (x86_sse_unaligned_move_optimal overrides x86_sse_partial_reg_dependency):
17583 if (x86_sse_unaligned_move_optimal)
17586 if (x86_sse_partial_reg_dependency == true)
17598 Code generation for unaligned packed loads of double precision data
17599 (x86_sse_unaligned_move_optimal overrides x86_sse_split_regs):
17600 if (x86_sse_unaligned_move_optimal)
17603 if (x86_sse_split_regs == true)
17616 ix86_expand_vector_move_misalign (machine_mode mode, rtx operands[])
17618 rtx op0, op1, orig_op0 = NULL_RTX, m;
17619 rtx (*load_unaligned) (rtx, rtx);
17620 rtx (*store_unaligned) (rtx, rtx);
17625 if (GET_MODE_SIZE (mode) == 64)
17627 switch (GET_MODE_CLASS (mode))
17629 case MODE_VECTOR_INT:
17631 if (GET_MODE (op0) != V16SImode)
17636 op0 = gen_reg_rtx (V16SImode);
17639 op0 = gen_lowpart (V16SImode, op0);
17641 op1 = gen_lowpart (V16SImode, op1);
17644 case MODE_VECTOR_FLOAT:
17645 switch (GET_MODE (op0))
17648 gcc_unreachable ();
17650 load_unaligned = gen_avx512f_loaddquv16si;
17651 store_unaligned = gen_avx512f_storedquv16si;
17654 load_unaligned = gen_avx512f_loadups512;
17655 store_unaligned = gen_avx512f_storeups512;
17658 load_unaligned = gen_avx512f_loadupd512;
17659 store_unaligned = gen_avx512f_storeupd512;
17664 emit_insn (load_unaligned (op0, op1));
17665 else if (MEM_P (op0))
17666 emit_insn (store_unaligned (op0, op1));
17668 gcc_unreachable ();
17670 emit_move_insn (orig_op0, gen_lowpart (GET_MODE (orig_op0), op0));
17674 gcc_unreachable ();
17681 && GET_MODE_SIZE (mode) == 32)
17683 switch (GET_MODE_CLASS (mode))
17685 case MODE_VECTOR_INT:
17687 if (GET_MODE (op0) != V32QImode)
17692 op0 = gen_reg_rtx (V32QImode);
17695 op0 = gen_lowpart (V32QImode, op0);
17697 op1 = gen_lowpart (V32QImode, op1);
17700 case MODE_VECTOR_FLOAT:
17701 ix86_avx256_split_vector_move_misalign (op0, op1);
17703 emit_move_insn (orig_op0, gen_lowpart (GET_MODE (orig_op0), op0));
17707 gcc_unreachable ();
17715 /* Normal *mov<mode>_internal pattern will handle
17716 unaligned loads just fine if misaligned_operand
17717 is true, and without the UNSPEC it can be combined
17718 with arithmetic instructions. */
17720 && (GET_MODE_CLASS (mode) == MODE_VECTOR_INT
17721 || GET_MODE_CLASS (mode) == MODE_VECTOR_FLOAT)
17722 && misaligned_operand (op1, GET_MODE (op1)))
17723 emit_insn (gen_rtx_SET (op0, op1));
17724 /* ??? If we have typed data, then it would appear that using
17725 movdqu is the only way to get unaligned data loaded with
17727 else if (TARGET_SSE2 && GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
17729 if (GET_MODE (op0) != V16QImode)
17732 op0 = gen_reg_rtx (V16QImode);
17734 op1 = gen_lowpart (V16QImode, op1);
17735 /* We will eventually emit movups based on insn attributes. */
17736 emit_insn (gen_sse2_loaddquv16qi (op0, op1));
17738 emit_move_insn (orig_op0, gen_lowpart (GET_MODE (orig_op0), op0));
17740 else if (TARGET_SSE2 && mode == V2DFmode)
17745 || TARGET_SSE_UNALIGNED_LOAD_OPTIMAL
17746 || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
17747 || optimize_insn_for_size_p ())
17749 /* We will eventually emit movups based on insn attributes. */
17750 emit_insn (gen_sse2_loadupd (op0, op1));
17754 /* When SSE registers are split into halves, we can avoid
17755 writing to the top half twice. */
17756 if (TARGET_SSE_SPLIT_REGS)
17758 emit_clobber (op0);
17763 /* ??? Not sure about the best option for the Intel chips.
17764 The following would seem to satisfy; the register is
17765 entirely cleared, breaking the dependency chain. We
17766 then store to the upper half, with a dependency depth
17767 of one. A rumor has it that Intel recommends two movsd
17768 followed by an unpacklpd, but this is unconfirmed. And
17769 given that the dependency depth of the unpacklpd would
17770 still be one, I'm not sure why this would be better. */
17771 zero = CONST0_RTX (V2DFmode);
17774 m = adjust_address (op1, DFmode, 0);
17775 emit_insn (gen_sse2_loadlpd (op0, zero, m));
17776 m = adjust_address (op1, DFmode, 8);
17777 emit_insn (gen_sse2_loadhpd (op0, op0, m));
17784 || TARGET_SSE_UNALIGNED_LOAD_OPTIMAL
17785 || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
17786 || optimize_insn_for_size_p ())
17788 if (GET_MODE (op0) != V4SFmode)
17791 op0 = gen_reg_rtx (V4SFmode);
17793 op1 = gen_lowpart (V4SFmode, op1);
17794 emit_insn (gen_sse_loadups (op0, op1));
17796 emit_move_insn (orig_op0,
17797 gen_lowpart (GET_MODE (orig_op0), op0));
17801 if (mode != V4SFmode)
17802 t = gen_reg_rtx (V4SFmode);
17806 if (TARGET_SSE_PARTIAL_REG_DEPENDENCY)
17807 emit_move_insn (t, CONST0_RTX (V4SFmode));
17811 m = adjust_address (op1, V2SFmode, 0);
17812 emit_insn (gen_sse_loadlps (t, t, m));
17813 m = adjust_address (op1, V2SFmode, 8);
17814 emit_insn (gen_sse_loadhps (t, t, m));
17815 if (mode != V4SFmode)
17816 emit_move_insn (op0, gen_lowpart (mode, t));
17819 else if (MEM_P (op0))
17821 if (TARGET_SSE2 && GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
17823 op0 = gen_lowpart (V16QImode, op0);
17824 op1 = gen_lowpart (V16QImode, op1);
17825 /* We will eventually emit movups based on insn attributes. */
17826 emit_insn (gen_sse2_storedquv16qi (op0, op1));
17828 else if (TARGET_SSE2 && mode == V2DFmode)
17831 || TARGET_SSE_UNALIGNED_STORE_OPTIMAL
17832 || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
17833 || optimize_insn_for_size_p ())
17834 /* We will eventually emit movups based on insn attributes. */
17835 emit_insn (gen_sse2_storeupd (op0, op1));
17838 m = adjust_address (op0, DFmode, 0);
17839 emit_insn (gen_sse2_storelpd (m, op1));
17840 m = adjust_address (op0, DFmode, 8);
17841 emit_insn (gen_sse2_storehpd (m, op1));
17846 if (mode != V4SFmode)
17847 op1 = gen_lowpart (V4SFmode, op1);
17850 || TARGET_SSE_UNALIGNED_STORE_OPTIMAL
17851 || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
17852 || optimize_insn_for_size_p ())
17854 op0 = gen_lowpart (V4SFmode, op0);
17855 emit_insn (gen_sse_storeups (op0, op1));
17859 m = adjust_address (op0, V2SFmode, 0);
17860 emit_insn (gen_sse_storelps (m, op1));
17861 m = adjust_address (op0, V2SFmode, 8);
17862 emit_insn (gen_sse_storehps (m, op1));
17867 gcc_unreachable ();
17870 /* Helper function of ix86_fixup_binary_operands to canonicalize
17871 operand order. Returns true if the operands should be swapped. */
17874 ix86_swap_binary_operands_p (enum rtx_code code, machine_mode mode,
17877 rtx dst = operands[0];
17878 rtx src1 = operands[1];
17879 rtx src2 = operands[2];
17881 /* If the operation is not commutative, we can't do anything. */
17882 if (GET_RTX_CLASS (code) != RTX_COMM_ARITH)
17885 /* Highest priority is that src1 should match dst. */
17886 if (rtx_equal_p (dst, src1))
17888 if (rtx_equal_p (dst, src2))
17891 /* Next highest priority is that immediate constants come second. */
17892 if (immediate_operand (src2, mode))
17894 if (immediate_operand (src1, mode))
17897 /* Lowest priority is that memory references should come second. */
17907 /* Fix up OPERANDS to satisfy ix86_binary_operator_ok. Return the
17908 destination to use for the operation. If different from the true
17909 destination in operands[0], a copy operation will be required. */
17912 ix86_fixup_binary_operands (enum rtx_code code, machine_mode mode,
17915 rtx dst = operands[0];
17916 rtx src1 = operands[1];
17917 rtx src2 = operands[2];
17919 /* Canonicalize operand order. */
17920 if (ix86_swap_binary_operands_p (code, mode, operands))
17922 /* It is invalid to swap operands of different modes. */
17923 gcc_assert (GET_MODE (src1) == GET_MODE (src2));
17925 std::swap (src1, src2);
17928 /* Both source operands cannot be in memory. */
17929 if (MEM_P (src1) && MEM_P (src2))
17931 /* Optimization: Only read from memory once. */
17932 if (rtx_equal_p (src1, src2))
17934 src2 = force_reg (mode, src2);
17937 else if (rtx_equal_p (dst, src1))
17938 src2 = force_reg (mode, src2);
17940 src1 = force_reg (mode, src1);
17943 /* If the destination is memory, and we do not have matching source
17944 operands, do things in registers. */
17945 if (MEM_P (dst) && !rtx_equal_p (dst, src1))
17946 dst = gen_reg_rtx (mode);
17948 /* Source 1 cannot be a constant. */
17949 if (CONSTANT_P (src1))
17950 src1 = force_reg (mode, src1);
17952 /* Source 1 cannot be a non-matching memory. */
17953 if (MEM_P (src1) && !rtx_equal_p (dst, src1))
17954 src1 = force_reg (mode, src1);
17956 /* Improve address combine. */
17958 && GET_MODE_CLASS (mode) == MODE_INT
17960 src2 = force_reg (mode, src2);
17962 operands[1] = src1;
17963 operands[2] = src2;
17967 /* Similarly, but assume that the destination has already been
17968 set up properly. */
17971 ix86_fixup_binary_operands_no_copy (enum rtx_code code,
17972 machine_mode mode, rtx operands[])
17974 rtx dst = ix86_fixup_binary_operands (code, mode, operands);
17975 gcc_assert (dst == operands[0]);
17978 /* Attempt to expand a binary operator. Make the expansion closer to the
17979 actual machine, then just general_operand, which will allow 3 separate
17980 memory references (one output, two input) in a single insn. */
17983 ix86_expand_binary_operator (enum rtx_code code, machine_mode mode,
17986 rtx src1, src2, dst, op, clob;
17988 dst = ix86_fixup_binary_operands (code, mode, operands);
17989 src1 = operands[1];
17990 src2 = operands[2];
17992 /* Emit the instruction. */
17994 op = gen_rtx_SET (dst, gen_rtx_fmt_ee (code, mode, src1, src2));
17996 if (reload_completed
17998 && !rtx_equal_p (dst, src1))
18000 /* This is going to be an LEA; avoid splitting it later. */
18005 clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
18006 emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, op, clob)));
18009 /* Fix up the destination if needed. */
18010 if (dst != operands[0])
18011 emit_move_insn (operands[0], dst);
18014 /* Expand vector logical operation CODE (AND, IOR, XOR) in MODE with
18015 the given OPERANDS. */
18018 ix86_expand_vector_logical_operator (enum rtx_code code, machine_mode mode,
18021 rtx op1 = NULL_RTX, op2 = NULL_RTX;
18022 if (GET_CODE (operands[1]) == SUBREG)
18027 else if (GET_CODE (operands[2]) == SUBREG)
18032 /* Optimize (__m128i) d | (__m128i) e and similar code
18033 when d and e are float vectors into float vector logical
18034 insn. In C/C++ without using intrinsics there is no other way
18035 to express vector logical operation on float vectors than
18036 to cast them temporarily to integer vectors. */
18038 && !TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
18039 && ((GET_CODE (op2) == SUBREG || GET_CODE (op2) == CONST_VECTOR))
18040 && GET_MODE_CLASS (GET_MODE (SUBREG_REG (op1))) == MODE_VECTOR_FLOAT
18041 && GET_MODE_SIZE (GET_MODE (SUBREG_REG (op1))) == GET_MODE_SIZE (mode)
18042 && SUBREG_BYTE (op1) == 0
18043 && (GET_CODE (op2) == CONST_VECTOR
18044 || (GET_MODE (SUBREG_REG (op1)) == GET_MODE (SUBREG_REG (op2))
18045 && SUBREG_BYTE (op2) == 0))
18046 && can_create_pseudo_p ())
18049 switch (GET_MODE (SUBREG_REG (op1)))
18057 dst = gen_reg_rtx (GET_MODE (SUBREG_REG (op1)));
18058 if (GET_CODE (op2) == CONST_VECTOR)
18060 op2 = gen_lowpart (GET_MODE (dst), op2);
18061 op2 = force_reg (GET_MODE (dst), op2);
18066 op2 = SUBREG_REG (operands[2]);
18067 if (!nonimmediate_operand (op2, GET_MODE (dst)))
18068 op2 = force_reg (GET_MODE (dst), op2);
18070 op1 = SUBREG_REG (op1);
18071 if (!nonimmediate_operand (op1, GET_MODE (dst)))
18072 op1 = force_reg (GET_MODE (dst), op1);
18073 emit_insn (gen_rtx_SET (dst,
18074 gen_rtx_fmt_ee (code, GET_MODE (dst),
18076 emit_move_insn (operands[0], gen_lowpart (mode, dst));
18082 if (!nonimmediate_operand (operands[1], mode))
18083 operands[1] = force_reg (mode, operands[1]);
18084 if (!nonimmediate_operand (operands[2], mode))
18085 operands[2] = force_reg (mode, operands[2]);
18086 ix86_fixup_binary_operands_no_copy (code, mode, operands);
18087 emit_insn (gen_rtx_SET (operands[0],
18088 gen_rtx_fmt_ee (code, mode, operands[1],
18092 /* Return TRUE or FALSE depending on whether the binary operator meets the
18093 appropriate constraints. */
18096 ix86_binary_operator_ok (enum rtx_code code, machine_mode mode,
18099 rtx dst = operands[0];
18100 rtx src1 = operands[1];
18101 rtx src2 = operands[2];
18103 /* Both source operands cannot be in memory. */
18104 if (MEM_P (src1) && MEM_P (src2))
18107 /* Canonicalize operand order for commutative operators. */
18108 if (ix86_swap_binary_operands_p (code, mode, operands))
18109 std::swap (src1, src2);
18111 /* If the destination is memory, we must have a matching source operand. */
18112 if (MEM_P (dst) && !rtx_equal_p (dst, src1))
18115 /* Source 1 cannot be a constant. */
18116 if (CONSTANT_P (src1))
18119 /* Source 1 cannot be a non-matching memory. */
18120 if (MEM_P (src1) && !rtx_equal_p (dst, src1))
18121 /* Support "andhi/andsi/anddi" as a zero-extending move. */
18122 return (code == AND
18125 || (TARGET_64BIT && mode == DImode))
18126 && satisfies_constraint_L (src2));
18131 /* Attempt to expand a unary operator. Make the expansion closer to the
18132 actual machine, then just general_operand, which will allow 2 separate
18133 memory references (one output, one input) in a single insn. */
18136 ix86_expand_unary_operator (enum rtx_code code, machine_mode mode,
18139 bool matching_memory = false;
18140 rtx src, dst, op, clob;
18145 /* If the destination is memory, and we do not have matching source
18146 operands, do things in registers. */
18149 if (rtx_equal_p (dst, src))
18150 matching_memory = true;
18152 dst = gen_reg_rtx (mode);
18155 /* When source operand is memory, destination must match. */
18156 if (MEM_P (src) && !matching_memory)
18157 src = force_reg (mode, src);
18159 /* Emit the instruction. */
18161 op = gen_rtx_SET (dst, gen_rtx_fmt_e (code, mode, src));
18167 clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
18168 emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, op, clob)));
18171 /* Fix up the destination if needed. */
18172 if (dst != operands[0])
18173 emit_move_insn (operands[0], dst);
18176 /* Split 32bit/64bit divmod with 8bit unsigned divmod if dividend and
18177 divisor are within the range [0-255]. */
18180 ix86_split_idivmod (machine_mode mode, rtx operands[],
18183 rtx_code_label *end_label, *qimode_label;
18184 rtx insn, div, mod;
18185 rtx scratch, tmp0, tmp1, tmp2;
18186 rtx (*gen_divmod4_1) (rtx, rtx, rtx, rtx);
18187 rtx (*gen_zero_extend) (rtx, rtx);
18188 rtx (*gen_test_ccno_1) (rtx, rtx);
18193 gen_divmod4_1 = signed_p ? gen_divmodsi4_1 : gen_udivmodsi4_1;
18194 gen_test_ccno_1 = gen_testsi_ccno_1;
18195 gen_zero_extend = gen_zero_extendqisi2;
18198 gen_divmod4_1 = signed_p ? gen_divmoddi4_1 : gen_udivmoddi4_1;
18199 gen_test_ccno_1 = gen_testdi_ccno_1;
18200 gen_zero_extend = gen_zero_extendqidi2;
18203 gcc_unreachable ();
18206 end_label = gen_label_rtx ();
18207 qimode_label = gen_label_rtx ();
18209 scratch = gen_reg_rtx (mode);
18211 /* Use 8bit unsigned divimod if dividend and divisor are within
18212 the range [0-255]. */
18213 emit_move_insn (scratch, operands[2]);
18214 scratch = expand_simple_binop (mode, IOR, scratch, operands[3],
18215 scratch, 1, OPTAB_DIRECT);
18216 emit_insn (gen_test_ccno_1 (scratch, GEN_INT (-0x100)));
18217 tmp0 = gen_rtx_REG (CCNOmode, FLAGS_REG);
18218 tmp0 = gen_rtx_EQ (VOIDmode, tmp0, const0_rtx);
18219 tmp0 = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp0,
18220 gen_rtx_LABEL_REF (VOIDmode, qimode_label),
18222 insn = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp0));
18223 predict_jump (REG_BR_PROB_BASE * 50 / 100);
18224 JUMP_LABEL (insn) = qimode_label;
18226 /* Generate original signed/unsigned divimod. */
18227 div = gen_divmod4_1 (operands[0], operands[1],
18228 operands[2], operands[3]);
18231 /* Branch to the end. */
18232 emit_jump_insn (gen_jump (end_label));
18235 /* Generate 8bit unsigned divide. */
18236 emit_label (qimode_label);
18237 /* Don't use operands[0] for result of 8bit divide since not all
18238 registers support QImode ZERO_EXTRACT. */
18239 tmp0 = simplify_gen_subreg (HImode, scratch, mode, 0);
18240 tmp1 = simplify_gen_subreg (HImode, operands[2], mode, 0);
18241 tmp2 = simplify_gen_subreg (QImode, operands[3], mode, 0);
18242 emit_insn (gen_udivmodhiqi3 (tmp0, tmp1, tmp2));
18246 div = gen_rtx_DIV (SImode, operands[2], operands[3]);
18247 mod = gen_rtx_MOD (SImode, operands[2], operands[3]);
18251 div = gen_rtx_UDIV (SImode, operands[2], operands[3]);
18252 mod = gen_rtx_UMOD (SImode, operands[2], operands[3]);
18255 /* Extract remainder from AH. */
18256 tmp1 = gen_rtx_ZERO_EXTRACT (mode, tmp0, GEN_INT (8), GEN_INT (8));
18257 if (REG_P (operands[1]))
18258 insn = emit_move_insn (operands[1], tmp1);
18261 /* Need a new scratch register since the old one has result
18263 scratch = gen_reg_rtx (mode);
18264 emit_move_insn (scratch, tmp1);
18265 insn = emit_move_insn (operands[1], scratch);
18267 set_unique_reg_note (insn, REG_EQUAL, mod);
18269 /* Zero extend quotient from AL. */
18270 tmp1 = gen_lowpart (QImode, tmp0);
18271 insn = emit_insn (gen_zero_extend (operands[0], tmp1));
18272 set_unique_reg_note (insn, REG_EQUAL, div);
18274 emit_label (end_label);
18277 #define LEA_MAX_STALL (3)
18278 #define LEA_SEARCH_THRESHOLD (LEA_MAX_STALL << 1)
18280 /* Increase given DISTANCE in half-cycles according to
18281 dependencies between PREV and NEXT instructions.
18282 Add 1 half-cycle if there is no dependency and
18283 go to next cycle if there is some dependecy. */
18285 static unsigned int
18286 increase_distance (rtx_insn *prev, rtx_insn *next, unsigned int distance)
18290 if (!prev || !next)
18291 return distance + (distance & 1) + 2;
18293 if (!DF_INSN_USES (next) || !DF_INSN_DEFS (prev))
18294 return distance + 1;
18296 FOR_EACH_INSN_USE (use, next)
18297 FOR_EACH_INSN_DEF (def, prev)
18298 if (!DF_REF_IS_ARTIFICIAL (def)
18299 && DF_REF_REGNO (use) == DF_REF_REGNO (def))
18300 return distance + (distance & 1) + 2;
18302 return distance + 1;
18305 /* Function checks if instruction INSN defines register number
18306 REGNO1 or REGNO2. */
18309 insn_defines_reg (unsigned int regno1, unsigned int regno2,
18314 FOR_EACH_INSN_DEF (def, insn)
18315 if (DF_REF_REG_DEF_P (def)
18316 && !DF_REF_IS_ARTIFICIAL (def)
18317 && (regno1 == DF_REF_REGNO (def)
18318 || regno2 == DF_REF_REGNO (def)))
18324 /* Function checks if instruction INSN uses register number
18325 REGNO as a part of address expression. */
18328 insn_uses_reg_mem (unsigned int regno, rtx insn)
18332 FOR_EACH_INSN_USE (use, insn)
18333 if (DF_REF_REG_MEM_P (use) && regno == DF_REF_REGNO (use))
18339 /* Search backward for non-agu definition of register number REGNO1
18340 or register number REGNO2 in basic block starting from instruction
18341 START up to head of basic block or instruction INSN.
18343 Function puts true value into *FOUND var if definition was found
18344 and false otherwise.
18346 Distance in half-cycles between START and found instruction or head
18347 of BB is added to DISTANCE and returned. */
18350 distance_non_agu_define_in_bb (unsigned int regno1, unsigned int regno2,
18351 rtx_insn *insn, int distance,
18352 rtx_insn *start, bool *found)
18354 basic_block bb = start ? BLOCK_FOR_INSN (start) : NULL;
18355 rtx_insn *prev = start;
18356 rtx_insn *next = NULL;
18362 && distance < LEA_SEARCH_THRESHOLD)
18364 if (NONDEBUG_INSN_P (prev) && NONJUMP_INSN_P (prev))
18366 distance = increase_distance (prev, next, distance);
18367 if (insn_defines_reg (regno1, regno2, prev))
18369 if (recog_memoized (prev) < 0
18370 || get_attr_type (prev) != TYPE_LEA)
18379 if (prev == BB_HEAD (bb))
18382 prev = PREV_INSN (prev);
18388 /* Search backward for non-agu definition of register number REGNO1
18389 or register number REGNO2 in INSN's basic block until
18390 1. Pass LEA_SEARCH_THRESHOLD instructions, or
18391 2. Reach neighbour BBs boundary, or
18392 3. Reach agu definition.
18393 Returns the distance between the non-agu definition point and INSN.
18394 If no definition point, returns -1. */
18397 distance_non_agu_define (unsigned int regno1, unsigned int regno2,
18400 basic_block bb = BLOCK_FOR_INSN (insn);
18402 bool found = false;
18404 if (insn != BB_HEAD (bb))
18405 distance = distance_non_agu_define_in_bb (regno1, regno2, insn,
18406 distance, PREV_INSN (insn),
18409 if (!found && distance < LEA_SEARCH_THRESHOLD)
18413 bool simple_loop = false;
18415 FOR_EACH_EDGE (e, ei, bb->preds)
18418 simple_loop = true;
18423 distance = distance_non_agu_define_in_bb (regno1, regno2,
18425 BB_END (bb), &found);
18428 int shortest_dist = -1;
18429 bool found_in_bb = false;
18431 FOR_EACH_EDGE (e, ei, bb->preds)
18434 = distance_non_agu_define_in_bb (regno1, regno2,
18440 if (shortest_dist < 0)
18441 shortest_dist = bb_dist;
18442 else if (bb_dist > 0)
18443 shortest_dist = MIN (bb_dist, shortest_dist);
18449 distance = shortest_dist;
18453 /* get_attr_type may modify recog data. We want to make sure
18454 that recog data is valid for instruction INSN, on which
18455 distance_non_agu_define is called. INSN is unchanged here. */
18456 extract_insn_cached (insn);
18461 return distance >> 1;
18464 /* Return the distance in half-cycles between INSN and the next
18465 insn that uses register number REGNO in memory address added
18466 to DISTANCE. Return -1 if REGNO0 is set.
18468 Put true value into *FOUND if register usage was found and
18470 Put true value into *REDEFINED if register redefinition was
18471 found and false otherwise. */
18474 distance_agu_use_in_bb (unsigned int regno,
18475 rtx_insn *insn, int distance, rtx_insn *start,
18476 bool *found, bool *redefined)
18478 basic_block bb = NULL;
18479 rtx_insn *next = start;
18480 rtx_insn *prev = NULL;
18483 *redefined = false;
18485 if (start != NULL_RTX)
18487 bb = BLOCK_FOR_INSN (start);
18488 if (start != BB_HEAD (bb))
18489 /* If insn and start belong to the same bb, set prev to insn,
18490 so the call to increase_distance will increase the distance
18491 between insns by 1. */
18497 && distance < LEA_SEARCH_THRESHOLD)
18499 if (NONDEBUG_INSN_P (next) && NONJUMP_INSN_P (next))
18501 distance = increase_distance(prev, next, distance);
18502 if (insn_uses_reg_mem (regno, next))
18504 /* Return DISTANCE if OP0 is used in memory
18505 address in NEXT. */
18510 if (insn_defines_reg (regno, INVALID_REGNUM, next))
18512 /* Return -1 if OP0 is set in NEXT. */
18520 if (next == BB_END (bb))
18523 next = NEXT_INSN (next);
18529 /* Return the distance between INSN and the next insn that uses
18530 register number REGNO0 in memory address. Return -1 if no such
18531 a use is found within LEA_SEARCH_THRESHOLD or REGNO0 is set. */
18534 distance_agu_use (unsigned int regno0, rtx_insn *insn)
18536 basic_block bb = BLOCK_FOR_INSN (insn);
18538 bool found = false;
18539 bool redefined = false;
18541 if (insn != BB_END (bb))
18542 distance = distance_agu_use_in_bb (regno0, insn, distance,
18544 &found, &redefined);
18546 if (!found && !redefined && distance < LEA_SEARCH_THRESHOLD)
18550 bool simple_loop = false;
18552 FOR_EACH_EDGE (e, ei, bb->succs)
18555 simple_loop = true;
18560 distance = distance_agu_use_in_bb (regno0, insn,
18561 distance, BB_HEAD (bb),
18562 &found, &redefined);
18565 int shortest_dist = -1;
18566 bool found_in_bb = false;
18567 bool redefined_in_bb = false;
18569 FOR_EACH_EDGE (e, ei, bb->succs)
18572 = distance_agu_use_in_bb (regno0, insn,
18573 distance, BB_HEAD (e->dest),
18574 &found_in_bb, &redefined_in_bb);
18577 if (shortest_dist < 0)
18578 shortest_dist = bb_dist;
18579 else if (bb_dist > 0)
18580 shortest_dist = MIN (bb_dist, shortest_dist);
18586 distance = shortest_dist;
18590 if (!found || redefined)
18593 return distance >> 1;
18596 /* Define this macro to tune LEA priority vs ADD, it take effect when
18597 there is a dilemma of choicing LEA or ADD
18598 Negative value: ADD is more preferred than LEA
18600 Positive value: LEA is more preferred than ADD*/
18601 #define IX86_LEA_PRIORITY 0
18603 /* Return true if usage of lea INSN has performance advantage
18604 over a sequence of instructions. Instructions sequence has
18605 SPLIT_COST cycles higher latency than lea latency. */
18608 ix86_lea_outperforms (rtx_insn *insn, unsigned int regno0, unsigned int regno1,
18609 unsigned int regno2, int split_cost, bool has_scale)
18611 int dist_define, dist_use;
18613 /* For Silvermont if using a 2-source or 3-source LEA for
18614 non-destructive destination purposes, or due to wanting
18615 ability to use SCALE, the use of LEA is justified. */
18616 if (TARGET_SILVERMONT || TARGET_INTEL)
18620 if (split_cost < 1)
18622 if (regno0 == regno1 || regno0 == regno2)
18627 dist_define = distance_non_agu_define (regno1, regno2, insn);
18628 dist_use = distance_agu_use (regno0, insn);
18630 if (dist_define < 0 || dist_define >= LEA_MAX_STALL)
18632 /* If there is no non AGU operand definition, no AGU
18633 operand usage and split cost is 0 then both lea
18634 and non lea variants have same priority. Currently
18635 we prefer lea for 64 bit code and non lea on 32 bit
18637 if (dist_use < 0 && split_cost == 0)
18638 return TARGET_64BIT || IX86_LEA_PRIORITY;
18643 /* With longer definitions distance lea is more preferable.
18644 Here we change it to take into account splitting cost and
18646 dist_define += split_cost + IX86_LEA_PRIORITY;
18648 /* If there is no use in memory addess then we just check
18649 that split cost exceeds AGU stall. */
18651 return dist_define > LEA_MAX_STALL;
18653 /* If this insn has both backward non-agu dependence and forward
18654 agu dependence, the one with short distance takes effect. */
18655 return dist_define >= dist_use;
18658 /* Return true if it is legal to clobber flags by INSN and
18659 false otherwise. */
18662 ix86_ok_to_clobber_flags (rtx_insn *insn)
18664 basic_block bb = BLOCK_FOR_INSN (insn);
18670 if (NONDEBUG_INSN_P (insn))
18672 FOR_EACH_INSN_USE (use, insn)
18673 if (DF_REF_REG_USE_P (use) && DF_REF_REGNO (use) == FLAGS_REG)
18676 if (insn_defines_reg (FLAGS_REG, INVALID_REGNUM, insn))
18680 if (insn == BB_END (bb))
18683 insn = NEXT_INSN (insn);
18686 live = df_get_live_out(bb);
18687 return !REGNO_REG_SET_P (live, FLAGS_REG);
18690 /* Return true if we need to split op0 = op1 + op2 into a sequence of
18691 move and add to avoid AGU stalls. */
18694 ix86_avoid_lea_for_add (rtx_insn *insn, rtx operands[])
18696 unsigned int regno0, regno1, regno2;
18698 /* Check if we need to optimize. */
18699 if (!TARGET_OPT_AGU || optimize_function_for_size_p (cfun))
18702 /* Check it is correct to split here. */
18703 if (!ix86_ok_to_clobber_flags(insn))
18706 regno0 = true_regnum (operands[0]);
18707 regno1 = true_regnum (operands[1]);
18708 regno2 = true_regnum (operands[2]);
18710 /* We need to split only adds with non destructive
18711 destination operand. */
18712 if (regno0 == regno1 || regno0 == regno2)
18715 return !ix86_lea_outperforms (insn, regno0, regno1, regno2, 1, false);
18718 /* Return true if we should emit lea instruction instead of mov
18722 ix86_use_lea_for_mov (rtx_insn *insn, rtx operands[])
18724 unsigned int regno0, regno1;
18726 /* Check if we need to optimize. */
18727 if (!TARGET_OPT_AGU || optimize_function_for_size_p (cfun))
18730 /* Use lea for reg to reg moves only. */
18731 if (!REG_P (operands[0]) || !REG_P (operands[1]))
18734 regno0 = true_regnum (operands[0]);
18735 regno1 = true_regnum (operands[1]);
18737 return ix86_lea_outperforms (insn, regno0, regno1, INVALID_REGNUM, 0, false);
18740 /* Return true if we need to split lea into a sequence of
18741 instructions to avoid AGU stalls. */
18744 ix86_avoid_lea_for_addr (rtx_insn *insn, rtx operands[])
18746 unsigned int regno0, regno1, regno2;
18748 struct ix86_address parts;
18751 /* Check we need to optimize. */
18752 if (!TARGET_AVOID_LEA_FOR_ADDR || optimize_function_for_size_p (cfun))
18755 /* The "at least two components" test below might not catch simple
18756 move or zero extension insns if parts.base is non-NULL and parts.disp
18757 is const0_rtx as the only components in the address, e.g. if the
18758 register is %rbp or %r13. As this test is much cheaper and moves or
18759 zero extensions are the common case, do this check first. */
18760 if (REG_P (operands[1])
18761 || (SImode_address_operand (operands[1], VOIDmode)
18762 && REG_P (XEXP (operands[1], 0))))
18765 /* Check if it is OK to split here. */
18766 if (!ix86_ok_to_clobber_flags (insn))
18769 ok = ix86_decompose_address (operands[1], &parts);
18772 /* There should be at least two components in the address. */
18773 if ((parts.base != NULL_RTX) + (parts.index != NULL_RTX)
18774 + (parts.disp != NULL_RTX) + (parts.scale > 1) < 2)
18777 /* We should not split into add if non legitimate pic
18778 operand is used as displacement. */
18779 if (parts.disp && flag_pic && !LEGITIMATE_PIC_OPERAND_P (parts.disp))
18782 regno0 = true_regnum (operands[0]) ;
18783 regno1 = INVALID_REGNUM;
18784 regno2 = INVALID_REGNUM;
18787 regno1 = true_regnum (parts.base);
18789 regno2 = true_regnum (parts.index);
18793 /* Compute how many cycles we will add to execution time
18794 if split lea into a sequence of instructions. */
18795 if (parts.base || parts.index)
18797 /* Have to use mov instruction if non desctructive
18798 destination form is used. */
18799 if (regno1 != regno0 && regno2 != regno0)
18802 /* Have to add index to base if both exist. */
18803 if (parts.base && parts.index)
18806 /* Have to use shift and adds if scale is 2 or greater. */
18807 if (parts.scale > 1)
18809 if (regno0 != regno1)
18811 else if (regno2 == regno0)
18814 split_cost += parts.scale;
18817 /* Have to use add instruction with immediate if
18818 disp is non zero. */
18819 if (parts.disp && parts.disp != const0_rtx)
18822 /* Subtract the price of lea. */
18826 return !ix86_lea_outperforms (insn, regno0, regno1, regno2, split_cost,
18830 /* Emit x86 binary operand CODE in mode MODE, where the first operand
18831 matches destination. RTX includes clobber of FLAGS_REG. */
18834 ix86_emit_binop (enum rtx_code code, machine_mode mode,
18839 op = gen_rtx_SET (dst, gen_rtx_fmt_ee (code, mode, dst, src));
18840 clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
18842 emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, op, clob)));
18845 /* Return true if regno1 def is nearest to the insn. */
18848 find_nearest_reg_def (rtx_insn *insn, int regno1, int regno2)
18850 rtx_insn *prev = insn;
18851 rtx_insn *start = BB_HEAD (BLOCK_FOR_INSN (insn));
18855 while (prev && prev != start)
18857 if (!INSN_P (prev) || !NONDEBUG_INSN_P (prev))
18859 prev = PREV_INSN (prev);
18862 if (insn_defines_reg (regno1, INVALID_REGNUM, prev))
18864 else if (insn_defines_reg (regno2, INVALID_REGNUM, prev))
18866 prev = PREV_INSN (prev);
18869 /* None of the regs is defined in the bb. */
18873 /* Split lea instructions into a sequence of instructions
18874 which are executed on ALU to avoid AGU stalls.
18875 It is assumed that it is allowed to clobber flags register
18876 at lea position. */
18879 ix86_split_lea_for_addr (rtx_insn *insn, rtx operands[], machine_mode mode)
18881 unsigned int regno0, regno1, regno2;
18882 struct ix86_address parts;
18886 ok = ix86_decompose_address (operands[1], &parts);
18889 target = gen_lowpart (mode, operands[0]);
18891 regno0 = true_regnum (target);
18892 regno1 = INVALID_REGNUM;
18893 regno2 = INVALID_REGNUM;
18897 parts.base = gen_lowpart (mode, parts.base);
18898 regno1 = true_regnum (parts.base);
18903 parts.index = gen_lowpart (mode, parts.index);
18904 regno2 = true_regnum (parts.index);
18908 parts.disp = gen_lowpart (mode, parts.disp);
18910 if (parts.scale > 1)
18912 /* Case r1 = r1 + ... */
18913 if (regno1 == regno0)
18915 /* If we have a case r1 = r1 + C * r2 then we
18916 should use multiplication which is very
18917 expensive. Assume cost model is wrong if we
18918 have such case here. */
18919 gcc_assert (regno2 != regno0);
18921 for (adds = parts.scale; adds > 0; adds--)
18922 ix86_emit_binop (PLUS, mode, target, parts.index);
18926 /* r1 = r2 + r3 * C case. Need to move r3 into r1. */
18927 if (regno0 != regno2)
18928 emit_insn (gen_rtx_SET (target, parts.index));
18930 /* Use shift for scaling. */
18931 ix86_emit_binop (ASHIFT, mode, target,
18932 GEN_INT (exact_log2 (parts.scale)));
18935 ix86_emit_binop (PLUS, mode, target, parts.base);
18937 if (parts.disp && parts.disp != const0_rtx)
18938 ix86_emit_binop (PLUS, mode, target, parts.disp);
18941 else if (!parts.base && !parts.index)
18943 gcc_assert(parts.disp);
18944 emit_insn (gen_rtx_SET (target, parts.disp));
18950 if (regno0 != regno2)
18951 emit_insn (gen_rtx_SET (target, parts.index));
18953 else if (!parts.index)
18955 if (regno0 != regno1)
18956 emit_insn (gen_rtx_SET (target, parts.base));
18960 if (regno0 == regno1)
18962 else if (regno0 == regno2)
18968 /* Find better operand for SET instruction, depending
18969 on which definition is farther from the insn. */
18970 if (find_nearest_reg_def (insn, regno1, regno2))
18971 tmp = parts.index, tmp1 = parts.base;
18973 tmp = parts.base, tmp1 = parts.index;
18975 emit_insn (gen_rtx_SET (target, tmp));
18977 if (parts.disp && parts.disp != const0_rtx)
18978 ix86_emit_binop (PLUS, mode, target, parts.disp);
18980 ix86_emit_binop (PLUS, mode, target, tmp1);
18984 ix86_emit_binop (PLUS, mode, target, tmp);
18987 if (parts.disp && parts.disp != const0_rtx)
18988 ix86_emit_binop (PLUS, mode, target, parts.disp);
18992 /* Return true if it is ok to optimize an ADD operation to LEA
18993 operation to avoid flag register consumation. For most processors,
18994 ADD is faster than LEA. For the processors like BONNELL, if the
18995 destination register of LEA holds an actual address which will be
18996 used soon, LEA is better and otherwise ADD is better. */
18999 ix86_lea_for_add_ok (rtx_insn *insn, rtx operands[])
19001 unsigned int regno0 = true_regnum (operands[0]);
19002 unsigned int regno1 = true_regnum (operands[1]);
19003 unsigned int regno2 = true_regnum (operands[2]);
19005 /* If a = b + c, (a!=b && a!=c), must use lea form. */
19006 if (regno0 != regno1 && regno0 != regno2)
19009 if (!TARGET_OPT_AGU || optimize_function_for_size_p (cfun))
19012 return ix86_lea_outperforms (insn, regno0, regno1, regno2, 0, false);
19015 /* Return true if destination reg of SET_BODY is shift count of
19019 ix86_dep_by_shift_count_body (const_rtx set_body, const_rtx use_body)
19025 /* Retrieve destination of SET_BODY. */
19026 switch (GET_CODE (set_body))
19029 set_dest = SET_DEST (set_body);
19030 if (!set_dest || !REG_P (set_dest))
19034 for (i = XVECLEN (set_body, 0) - 1; i >= 0; i--)
19035 if (ix86_dep_by_shift_count_body (XVECEXP (set_body, 0, i),
19043 /* Retrieve shift count of USE_BODY. */
19044 switch (GET_CODE (use_body))
19047 shift_rtx = XEXP (use_body, 1);
19050 for (i = XVECLEN (use_body, 0) - 1; i >= 0; i--)
19051 if (ix86_dep_by_shift_count_body (set_body,
19052 XVECEXP (use_body, 0, i)))
19060 && (GET_CODE (shift_rtx) == ASHIFT
19061 || GET_CODE (shift_rtx) == LSHIFTRT
19062 || GET_CODE (shift_rtx) == ASHIFTRT
19063 || GET_CODE (shift_rtx) == ROTATE
19064 || GET_CODE (shift_rtx) == ROTATERT))
19066 rtx shift_count = XEXP (shift_rtx, 1);
19068 /* Return true if shift count is dest of SET_BODY. */
19069 if (REG_P (shift_count))
19071 /* Add check since it can be invoked before register
19072 allocation in pre-reload schedule. */
19073 if (reload_completed
19074 && true_regnum (set_dest) == true_regnum (shift_count))
19076 else if (REGNO(set_dest) == REGNO(shift_count))
19084 /* Return true if destination reg of SET_INSN is shift count of
19088 ix86_dep_by_shift_count (const_rtx set_insn, const_rtx use_insn)
19090 return ix86_dep_by_shift_count_body (PATTERN (set_insn),
19091 PATTERN (use_insn));
19094 /* Return TRUE or FALSE depending on whether the unary operator meets the
19095 appropriate constraints. */
19098 ix86_unary_operator_ok (enum rtx_code,
19102 /* If one of operands is memory, source and destination must match. */
19103 if ((MEM_P (operands[0])
19104 || MEM_P (operands[1]))
19105 && ! rtx_equal_p (operands[0], operands[1]))
19110 /* Return TRUE if the operands to a vec_interleave_{high,low}v2df
19111 are ok, keeping in mind the possible movddup alternative. */
19114 ix86_vec_interleave_v2df_operator_ok (rtx operands[3], bool high)
19116 if (MEM_P (operands[0]))
19117 return rtx_equal_p (operands[0], operands[1 + high]);
19118 if (MEM_P (operands[1]) && MEM_P (operands[2]))
19119 return TARGET_SSE3 && rtx_equal_p (operands[1], operands[2]);
19123 /* Post-reload splitter for converting an SF or DFmode value in an
19124 SSE register into an unsigned SImode. */
19127 ix86_split_convert_uns_si_sse (rtx operands[])
19129 machine_mode vecmode;
19130 rtx value, large, zero_or_two31, input, two31, x;
19132 large = operands[1];
19133 zero_or_two31 = operands[2];
19134 input = operands[3];
19135 two31 = operands[4];
19136 vecmode = GET_MODE (large);
19137 value = gen_rtx_REG (vecmode, REGNO (operands[0]));
19139 /* Load up the value into the low element. We must ensure that the other
19140 elements are valid floats -- zero is the easiest such value. */
19143 if (vecmode == V4SFmode)
19144 emit_insn (gen_vec_setv4sf_0 (value, CONST0_RTX (V4SFmode), input));
19146 emit_insn (gen_sse2_loadlpd (value, CONST0_RTX (V2DFmode), input));
19150 input = gen_rtx_REG (vecmode, REGNO (input));
19151 emit_move_insn (value, CONST0_RTX (vecmode));
19152 if (vecmode == V4SFmode)
19153 emit_insn (gen_sse_movss (value, value, input));
19155 emit_insn (gen_sse2_movsd (value, value, input));
19158 emit_move_insn (large, two31);
19159 emit_move_insn (zero_or_two31, MEM_P (two31) ? large : two31);
19161 x = gen_rtx_fmt_ee (LE, vecmode, large, value);
19162 emit_insn (gen_rtx_SET (large, x));
19164 x = gen_rtx_AND (vecmode, zero_or_two31, large);
19165 emit_insn (gen_rtx_SET (zero_or_two31, x));
19167 x = gen_rtx_MINUS (vecmode, value, zero_or_two31);
19168 emit_insn (gen_rtx_SET (value, x));
19170 large = gen_rtx_REG (V4SImode, REGNO (large));
19171 emit_insn (gen_ashlv4si3 (large, large, GEN_INT (31)));
19173 x = gen_rtx_REG (V4SImode, REGNO (value));
19174 if (vecmode == V4SFmode)
19175 emit_insn (gen_fix_truncv4sfv4si2 (x, value));
19177 emit_insn (gen_sse2_cvttpd2dq (x, value));
19180 emit_insn (gen_xorv4si3 (value, value, large));
19183 /* Convert an unsigned DImode value into a DFmode, using only SSE.
19184 Expects the 64-bit DImode to be supplied in a pair of integral
19185 registers. Requires SSE2; will use SSE3 if available. For x86_32,
19186 -mfpmath=sse, !optimize_size only. */
19189 ix86_expand_convert_uns_didf_sse (rtx target, rtx input)
19191 REAL_VALUE_TYPE bias_lo_rvt, bias_hi_rvt;
19192 rtx int_xmm, fp_xmm;
19193 rtx biases, exponents;
19196 int_xmm = gen_reg_rtx (V4SImode);
19197 if (TARGET_INTER_UNIT_MOVES_TO_VEC)
19198 emit_insn (gen_movdi_to_sse (int_xmm, input));
19199 else if (TARGET_SSE_SPLIT_REGS)
19201 emit_clobber (int_xmm);
19202 emit_move_insn (gen_lowpart (DImode, int_xmm), input);
19206 x = gen_reg_rtx (V2DImode);
19207 ix86_expand_vector_init_one_nonzero (false, V2DImode, x, input, 0);
19208 emit_move_insn (int_xmm, gen_lowpart (V4SImode, x));
19211 x = gen_rtx_CONST_VECTOR (V4SImode,
19212 gen_rtvec (4, GEN_INT (0x43300000UL),
19213 GEN_INT (0x45300000UL),
19214 const0_rtx, const0_rtx));
19215 exponents = validize_mem (force_const_mem (V4SImode, x));
19217 /* int_xmm = {0x45300000UL, fp_xmm/hi, 0x43300000, fp_xmm/lo } */
19218 emit_insn (gen_vec_interleave_lowv4si (int_xmm, int_xmm, exponents));
19220 /* Concatenating (juxtaposing) (0x43300000UL ## fp_value_low_xmm)
19221 yields a valid DF value equal to (0x1.0p52 + double(fp_value_lo_xmm)).
19222 Similarly (0x45300000UL ## fp_value_hi_xmm) yields
19223 (0x1.0p84 + double(fp_value_hi_xmm)).
19224 Note these exponents differ by 32. */
19226 fp_xmm = copy_to_mode_reg (V2DFmode, gen_lowpart (V2DFmode, int_xmm));
19228 /* Subtract off those 0x1.0p52 and 0x1.0p84 biases, to produce values
19229 in [0,2**32-1] and [0]+[2**32,2**64-1] respectively. */
19230 real_ldexp (&bias_lo_rvt, &dconst1, 52);
19231 real_ldexp (&bias_hi_rvt, &dconst1, 84);
19232 biases = const_double_from_real_value (bias_lo_rvt, DFmode);
19233 x = const_double_from_real_value (bias_hi_rvt, DFmode);
19234 biases = gen_rtx_CONST_VECTOR (V2DFmode, gen_rtvec (2, biases, x));
19235 biases = validize_mem (force_const_mem (V2DFmode, biases));
19236 emit_insn (gen_subv2df3 (fp_xmm, fp_xmm, biases));
19238 /* Add the upper and lower DFmode values together. */
19240 emit_insn (gen_sse3_haddv2df3 (fp_xmm, fp_xmm, fp_xmm));
19243 x = copy_to_mode_reg (V2DFmode, fp_xmm);
19244 emit_insn (gen_vec_interleave_highv2df (fp_xmm, fp_xmm, fp_xmm));
19245 emit_insn (gen_addv2df3 (fp_xmm, fp_xmm, x));
19248 ix86_expand_vector_extract (false, target, fp_xmm, 0);
19251 /* Not used, but eases macroization of patterns. */
19253 ix86_expand_convert_uns_sixf_sse (rtx, rtx)
19255 gcc_unreachable ();
19258 /* Convert an unsigned SImode value into a DFmode. Only currently used
19259 for SSE, but applicable anywhere. */
19262 ix86_expand_convert_uns_sidf_sse (rtx target, rtx input)
19264 REAL_VALUE_TYPE TWO31r;
19267 x = expand_simple_binop (SImode, PLUS, input, GEN_INT (-2147483647 - 1),
19268 NULL, 1, OPTAB_DIRECT);
19270 fp = gen_reg_rtx (DFmode);
19271 emit_insn (gen_floatsidf2 (fp, x));
19273 real_ldexp (&TWO31r, &dconst1, 31);
19274 x = const_double_from_real_value (TWO31r, DFmode);
19276 x = expand_simple_binop (DFmode, PLUS, fp, x, target, 0, OPTAB_DIRECT);
19278 emit_move_insn (target, x);
19281 /* Convert a signed DImode value into a DFmode. Only used for SSE in
19282 32-bit mode; otherwise we have a direct convert instruction. */
19285 ix86_expand_convert_sign_didf_sse (rtx target, rtx input)
19287 REAL_VALUE_TYPE TWO32r;
19288 rtx fp_lo, fp_hi, x;
19290 fp_lo = gen_reg_rtx (DFmode);
19291 fp_hi = gen_reg_rtx (DFmode);
19293 emit_insn (gen_floatsidf2 (fp_hi, gen_highpart (SImode, input)));
19295 real_ldexp (&TWO32r, &dconst1, 32);
19296 x = const_double_from_real_value (TWO32r, DFmode);
19297 fp_hi = expand_simple_binop (DFmode, MULT, fp_hi, x, fp_hi, 0, OPTAB_DIRECT);
19299 ix86_expand_convert_uns_sidf_sse (fp_lo, gen_lowpart (SImode, input));
19301 x = expand_simple_binop (DFmode, PLUS, fp_hi, fp_lo, target,
19304 emit_move_insn (target, x);
19307 /* Convert an unsigned SImode value into a SFmode, using only SSE.
19308 For x86_32, -mfpmath=sse, !optimize_size only. */
19310 ix86_expand_convert_uns_sisf_sse (rtx target, rtx input)
19312 REAL_VALUE_TYPE ONE16r;
19313 rtx fp_hi, fp_lo, int_hi, int_lo, x;
19315 real_ldexp (&ONE16r, &dconst1, 16);
19316 x = const_double_from_real_value (ONE16r, SFmode);
19317 int_lo = expand_simple_binop (SImode, AND, input, GEN_INT(0xffff),
19318 NULL, 0, OPTAB_DIRECT);
19319 int_hi = expand_simple_binop (SImode, LSHIFTRT, input, GEN_INT(16),
19320 NULL, 0, OPTAB_DIRECT);
19321 fp_hi = gen_reg_rtx (SFmode);
19322 fp_lo = gen_reg_rtx (SFmode);
19323 emit_insn (gen_floatsisf2 (fp_hi, int_hi));
19324 emit_insn (gen_floatsisf2 (fp_lo, int_lo));
19325 fp_hi = expand_simple_binop (SFmode, MULT, fp_hi, x, fp_hi,
19327 fp_hi = expand_simple_binop (SFmode, PLUS, fp_hi, fp_lo, target,
19329 if (!rtx_equal_p (target, fp_hi))
19330 emit_move_insn (target, fp_hi);
19333 /* floatunsv{4,8}siv{4,8}sf2 expander. Expand code to convert
19334 a vector of unsigned ints VAL to vector of floats TARGET. */
19337 ix86_expand_vector_convert_uns_vsivsf (rtx target, rtx val)
19340 REAL_VALUE_TYPE TWO16r;
19341 machine_mode intmode = GET_MODE (val);
19342 machine_mode fltmode = GET_MODE (target);
19343 rtx (*cvt) (rtx, rtx);
19345 if (intmode == V4SImode)
19346 cvt = gen_floatv4siv4sf2;
19348 cvt = gen_floatv8siv8sf2;
19349 tmp[0] = ix86_build_const_vector (intmode, 1, GEN_INT (0xffff));
19350 tmp[0] = force_reg (intmode, tmp[0]);
19351 tmp[1] = expand_simple_binop (intmode, AND, val, tmp[0], NULL_RTX, 1,
19353 tmp[2] = expand_simple_binop (intmode, LSHIFTRT, val, GEN_INT (16),
19354 NULL_RTX, 1, OPTAB_DIRECT);
19355 tmp[3] = gen_reg_rtx (fltmode);
19356 emit_insn (cvt (tmp[3], tmp[1]));
19357 tmp[4] = gen_reg_rtx (fltmode);
19358 emit_insn (cvt (tmp[4], tmp[2]));
19359 real_ldexp (&TWO16r, &dconst1, 16);
19360 tmp[5] = const_double_from_real_value (TWO16r, SFmode);
19361 tmp[5] = force_reg (fltmode, ix86_build_const_vector (fltmode, 1, tmp[5]));
19362 tmp[6] = expand_simple_binop (fltmode, MULT, tmp[4], tmp[5], NULL_RTX, 1,
19364 tmp[7] = expand_simple_binop (fltmode, PLUS, tmp[3], tmp[6], target, 1,
19366 if (tmp[7] != target)
19367 emit_move_insn (target, tmp[7]);
19370 /* Adjust a V*SFmode/V*DFmode value VAL so that *sfix_trunc* resp. fix_trunc*
19371 pattern can be used on it instead of *ufix_trunc* resp. fixuns_trunc*.
19372 This is done by doing just signed conversion if < 0x1p31, and otherwise by
19373 subtracting 0x1p31 first and xoring in 0x80000000 from *XORP afterwards. */
19376 ix86_expand_adjust_ufix_to_sfix_si (rtx val, rtx *xorp)
19378 REAL_VALUE_TYPE TWO31r;
19379 rtx two31r, tmp[4];
19380 machine_mode mode = GET_MODE (val);
19381 machine_mode scalarmode = GET_MODE_INNER (mode);
19382 machine_mode intmode = GET_MODE_SIZE (mode) == 32 ? V8SImode : V4SImode;
19383 rtx (*cmp) (rtx, rtx, rtx, rtx);
19386 for (i = 0; i < 3; i++)
19387 tmp[i] = gen_reg_rtx (mode);
19388 real_ldexp (&TWO31r, &dconst1, 31);
19389 two31r = const_double_from_real_value (TWO31r, scalarmode);
19390 two31r = ix86_build_const_vector (mode, 1, two31r);
19391 two31r = force_reg (mode, two31r);
19394 case V8SFmode: cmp = gen_avx_maskcmpv8sf3; break;
19395 case V4SFmode: cmp = gen_sse_maskcmpv4sf3; break;
19396 case V4DFmode: cmp = gen_avx_maskcmpv4df3; break;
19397 case V2DFmode: cmp = gen_sse2_maskcmpv2df3; break;
19398 default: gcc_unreachable ();
19400 tmp[3] = gen_rtx_LE (mode, two31r, val);
19401 emit_insn (cmp (tmp[0], two31r, val, tmp[3]));
19402 tmp[1] = expand_simple_binop (mode, AND, tmp[0], two31r, tmp[1],
19404 if (intmode == V4SImode || TARGET_AVX2)
19405 *xorp = expand_simple_binop (intmode, ASHIFT,
19406 gen_lowpart (intmode, tmp[0]),
19407 GEN_INT (31), NULL_RTX, 0,
19411 rtx two31 = GEN_INT (HOST_WIDE_INT_1U << 31);
19412 two31 = ix86_build_const_vector (intmode, 1, two31);
19413 *xorp = expand_simple_binop (intmode, AND,
19414 gen_lowpart (intmode, tmp[0]),
19415 two31, NULL_RTX, 0,
19418 return expand_simple_binop (mode, MINUS, val, tmp[1], tmp[2],
19422 /* A subroutine of ix86_build_signbit_mask. If VECT is true,
19423 then replicate the value for all elements of the vector
19427 ix86_build_const_vector (machine_mode mode, bool vect, rtx value)
19431 machine_mode scalar_mode;
19454 n_elt = GET_MODE_NUNITS (mode);
19455 v = rtvec_alloc (n_elt);
19456 scalar_mode = GET_MODE_INNER (mode);
19458 RTVEC_ELT (v, 0) = value;
19460 for (i = 1; i < n_elt; ++i)
19461 RTVEC_ELT (v, i) = vect ? value : CONST0_RTX (scalar_mode);
19463 return gen_rtx_CONST_VECTOR (mode, v);
19466 gcc_unreachable ();
19470 /* A subroutine of ix86_expand_fp_absneg_operator, copysign expanders
19471 and ix86_expand_int_vcond. Create a mask for the sign bit in MODE
19472 for an SSE register. If VECT is true, then replicate the mask for
19473 all elements of the vector register. If INVERT is true, then create
19474 a mask excluding the sign bit. */
19477 ix86_build_signbit_mask (machine_mode mode, bool vect, bool invert)
19479 machine_mode vec_mode, imode;
19492 mode = GET_MODE_INNER (mode);
19503 mode = GET_MODE_INNER (mode);
19509 vec_mode = VOIDmode;
19514 gcc_unreachable ();
19517 w = wi::set_bit_in_zero (GET_MODE_BITSIZE (mode) - 1,
19518 GET_MODE_BITSIZE (mode));
19520 w = wi::bit_not (w);
19522 /* Force this value into the low part of a fp vector constant. */
19523 mask = immed_wide_int_const (w, imode);
19524 mask = gen_lowpart (mode, mask);
19526 if (vec_mode == VOIDmode)
19527 return force_reg (mode, mask);
19529 v = ix86_build_const_vector (vec_mode, vect, mask);
19530 return force_reg (vec_mode, v);
19533 /* Generate code for floating point ABS or NEG. */
19536 ix86_expand_fp_absneg_operator (enum rtx_code code, machine_mode mode,
19539 rtx mask, set, dst, src;
19540 bool use_sse = false;
19541 bool vector_mode = VECTOR_MODE_P (mode);
19542 machine_mode vmode = mode;
19546 else if (mode == TFmode)
19548 else if (TARGET_SSE_MATH)
19550 use_sse = SSE_FLOAT_MODE_P (mode);
19551 if (mode == SFmode)
19553 else if (mode == DFmode)
19557 /* NEG and ABS performed with SSE use bitwise mask operations.
19558 Create the appropriate mask now. */
19560 mask = ix86_build_signbit_mask (vmode, vector_mode, code == ABS);
19567 set = gen_rtx_fmt_e (code, mode, src);
19568 set = gen_rtx_SET (dst, set);
19575 use = gen_rtx_USE (VOIDmode, mask);
19577 par = gen_rtvec (2, set, use);
19580 clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
19581 par = gen_rtvec (3, set, use, clob);
19583 emit_insn (gen_rtx_PARALLEL (VOIDmode, par));
19589 /* Expand a copysign operation. Special case operand 0 being a constant. */
19592 ix86_expand_copysign (rtx operands[])
19594 machine_mode mode, vmode;
19595 rtx dest, op0, op1, mask, nmask;
19597 dest = operands[0];
19601 mode = GET_MODE (dest);
19603 if (mode == SFmode)
19605 else if (mode == DFmode)
19610 if (CONST_DOUBLE_P (op0))
19612 rtx (*copysign_insn)(rtx, rtx, rtx, rtx);
19614 if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0)))
19615 op0 = simplify_unary_operation (ABS, mode, op0, mode);
19617 if (mode == SFmode || mode == DFmode)
19619 if (op0 == CONST0_RTX (mode))
19620 op0 = CONST0_RTX (vmode);
19623 rtx v = ix86_build_const_vector (vmode, false, op0);
19625 op0 = force_reg (vmode, v);
19628 else if (op0 != CONST0_RTX (mode))
19629 op0 = force_reg (mode, op0);
19631 mask = ix86_build_signbit_mask (vmode, 0, 0);
19633 if (mode == SFmode)
19634 copysign_insn = gen_copysignsf3_const;
19635 else if (mode == DFmode)
19636 copysign_insn = gen_copysigndf3_const;
19638 copysign_insn = gen_copysigntf3_const;
19640 emit_insn (copysign_insn (dest, op0, op1, mask));
19644 rtx (*copysign_insn)(rtx, rtx, rtx, rtx, rtx, rtx);
19646 nmask = ix86_build_signbit_mask (vmode, 0, 1);
19647 mask = ix86_build_signbit_mask (vmode, 0, 0);
19649 if (mode == SFmode)
19650 copysign_insn = gen_copysignsf3_var;
19651 else if (mode == DFmode)
19652 copysign_insn = gen_copysigndf3_var;
19654 copysign_insn = gen_copysigntf3_var;
19656 emit_insn (copysign_insn (dest, NULL_RTX, op0, op1, nmask, mask));
19660 /* Deconstruct a copysign operation into bit masks. Operand 0 is known to
19661 be a constant, and so has already been expanded into a vector constant. */
19664 ix86_split_copysign_const (rtx operands[])
19666 machine_mode mode, vmode;
19667 rtx dest, op0, mask, x;
19669 dest = operands[0];
19671 mask = operands[3];
19673 mode = GET_MODE (dest);
19674 vmode = GET_MODE (mask);
19676 dest = simplify_gen_subreg (vmode, dest, mode, 0);
19677 x = gen_rtx_AND (vmode, dest, mask);
19678 emit_insn (gen_rtx_SET (dest, x));
19680 if (op0 != CONST0_RTX (vmode))
19682 x = gen_rtx_IOR (vmode, dest, op0);
19683 emit_insn (gen_rtx_SET (dest, x));
19687 /* Deconstruct a copysign operation into bit masks. Operand 0 is variable,
19688 so we have to do two masks. */
19691 ix86_split_copysign_var (rtx operands[])
19693 machine_mode mode, vmode;
19694 rtx dest, scratch, op0, op1, mask, nmask, x;
19696 dest = operands[0];
19697 scratch = operands[1];
19700 nmask = operands[4];
19701 mask = operands[5];
19703 mode = GET_MODE (dest);
19704 vmode = GET_MODE (mask);
19706 if (rtx_equal_p (op0, op1))
19708 /* Shouldn't happen often (it's useless, obviously), but when it does
19709 we'd generate incorrect code if we continue below. */
19710 emit_move_insn (dest, op0);
19714 if (REG_P (mask) && REGNO (dest) == REGNO (mask)) /* alternative 0 */
19716 gcc_assert (REGNO (op1) == REGNO (scratch));
19718 x = gen_rtx_AND (vmode, scratch, mask);
19719 emit_insn (gen_rtx_SET (scratch, x));
19722 op0 = simplify_gen_subreg (vmode, op0, mode, 0);
19723 x = gen_rtx_NOT (vmode, dest);
19724 x = gen_rtx_AND (vmode, x, op0);
19725 emit_insn (gen_rtx_SET (dest, x));
19729 if (REGNO (op1) == REGNO (scratch)) /* alternative 1,3 */
19731 x = gen_rtx_AND (vmode, scratch, mask);
19733 else /* alternative 2,4 */
19735 gcc_assert (REGNO (mask) == REGNO (scratch));
19736 op1 = simplify_gen_subreg (vmode, op1, mode, 0);
19737 x = gen_rtx_AND (vmode, scratch, op1);
19739 emit_insn (gen_rtx_SET (scratch, x));
19741 if (REGNO (op0) == REGNO (dest)) /* alternative 1,2 */
19743 dest = simplify_gen_subreg (vmode, op0, mode, 0);
19744 x = gen_rtx_AND (vmode, dest, nmask);
19746 else /* alternative 3,4 */
19748 gcc_assert (REGNO (nmask) == REGNO (dest));
19750 op0 = simplify_gen_subreg (vmode, op0, mode, 0);
19751 x = gen_rtx_AND (vmode, dest, op0);
19753 emit_insn (gen_rtx_SET (dest, x));
19756 x = gen_rtx_IOR (vmode, dest, scratch);
19757 emit_insn (gen_rtx_SET (dest, x));
19760 /* Return TRUE or FALSE depending on whether the first SET in INSN
19761 has source and destination with matching CC modes, and that the
19762 CC mode is at least as constrained as REQ_MODE. */
19765 ix86_match_ccmode (rtx insn, machine_mode req_mode)
19768 machine_mode set_mode;
19770 set = PATTERN (insn);
19771 if (GET_CODE (set) == PARALLEL)
19772 set = XVECEXP (set, 0, 0);
19773 gcc_assert (GET_CODE (set) == SET);
19774 gcc_assert (GET_CODE (SET_SRC (set)) == COMPARE);
19776 set_mode = GET_MODE (SET_DEST (set));
19780 if (req_mode != CCNOmode
19781 && (req_mode != CCmode
19782 || XEXP (SET_SRC (set), 1) != const0_rtx))
19786 if (req_mode == CCGCmode)
19790 if (req_mode == CCGOCmode || req_mode == CCNOmode)
19794 if (req_mode == CCZmode)
19805 if (set_mode != req_mode)
19810 gcc_unreachable ();
19813 return GET_MODE (SET_SRC (set)) == set_mode;
19816 /* Generate insn patterns to do an integer compare of OPERANDS. */
19819 ix86_expand_int_compare (enum rtx_code code, rtx op0, rtx op1)
19821 machine_mode cmpmode;
19824 cmpmode = SELECT_CC_MODE (code, op0, op1);
19825 flags = gen_rtx_REG (cmpmode, FLAGS_REG);
19827 /* This is very simple, but making the interface the same as in the
19828 FP case makes the rest of the code easier. */
19829 tmp = gen_rtx_COMPARE (cmpmode, op0, op1);
19830 emit_insn (gen_rtx_SET (flags, tmp));
19832 /* Return the test that should be put into the flags user, i.e.
19833 the bcc, scc, or cmov instruction. */
19834 return gen_rtx_fmt_ee (code, VOIDmode, flags, const0_rtx);
19837 /* Figure out whether to use ordered or unordered fp comparisons.
19838 Return the appropriate mode to use. */
19841 ix86_fp_compare_mode (enum rtx_code)
19843 /* ??? In order to make all comparisons reversible, we do all comparisons
19844 non-trapping when compiling for IEEE. Once gcc is able to distinguish
19845 all forms trapping and nontrapping comparisons, we can make inequality
19846 comparisons trapping again, since it results in better code when using
19847 FCOM based compares. */
19848 return TARGET_IEEE_FP ? CCFPUmode : CCFPmode;
19852 ix86_cc_mode (enum rtx_code code, rtx op0, rtx op1)
19854 machine_mode mode = GET_MODE (op0);
19856 if (SCALAR_FLOAT_MODE_P (mode))
19858 gcc_assert (!DECIMAL_FLOAT_MODE_P (mode));
19859 return ix86_fp_compare_mode (code);
19864 /* Only zero flag is needed. */
19865 case EQ: /* ZF=0 */
19866 case NE: /* ZF!=0 */
19868 /* Codes needing carry flag. */
19869 case GEU: /* CF=0 */
19870 case LTU: /* CF=1 */
19871 /* Detect overflow checks. They need just the carry flag. */
19872 if (GET_CODE (op0) == PLUS
19873 && rtx_equal_p (op1, XEXP (op0, 0)))
19877 case GTU: /* CF=0 & ZF=0 */
19878 case LEU: /* CF=1 | ZF=1 */
19880 /* Codes possibly doable only with sign flag when
19881 comparing against zero. */
19882 case GE: /* SF=OF or SF=0 */
19883 case LT: /* SF<>OF or SF=1 */
19884 if (op1 == const0_rtx)
19887 /* For other cases Carry flag is not required. */
19889 /* Codes doable only with sign flag when comparing
19890 against zero, but we miss jump instruction for it
19891 so we need to use relational tests against overflow
19892 that thus needs to be zero. */
19893 case GT: /* ZF=0 & SF=OF */
19894 case LE: /* ZF=1 | SF<>OF */
19895 if (op1 == const0_rtx)
19899 /* strcmp pattern do (use flags) and combine may ask us for proper
19904 gcc_unreachable ();
19908 /* Return the fixed registers used for condition codes. */
19911 ix86_fixed_condition_code_regs (unsigned int *p1, unsigned int *p2)
19918 /* If two condition code modes are compatible, return a condition code
19919 mode which is compatible with both. Otherwise, return
19922 static machine_mode
19923 ix86_cc_modes_compatible (machine_mode m1, machine_mode m2)
19928 if (GET_MODE_CLASS (m1) != MODE_CC || GET_MODE_CLASS (m2) != MODE_CC)
19931 if ((m1 == CCGCmode && m2 == CCGOCmode)
19932 || (m1 == CCGOCmode && m2 == CCGCmode))
19935 if (m1 == CCZmode && (m2 == CCGCmode || m2 == CCGOCmode))
19937 else if (m2 == CCZmode && (m1 == CCGCmode || m1 == CCGOCmode))
19943 gcc_unreachable ();
19975 /* These are only compatible with themselves, which we already
19982 /* Return a comparison we can do and that it is equivalent to
19983 swap_condition (code) apart possibly from orderedness.
19984 But, never change orderedness if TARGET_IEEE_FP, returning
19985 UNKNOWN in that case if necessary. */
19987 static enum rtx_code
19988 ix86_fp_swap_condition (enum rtx_code code)
19992 case GT: /* GTU - CF=0 & ZF=0 */
19993 return TARGET_IEEE_FP ? UNKNOWN : UNLT;
19994 case GE: /* GEU - CF=0 */
19995 return TARGET_IEEE_FP ? UNKNOWN : UNLE;
19996 case UNLT: /* LTU - CF=1 */
19997 return TARGET_IEEE_FP ? UNKNOWN : GT;
19998 case UNLE: /* LEU - CF=1 | ZF=1 */
19999 return TARGET_IEEE_FP ? UNKNOWN : GE;
20001 return swap_condition (code);
20005 /* Return cost of comparison CODE using the best strategy for performance.
20006 All following functions do use number of instructions as a cost metrics.
20007 In future this should be tweaked to compute bytes for optimize_size and
20008 take into account performance of various instructions on various CPUs. */
20011 ix86_fp_comparison_cost (enum rtx_code code)
20015 /* The cost of code using bit-twiddling on %ah. */
20032 arith_cost = TARGET_IEEE_FP ? 5 : 4;
20036 arith_cost = TARGET_IEEE_FP ? 6 : 4;
20039 gcc_unreachable ();
20042 switch (ix86_fp_comparison_strategy (code))
20044 case IX86_FPCMP_COMI:
20045 return arith_cost > 4 ? 3 : 2;
20046 case IX86_FPCMP_SAHF:
20047 return arith_cost > 4 ? 4 : 3;
20053 /* Return strategy to use for floating-point. We assume that fcomi is always
20054 preferrable where available, since that is also true when looking at size
20055 (2 bytes, vs. 3 for fnstsw+sahf and at least 5 for fnstsw+test). */
20057 enum ix86_fpcmp_strategy
20058 ix86_fp_comparison_strategy (enum rtx_code)
20060 /* Do fcomi/sahf based test when profitable. */
20063 return IX86_FPCMP_COMI;
20065 if (TARGET_SAHF && (TARGET_USE_SAHF || optimize_insn_for_size_p ()))
20066 return IX86_FPCMP_SAHF;
20068 return IX86_FPCMP_ARITH;
20071 /* Swap, force into registers, or otherwise massage the two operands
20072 to a fp comparison. The operands are updated in place; the new
20073 comparison code is returned. */
20075 static enum rtx_code
20076 ix86_prepare_fp_compare_args (enum rtx_code code, rtx *pop0, rtx *pop1)
20078 machine_mode fpcmp_mode = ix86_fp_compare_mode (code);
20079 rtx op0 = *pop0, op1 = *pop1;
20080 machine_mode op_mode = GET_MODE (op0);
20081 int is_sse = TARGET_SSE_MATH && SSE_FLOAT_MODE_P (op_mode);
20083 /* All of the unordered compare instructions only work on registers.
20084 The same is true of the fcomi compare instructions. The XFmode
20085 compare instructions require registers except when comparing
20086 against zero or when converting operand 1 from fixed point to
20090 && (fpcmp_mode == CCFPUmode
20091 || (op_mode == XFmode
20092 && ! (standard_80387_constant_p (op0) == 1
20093 || standard_80387_constant_p (op1) == 1)
20094 && GET_CODE (op1) != FLOAT)
20095 || ix86_fp_comparison_strategy (code) == IX86_FPCMP_COMI))
20097 op0 = force_reg (op_mode, op0);
20098 op1 = force_reg (op_mode, op1);
20102 /* %%% We only allow op1 in memory; op0 must be st(0). So swap
20103 things around if they appear profitable, otherwise force op0
20104 into a register. */
20106 if (standard_80387_constant_p (op0) == 0
20108 && ! (standard_80387_constant_p (op1) == 0
20111 enum rtx_code new_code = ix86_fp_swap_condition (code);
20112 if (new_code != UNKNOWN)
20114 std::swap (op0, op1);
20120 op0 = force_reg (op_mode, op0);
20122 if (CONSTANT_P (op1))
20124 int tmp = standard_80387_constant_p (op1);
20126 op1 = validize_mem (force_const_mem (op_mode, op1));
20130 op1 = force_reg (op_mode, op1);
20133 op1 = force_reg (op_mode, op1);
20137 /* Try to rearrange the comparison to make it cheaper. */
20138 if (ix86_fp_comparison_cost (code)
20139 > ix86_fp_comparison_cost (swap_condition (code))
20140 && (REG_P (op1) || can_create_pseudo_p ()))
20142 std::swap (op0, op1);
20143 code = swap_condition (code);
20145 op0 = force_reg (op_mode, op0);
20153 /* Convert comparison codes we use to represent FP comparison to integer
20154 code that will result in proper branch. Return UNKNOWN if no such code
20158 ix86_fp_compare_code_to_integer (enum rtx_code code)
20187 /* Generate insn patterns to do a floating point compare of OPERANDS. */
20190 ix86_expand_fp_compare (enum rtx_code code, rtx op0, rtx op1, rtx scratch)
20192 machine_mode fpcmp_mode, intcmp_mode;
20195 fpcmp_mode = ix86_fp_compare_mode (code);
20196 code = ix86_prepare_fp_compare_args (code, &op0, &op1);
20198 /* Do fcomi/sahf based test when profitable. */
20199 switch (ix86_fp_comparison_strategy (code))
20201 case IX86_FPCMP_COMI:
20202 intcmp_mode = fpcmp_mode;
20203 tmp = gen_rtx_COMPARE (fpcmp_mode, op0, op1);
20204 tmp = gen_rtx_SET (gen_rtx_REG (fpcmp_mode, FLAGS_REG), tmp);
20208 case IX86_FPCMP_SAHF:
20209 intcmp_mode = fpcmp_mode;
20210 tmp = gen_rtx_COMPARE (fpcmp_mode, op0, op1);
20211 tmp = gen_rtx_SET (gen_rtx_REG (fpcmp_mode, FLAGS_REG), tmp);
20214 scratch = gen_reg_rtx (HImode);
20215 tmp2 = gen_rtx_CLOBBER (VOIDmode, scratch);
20216 emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, tmp, tmp2)));
20219 case IX86_FPCMP_ARITH:
20220 /* Sadness wrt reg-stack pops killing fpsr -- gotta get fnstsw first. */
20221 tmp = gen_rtx_COMPARE (fpcmp_mode, op0, op1);
20222 tmp2 = gen_rtx_UNSPEC (HImode, gen_rtvec (1, tmp), UNSPEC_FNSTSW);
20224 scratch = gen_reg_rtx (HImode);
20225 emit_insn (gen_rtx_SET (scratch, tmp2));
20227 /* In the unordered case, we have to check C2 for NaN's, which
20228 doesn't happen to work out to anything nice combination-wise.
20229 So do some bit twiddling on the value we've got in AH to come
20230 up with an appropriate set of condition codes. */
20232 intcmp_mode = CCNOmode;
20237 if (code == GT || !TARGET_IEEE_FP)
20239 emit_insn (gen_testqi_ext_ccno_0 (scratch, GEN_INT (0x45)));
20244 emit_insn (gen_andqi_ext_0 (scratch, scratch, GEN_INT (0x45)));
20245 emit_insn (gen_addqi_ext_1 (scratch, scratch, constm1_rtx));
20246 emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x44)));
20247 intcmp_mode = CCmode;
20253 if (code == LT && TARGET_IEEE_FP)
20255 emit_insn (gen_andqi_ext_0 (scratch, scratch, GEN_INT (0x45)));
20256 emit_insn (gen_cmpqi_ext_3 (scratch, const1_rtx));
20257 intcmp_mode = CCmode;
20262 emit_insn (gen_testqi_ext_ccno_0 (scratch, const1_rtx));
20268 if (code == GE || !TARGET_IEEE_FP)
20270 emit_insn (gen_testqi_ext_ccno_0 (scratch, GEN_INT (0x05)));
20275 emit_insn (gen_andqi_ext_0 (scratch, scratch, GEN_INT (0x45)));
20276 emit_insn (gen_xorqi_cc_ext_1 (scratch, scratch, const1_rtx));
20282 if (code == LE && TARGET_IEEE_FP)
20284 emit_insn (gen_andqi_ext_0 (scratch, scratch, GEN_INT (0x45)));
20285 emit_insn (gen_addqi_ext_1 (scratch, scratch, constm1_rtx));
20286 emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x40)));
20287 intcmp_mode = CCmode;
20292 emit_insn (gen_testqi_ext_ccno_0 (scratch, GEN_INT (0x45)));
20298 if (code == EQ && TARGET_IEEE_FP)
20300 emit_insn (gen_andqi_ext_0 (scratch, scratch, GEN_INT (0x45)));
20301 emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x40)));
20302 intcmp_mode = CCmode;
20307 emit_insn (gen_testqi_ext_ccno_0 (scratch, GEN_INT (0x40)));
20313 if (code == NE && TARGET_IEEE_FP)
20315 emit_insn (gen_andqi_ext_0 (scratch, scratch, GEN_INT (0x45)));
20316 emit_insn (gen_xorqi_cc_ext_1 (scratch, scratch,
20322 emit_insn (gen_testqi_ext_ccno_0 (scratch, GEN_INT (0x40)));
20328 emit_insn (gen_testqi_ext_ccno_0 (scratch, GEN_INT (0x04)));
20332 emit_insn (gen_testqi_ext_ccno_0 (scratch, GEN_INT (0x04)));
20337 gcc_unreachable ();
20345 /* Return the test that should be put into the flags user, i.e.
20346 the bcc, scc, or cmov instruction. */
20347 return gen_rtx_fmt_ee (code, VOIDmode,
20348 gen_rtx_REG (intcmp_mode, FLAGS_REG),
20353 ix86_expand_compare (enum rtx_code code, rtx op0, rtx op1)
20357 if (GET_MODE_CLASS (GET_MODE (op0)) == MODE_CC)
20358 ret = gen_rtx_fmt_ee (code, VOIDmode, op0, op1);
20360 else if (SCALAR_FLOAT_MODE_P (GET_MODE (op0)))
20362 gcc_assert (!DECIMAL_FLOAT_MODE_P (GET_MODE (op0)));
20363 ret = ix86_expand_fp_compare (code, op0, op1, NULL_RTX);
20366 ret = ix86_expand_int_compare (code, op0, op1);
20372 ix86_expand_branch (enum rtx_code code, rtx op0, rtx op1, rtx label)
20374 machine_mode mode = GET_MODE (op0);
20386 tmp = ix86_expand_compare (code, op0, op1);
20387 tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
20388 gen_rtx_LABEL_REF (VOIDmode, label),
20390 emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
20397 /* Expand DImode branch into multiple compare+branch. */
20400 rtx_code_label *label2;
20401 enum rtx_code code1, code2, code3;
20402 machine_mode submode;
20404 if (CONSTANT_P (op0) && !CONSTANT_P (op1))
20406 std::swap (op0, op1);
20407 code = swap_condition (code);
20410 split_double_mode (mode, &op0, 1, lo+0, hi+0);
20411 split_double_mode (mode, &op1, 1, lo+1, hi+1);
20413 submode = mode == DImode ? SImode : DImode;
20415 /* When comparing for equality, we can use (hi0^hi1)|(lo0^lo1) to
20416 avoid two branches. This costs one extra insn, so disable when
20417 optimizing for size. */
20419 if ((code == EQ || code == NE)
20420 && (!optimize_insn_for_size_p ()
20421 || hi[1] == const0_rtx || lo[1] == const0_rtx))
20426 if (hi[1] != const0_rtx)
20427 xor1 = expand_binop (submode, xor_optab, xor1, hi[1],
20428 NULL_RTX, 0, OPTAB_WIDEN);
20431 if (lo[1] != const0_rtx)
20432 xor0 = expand_binop (submode, xor_optab, xor0, lo[1],
20433 NULL_RTX, 0, OPTAB_WIDEN);
20435 tmp = expand_binop (submode, ior_optab, xor1, xor0,
20436 NULL_RTX, 0, OPTAB_WIDEN);
20438 ix86_expand_branch (code, tmp, const0_rtx, label);
20442 /* Otherwise, if we are doing less-than or greater-or-equal-than,
20443 op1 is a constant and the low word is zero, then we can just
20444 examine the high word. Similarly for low word -1 and
20445 less-or-equal-than or greater-than. */
20447 if (CONST_INT_P (hi[1]))
20450 case LT: case LTU: case GE: case GEU:
20451 if (lo[1] == const0_rtx)
20453 ix86_expand_branch (code, hi[0], hi[1], label);
20457 case LE: case LEU: case GT: case GTU:
20458 if (lo[1] == constm1_rtx)
20460 ix86_expand_branch (code, hi[0], hi[1], label);
20468 /* Otherwise, we need two or three jumps. */
20470 label2 = gen_label_rtx ();
20473 code2 = swap_condition (code);
20474 code3 = unsigned_condition (code);
20478 case LT: case GT: case LTU: case GTU:
20481 case LE: code1 = LT; code2 = GT; break;
20482 case GE: code1 = GT; code2 = LT; break;
20483 case LEU: code1 = LTU; code2 = GTU; break;
20484 case GEU: code1 = GTU; code2 = LTU; break;
20486 case EQ: code1 = UNKNOWN; code2 = NE; break;
20487 case NE: code2 = UNKNOWN; break;
20490 gcc_unreachable ();
20495 * if (hi(a) < hi(b)) goto true;
20496 * if (hi(a) > hi(b)) goto false;
20497 * if (lo(a) < lo(b)) goto true;
20501 if (code1 != UNKNOWN)
20502 ix86_expand_branch (code1, hi[0], hi[1], label);
20503 if (code2 != UNKNOWN)
20504 ix86_expand_branch (code2, hi[0], hi[1], label2);
20506 ix86_expand_branch (code3, lo[0], lo[1], label);
20508 if (code2 != UNKNOWN)
20509 emit_label (label2);
20514 gcc_assert (GET_MODE_CLASS (GET_MODE (op0)) == MODE_CC);
20519 /* Split branch based on floating point condition. */
20521 ix86_split_fp_branch (enum rtx_code code, rtx op1, rtx op2,
20522 rtx target1, rtx target2, rtx tmp)
20527 if (target2 != pc_rtx)
20529 std::swap (target1, target2);
20530 code = reverse_condition_maybe_unordered (code);
20533 condition = ix86_expand_fp_compare (code, op1, op2,
20536 i = emit_jump_insn (gen_rtx_SET
20538 gen_rtx_IF_THEN_ELSE (VOIDmode,
20539 condition, target1, target2)));
20540 if (split_branch_probability >= 0)
20541 add_int_reg_note (i, REG_BR_PROB, split_branch_probability);
20545 ix86_expand_setcc (rtx dest, enum rtx_code code, rtx op0, rtx op1)
20549 gcc_assert (GET_MODE (dest) == QImode);
20551 ret = ix86_expand_compare (code, op0, op1);
20552 PUT_MODE (ret, QImode);
20553 emit_insn (gen_rtx_SET (dest, ret));
20556 /* Expand comparison setting or clearing carry flag. Return true when
20557 successful and set pop for the operation. */
20559 ix86_expand_carry_flag_compare (enum rtx_code code, rtx op0, rtx op1, rtx *pop)
20561 machine_mode mode =
20562 GET_MODE (op0) != VOIDmode ? GET_MODE (op0) : GET_MODE (op1);
20564 /* Do not handle double-mode compares that go through special path. */
20565 if (mode == (TARGET_64BIT ? TImode : DImode))
20568 if (SCALAR_FLOAT_MODE_P (mode))
20571 rtx_insn *compare_seq;
20573 gcc_assert (!DECIMAL_FLOAT_MODE_P (mode));
20575 /* Shortcut: following common codes never translate
20576 into carry flag compares. */
20577 if (code == EQ || code == NE || code == UNEQ || code == LTGT
20578 || code == ORDERED || code == UNORDERED)
20581 /* These comparisons require zero flag; swap operands so they won't. */
20582 if ((code == GT || code == UNLE || code == LE || code == UNGT)
20583 && !TARGET_IEEE_FP)
20585 std::swap (op0, op1);
20586 code = swap_condition (code);
20589 /* Try to expand the comparison and verify that we end up with
20590 carry flag based comparison. This fails to be true only when
20591 we decide to expand comparison using arithmetic that is not
20592 too common scenario. */
20594 compare_op = ix86_expand_fp_compare (code, op0, op1, NULL_RTX);
20595 compare_seq = get_insns ();
20598 if (GET_MODE (XEXP (compare_op, 0)) == CCFPmode
20599 || GET_MODE (XEXP (compare_op, 0)) == CCFPUmode)
20600 code = ix86_fp_compare_code_to_integer (GET_CODE (compare_op));
20602 code = GET_CODE (compare_op);
20604 if (code != LTU && code != GEU)
20607 emit_insn (compare_seq);
20612 if (!INTEGRAL_MODE_P (mode))
20621 /* Convert a==0 into (unsigned)a<1. */
20624 if (op1 != const0_rtx)
20627 code = (code == EQ ? LTU : GEU);
20630 /* Convert a>b into b<a or a>=b-1. */
20633 if (CONST_INT_P (op1))
20635 op1 = gen_int_mode (INTVAL (op1) + 1, GET_MODE (op0));
20636 /* Bail out on overflow. We still can swap operands but that
20637 would force loading of the constant into register. */
20638 if (op1 == const0_rtx
20639 || !x86_64_immediate_operand (op1, GET_MODE (op1)))
20641 code = (code == GTU ? GEU : LTU);
20645 std::swap (op0, op1);
20646 code = (code == GTU ? LTU : GEU);
20650 /* Convert a>=0 into (unsigned)a<0x80000000. */
20653 if (mode == DImode || op1 != const0_rtx)
20655 op1 = gen_int_mode (1 << (GET_MODE_BITSIZE (mode) - 1), mode);
20656 code = (code == LT ? GEU : LTU);
20660 if (mode == DImode || op1 != constm1_rtx)
20662 op1 = gen_int_mode (1 << (GET_MODE_BITSIZE (mode) - 1), mode);
20663 code = (code == LE ? GEU : LTU);
20669 /* Swapping operands may cause constant to appear as first operand. */
20670 if (!nonimmediate_operand (op0, VOIDmode))
20672 if (!can_create_pseudo_p ())
20674 op0 = force_reg (mode, op0);
20676 *pop = ix86_expand_compare (code, op0, op1);
20677 gcc_assert (GET_CODE (*pop) == LTU || GET_CODE (*pop) == GEU);
20682 ix86_expand_int_movcc (rtx operands[])
20684 enum rtx_code code = GET_CODE (operands[1]), compare_code;
20685 rtx_insn *compare_seq;
20687 machine_mode mode = GET_MODE (operands[0]);
20688 bool sign_bit_compare_p = false;
20689 rtx op0 = XEXP (operands[1], 0);
20690 rtx op1 = XEXP (operands[1], 1);
20692 if (GET_MODE (op0) == TImode
20693 || (GET_MODE (op0) == DImode
20698 compare_op = ix86_expand_compare (code, op0, op1);
20699 compare_seq = get_insns ();
20702 compare_code = GET_CODE (compare_op);
20704 if ((op1 == const0_rtx && (code == GE || code == LT))
20705 || (op1 == constm1_rtx && (code == GT || code == LE)))
20706 sign_bit_compare_p = true;
20708 /* Don't attempt mode expansion here -- if we had to expand 5 or 6
20709 HImode insns, we'd be swallowed in word prefix ops. */
20711 if ((mode != HImode || TARGET_FAST_PREFIX)
20712 && (mode != (TARGET_64BIT ? TImode : DImode))
20713 && CONST_INT_P (operands[2])
20714 && CONST_INT_P (operands[3]))
20716 rtx out = operands[0];
20717 HOST_WIDE_INT ct = INTVAL (operands[2]);
20718 HOST_WIDE_INT cf = INTVAL (operands[3]);
20719 HOST_WIDE_INT diff;
20722 /* Sign bit compares are better done using shifts than we do by using
20724 if (sign_bit_compare_p
20725 || ix86_expand_carry_flag_compare (code, op0, op1, &compare_op))
20727 /* Detect overlap between destination and compare sources. */
20730 if (!sign_bit_compare_p)
20733 bool fpcmp = false;
20735 compare_code = GET_CODE (compare_op);
20737 flags = XEXP (compare_op, 0);
20739 if (GET_MODE (flags) == CCFPmode
20740 || GET_MODE (flags) == CCFPUmode)
20744 = ix86_fp_compare_code_to_integer (compare_code);
20747 /* To simplify rest of code, restrict to the GEU case. */
20748 if (compare_code == LTU)
20750 std::swap (ct, cf);
20751 compare_code = reverse_condition (compare_code);
20752 code = reverse_condition (code);
20757 PUT_CODE (compare_op,
20758 reverse_condition_maybe_unordered
20759 (GET_CODE (compare_op)));
20761 PUT_CODE (compare_op,
20762 reverse_condition (GET_CODE (compare_op)));
20766 if (reg_overlap_mentioned_p (out, op0)
20767 || reg_overlap_mentioned_p (out, op1))
20768 tmp = gen_reg_rtx (mode);
20770 if (mode == DImode)
20771 emit_insn (gen_x86_movdicc_0_m1 (tmp, flags, compare_op));
20773 emit_insn (gen_x86_movsicc_0_m1 (gen_lowpart (SImode, tmp),
20774 flags, compare_op));
20778 if (code == GT || code == GE)
20779 code = reverse_condition (code);
20782 std::swap (ct, cf);
20785 tmp = emit_store_flag (tmp, code, op0, op1, VOIDmode, 0, -1);
20798 tmp = expand_simple_binop (mode, PLUS,
20800 copy_rtx (tmp), 1, OPTAB_DIRECT);
20811 tmp = expand_simple_binop (mode, IOR,
20813 copy_rtx (tmp), 1, OPTAB_DIRECT);
20815 else if (diff == -1 && ct)
20825 tmp = expand_simple_unop (mode, NOT, tmp, copy_rtx (tmp), 1);
20827 tmp = expand_simple_binop (mode, PLUS,
20828 copy_rtx (tmp), GEN_INT (cf),
20829 copy_rtx (tmp), 1, OPTAB_DIRECT);
20837 * andl cf - ct, dest
20847 tmp = expand_simple_unop (mode, NOT, tmp, copy_rtx (tmp), 1);
20850 tmp = expand_simple_binop (mode, AND,
20852 gen_int_mode (cf - ct, mode),
20853 copy_rtx (tmp), 1, OPTAB_DIRECT);
20855 tmp = expand_simple_binop (mode, PLUS,
20856 copy_rtx (tmp), GEN_INT (ct),
20857 copy_rtx (tmp), 1, OPTAB_DIRECT);
20860 if (!rtx_equal_p (tmp, out))
20861 emit_move_insn (copy_rtx (out), copy_rtx (tmp));
20868 machine_mode cmp_mode = GET_MODE (op0);
20869 enum rtx_code new_code;
20871 if (SCALAR_FLOAT_MODE_P (cmp_mode))
20873 gcc_assert (!DECIMAL_FLOAT_MODE_P (cmp_mode));
20875 /* We may be reversing unordered compare to normal compare, that
20876 is not valid in general (we may convert non-trapping condition
20877 to trapping one), however on i386 we currently emit all
20878 comparisons unordered. */
20879 new_code = reverse_condition_maybe_unordered (code);
20882 new_code = ix86_reverse_condition (code, cmp_mode);
20883 if (new_code != UNKNOWN)
20885 std::swap (ct, cf);
20891 compare_code = UNKNOWN;
20892 if (GET_MODE_CLASS (GET_MODE (op0)) == MODE_INT
20893 && CONST_INT_P (op1))
20895 if (op1 == const0_rtx
20896 && (code == LT || code == GE))
20897 compare_code = code;
20898 else if (op1 == constm1_rtx)
20902 else if (code == GT)
20907 /* Optimize dest = (op0 < 0) ? -1 : cf. */
20908 if (compare_code != UNKNOWN
20909 && GET_MODE (op0) == GET_MODE (out)
20910 && (cf == -1 || ct == -1))
20912 /* If lea code below could be used, only optimize
20913 if it results in a 2 insn sequence. */
20915 if (! (diff == 1 || diff == 2 || diff == 4 || diff == 8
20916 || diff == 3 || diff == 5 || diff == 9)
20917 || (compare_code == LT && ct == -1)
20918 || (compare_code == GE && cf == -1))
20921 * notl op1 (if necessary)
20929 code = reverse_condition (code);
20932 out = emit_store_flag (out, code, op0, op1, VOIDmode, 0, -1);
20934 out = expand_simple_binop (mode, IOR,
20936 out, 1, OPTAB_DIRECT);
20937 if (out != operands[0])
20938 emit_move_insn (operands[0], out);
20945 if ((diff == 1 || diff == 2 || diff == 4 || diff == 8
20946 || diff == 3 || diff == 5 || diff == 9)
20947 && ((mode != QImode && mode != HImode) || !TARGET_PARTIAL_REG_STALL)
20949 || x86_64_immediate_operand (GEN_INT (cf), VOIDmode)))
20955 * lea cf(dest*(ct-cf)),dest
20959 * This also catches the degenerate setcc-only case.
20965 out = emit_store_flag (out, code, op0, op1, VOIDmode, 0, 1);
20968 /* On x86_64 the lea instruction operates on Pmode, so we need
20969 to get arithmetics done in proper mode to match. */
20971 tmp = copy_rtx (out);
20975 out1 = copy_rtx (out);
20976 tmp = gen_rtx_MULT (mode, out1, GEN_INT (diff & ~1));
20980 tmp = gen_rtx_PLUS (mode, tmp, out1);
20986 tmp = gen_rtx_PLUS (mode, tmp, GEN_INT (cf));
20989 if (!rtx_equal_p (tmp, out))
20992 out = force_operand (tmp, copy_rtx (out));
20994 emit_insn (gen_rtx_SET (copy_rtx (out), copy_rtx (tmp)));
20996 if (!rtx_equal_p (out, operands[0]))
20997 emit_move_insn (operands[0], copy_rtx (out));
21003 * General case: Jumpful:
21004 * xorl dest,dest cmpl op1, op2
21005 * cmpl op1, op2 movl ct, dest
21006 * setcc dest jcc 1f
21007 * decl dest movl cf, dest
21008 * andl (cf-ct),dest 1:
21011 * Size 20. Size 14.
21013 * This is reasonably steep, but branch mispredict costs are
21014 * high on modern cpus, so consider failing only if optimizing
21018 if ((!TARGET_CMOVE || (mode == QImode && TARGET_PARTIAL_REG_STALL))
21019 && BRANCH_COST (optimize_insn_for_speed_p (),
21024 machine_mode cmp_mode = GET_MODE (op0);
21025 enum rtx_code new_code;
21027 if (SCALAR_FLOAT_MODE_P (cmp_mode))
21029 gcc_assert (!DECIMAL_FLOAT_MODE_P (cmp_mode));
21031 /* We may be reversing unordered compare to normal compare,
21032 that is not valid in general (we may convert non-trapping
21033 condition to trapping one), however on i386 we currently
21034 emit all comparisons unordered. */
21035 new_code = reverse_condition_maybe_unordered (code);
21039 new_code = ix86_reverse_condition (code, cmp_mode);
21040 if (compare_code != UNKNOWN && new_code != UNKNOWN)
21041 compare_code = reverse_condition (compare_code);
21044 if (new_code != UNKNOWN)
21052 if (compare_code != UNKNOWN)
21054 /* notl op1 (if needed)
21059 For x < 0 (resp. x <= -1) there will be no notl,
21060 so if possible swap the constants to get rid of the
21062 True/false will be -1/0 while code below (store flag
21063 followed by decrement) is 0/-1, so the constants need
21064 to be exchanged once more. */
21066 if (compare_code == GE || !cf)
21068 code = reverse_condition (code);
21072 std::swap (ct, cf);
21074 out = emit_store_flag (out, code, op0, op1, VOIDmode, 0, -1);
21078 out = emit_store_flag (out, code, op0, op1, VOIDmode, 0, 1);
21080 out = expand_simple_binop (mode, PLUS, copy_rtx (out),
21082 copy_rtx (out), 1, OPTAB_DIRECT);
21085 out = expand_simple_binop (mode, AND, copy_rtx (out),
21086 gen_int_mode (cf - ct, mode),
21087 copy_rtx (out), 1, OPTAB_DIRECT);
21089 out = expand_simple_binop (mode, PLUS, copy_rtx (out), GEN_INT (ct),
21090 copy_rtx (out), 1, OPTAB_DIRECT);
21091 if (!rtx_equal_p (out, operands[0]))
21092 emit_move_insn (operands[0], copy_rtx (out));
21098 if (!TARGET_CMOVE || (mode == QImode && TARGET_PARTIAL_REG_STALL))
21100 /* Try a few things more with specific constants and a variable. */
21103 rtx var, orig_out, out, tmp;
21105 if (BRANCH_COST (optimize_insn_for_speed_p (), false) <= 2)
21108 /* If one of the two operands is an interesting constant, load a
21109 constant with the above and mask it in with a logical operation. */
21111 if (CONST_INT_P (operands[2]))
21114 if (INTVAL (operands[2]) == 0 && operands[3] != constm1_rtx)
21115 operands[3] = constm1_rtx, op = and_optab;
21116 else if (INTVAL (operands[2]) == -1 && operands[3] != const0_rtx)
21117 operands[3] = const0_rtx, op = ior_optab;
21121 else if (CONST_INT_P (operands[3]))
21124 if (INTVAL (operands[3]) == 0 && operands[2] != constm1_rtx)
21125 operands[2] = constm1_rtx, op = and_optab;
21126 else if (INTVAL (operands[3]) == -1 && operands[3] != const0_rtx)
21127 operands[2] = const0_rtx, op = ior_optab;
21134 orig_out = operands[0];
21135 tmp = gen_reg_rtx (mode);
21138 /* Recurse to get the constant loaded. */
21139 if (ix86_expand_int_movcc (operands) == 0)
21142 /* Mask in the interesting variable. */
21143 out = expand_binop (mode, op, var, tmp, orig_out, 0,
21145 if (!rtx_equal_p (out, orig_out))
21146 emit_move_insn (copy_rtx (orig_out), copy_rtx (out));
21152 * For comparison with above,
21162 if (! nonimmediate_operand (operands[2], mode))
21163 operands[2] = force_reg (mode, operands[2]);
21164 if (! nonimmediate_operand (operands[3], mode))
21165 operands[3] = force_reg (mode, operands[3]);
21167 if (! register_operand (operands[2], VOIDmode)
21169 || ! register_operand (operands[3], VOIDmode)))
21170 operands[2] = force_reg (mode, operands[2]);
21173 && ! register_operand (operands[3], VOIDmode))
21174 operands[3] = force_reg (mode, operands[3]);
21176 emit_insn (compare_seq);
21177 emit_insn (gen_rtx_SET (operands[0],
21178 gen_rtx_IF_THEN_ELSE (mode,
21179 compare_op, operands[2],
21184 /* Swap, force into registers, or otherwise massage the two operands
21185 to an sse comparison with a mask result. Thus we differ a bit from
21186 ix86_prepare_fp_compare_args which expects to produce a flags result.
21188 The DEST operand exists to help determine whether to commute commutative
21189 operators. The POP0/POP1 operands are updated in place. The new
21190 comparison code is returned, or UNKNOWN if not implementable. */
21192 static enum rtx_code
21193 ix86_prepare_sse_fp_compare_args (rtx dest, enum rtx_code code,
21194 rtx *pop0, rtx *pop1)
21200 /* AVX supports all the needed comparisons. */
21203 /* We have no LTGT as an operator. We could implement it with
21204 NE & ORDERED, but this requires an extra temporary. It's
21205 not clear that it's worth it. */
21212 /* These are supported directly. */
21219 /* AVX has 3 operand comparisons, no need to swap anything. */
21222 /* For commutative operators, try to canonicalize the destination
21223 operand to be first in the comparison - this helps reload to
21224 avoid extra moves. */
21225 if (!dest || !rtx_equal_p (dest, *pop1))
21233 /* These are not supported directly before AVX, and furthermore
21234 ix86_expand_sse_fp_minmax only optimizes LT/UNGE. Swap the
21235 comparison operands to transform into something that is
21237 std::swap (*pop0, *pop1);
21238 code = swap_condition (code);
21242 gcc_unreachable ();
21248 /* Detect conditional moves that exactly match min/max operational
21249 semantics. Note that this is IEEE safe, as long as we don't
21250 interchange the operands.
21252 Returns FALSE if this conditional move doesn't match a MIN/MAX,
21253 and TRUE if the operation is successful and instructions are emitted. */
21256 ix86_expand_sse_fp_minmax (rtx dest, enum rtx_code code, rtx cmp_op0,
21257 rtx cmp_op1, rtx if_true, rtx if_false)
21265 else if (code == UNGE)
21266 std::swap (if_true, if_false);
21270 if (rtx_equal_p (cmp_op0, if_true) && rtx_equal_p (cmp_op1, if_false))
21272 else if (rtx_equal_p (cmp_op1, if_true) && rtx_equal_p (cmp_op0, if_false))
21277 mode = GET_MODE (dest);
21279 /* We want to check HONOR_NANS and HONOR_SIGNED_ZEROS here,
21280 but MODE may be a vector mode and thus not appropriate. */
21281 if (!flag_finite_math_only || !flag_unsafe_math_optimizations)
21283 int u = is_min ? UNSPEC_IEEE_MIN : UNSPEC_IEEE_MAX;
21286 if_true = force_reg (mode, if_true);
21287 v = gen_rtvec (2, if_true, if_false);
21288 tmp = gen_rtx_UNSPEC (mode, v, u);
21292 code = is_min ? SMIN : SMAX;
21293 tmp = gen_rtx_fmt_ee (code, mode, if_true, if_false);
21296 emit_insn (gen_rtx_SET (dest, tmp));
21300 /* Expand an sse vector comparison. Return the register with the result. */
21303 ix86_expand_sse_cmp (rtx dest, enum rtx_code code, rtx cmp_op0, rtx cmp_op1,
21304 rtx op_true, rtx op_false)
21306 machine_mode mode = GET_MODE (dest);
21307 machine_mode cmp_ops_mode = GET_MODE (cmp_op0);
21309 /* In general case result of comparison can differ from operands' type. */
21310 machine_mode cmp_mode;
21312 /* In AVX512F the result of comparison is an integer mask. */
21313 bool maskcmp = false;
21316 if (GET_MODE_SIZE (cmp_ops_mode) == 64)
21318 cmp_mode = mode_for_size (GET_MODE_NUNITS (cmp_ops_mode), MODE_INT, 0);
21319 gcc_assert (cmp_mode != BLKmode);
21324 cmp_mode = cmp_ops_mode;
21327 cmp_op0 = force_reg (cmp_ops_mode, cmp_op0);
21328 if (!nonimmediate_operand (cmp_op1, cmp_ops_mode))
21329 cmp_op1 = force_reg (cmp_ops_mode, cmp_op1);
21332 || reg_overlap_mentioned_p (dest, op_true)
21333 || reg_overlap_mentioned_p (dest, op_false))
21334 dest = gen_reg_rtx (maskcmp ? cmp_mode : mode);
21336 /* Compare patterns for int modes are unspec in AVX512F only. */
21337 if (maskcmp && (code == GT || code == EQ))
21339 rtx (*gen)(rtx, rtx, rtx);
21341 switch (cmp_ops_mode)
21344 gcc_assert (TARGET_AVX512BW);
21345 gen = code == GT ? gen_avx512bw_gtv64qi3 : gen_avx512bw_eqv64qi3_1;
21348 gcc_assert (TARGET_AVX512BW);
21349 gen = code == GT ? gen_avx512bw_gtv32hi3 : gen_avx512bw_eqv32hi3_1;
21352 gen = code == GT ? gen_avx512f_gtv16si3 : gen_avx512f_eqv16si3_1;
21355 gen = code == GT ? gen_avx512f_gtv8di3 : gen_avx512f_eqv8di3_1;
21363 emit_insn (gen (dest, cmp_op0, cmp_op1));
21367 x = gen_rtx_fmt_ee (code, cmp_mode, cmp_op0, cmp_op1);
21369 if (cmp_mode != mode && !maskcmp)
21371 x = force_reg (cmp_ops_mode, x);
21372 convert_move (dest, x, false);
21375 emit_insn (gen_rtx_SET (dest, x));
21380 /* Expand DEST = CMP ? OP_TRUE : OP_FALSE into a sequence of logical
21381 operations. This is used for both scalar and vector conditional moves. */
21384 ix86_expand_sse_movcc (rtx dest, rtx cmp, rtx op_true, rtx op_false)
21386 machine_mode mode = GET_MODE (dest);
21387 machine_mode cmpmode = GET_MODE (cmp);
21389 /* In AVX512F the result of comparison is an integer mask. */
21390 bool maskcmp = (mode != cmpmode && TARGET_AVX512F);
21394 if (vector_all_ones_operand (op_true, mode)
21395 && rtx_equal_p (op_false, CONST0_RTX (mode))
21398 emit_insn (gen_rtx_SET (dest, cmp));
21400 else if (op_false == CONST0_RTX (mode)
21403 op_true = force_reg (mode, op_true);
21404 x = gen_rtx_AND (mode, cmp, op_true);
21405 emit_insn (gen_rtx_SET (dest, x));
21407 else if (op_true == CONST0_RTX (mode)
21410 op_false = force_reg (mode, op_false);
21411 x = gen_rtx_NOT (mode, cmp);
21412 x = gen_rtx_AND (mode, x, op_false);
21413 emit_insn (gen_rtx_SET (dest, x));
21415 else if (INTEGRAL_MODE_P (mode) && op_true == CONSTM1_RTX (mode)
21418 op_false = force_reg (mode, op_false);
21419 x = gen_rtx_IOR (mode, cmp, op_false);
21420 emit_insn (gen_rtx_SET (dest, x));
21422 else if (TARGET_XOP
21425 op_true = force_reg (mode, op_true);
21427 if (!nonimmediate_operand (op_false, mode))
21428 op_false = force_reg (mode, op_false);
21430 emit_insn (gen_rtx_SET (dest, gen_rtx_IF_THEN_ELSE (mode, cmp,
21436 rtx (*gen) (rtx, rtx, rtx, rtx) = NULL;
21439 if (!nonimmediate_operand (op_true, mode))
21440 op_true = force_reg (mode, op_true);
21442 op_false = force_reg (mode, op_false);
21448 gen = gen_sse4_1_blendvps;
21452 gen = gen_sse4_1_blendvpd;
21460 gen = gen_sse4_1_pblendvb;
21461 if (mode != V16QImode)
21462 d = gen_reg_rtx (V16QImode);
21463 op_false = gen_lowpart (V16QImode, op_false);
21464 op_true = gen_lowpart (V16QImode, op_true);
21465 cmp = gen_lowpart (V16QImode, cmp);
21470 gen = gen_avx_blendvps256;
21474 gen = gen_avx_blendvpd256;
21482 gen = gen_avx2_pblendvb;
21483 if (mode != V32QImode)
21484 d = gen_reg_rtx (V32QImode);
21485 op_false = gen_lowpart (V32QImode, op_false);
21486 op_true = gen_lowpart (V32QImode, op_true);
21487 cmp = gen_lowpart (V32QImode, cmp);
21492 gen = gen_avx512bw_blendmv64qi;
21495 gen = gen_avx512bw_blendmv32hi;
21498 gen = gen_avx512f_blendmv16si;
21501 gen = gen_avx512f_blendmv8di;
21504 gen = gen_avx512f_blendmv8df;
21507 gen = gen_avx512f_blendmv16sf;
21516 emit_insn (gen (d, op_false, op_true, cmp));
21518 emit_move_insn (dest, gen_lowpart (GET_MODE (dest), d));
21522 op_true = force_reg (mode, op_true);
21524 t2 = gen_reg_rtx (mode);
21526 t3 = gen_reg_rtx (mode);
21530 x = gen_rtx_AND (mode, op_true, cmp);
21531 emit_insn (gen_rtx_SET (t2, x));
21533 x = gen_rtx_NOT (mode, cmp);
21534 x = gen_rtx_AND (mode, x, op_false);
21535 emit_insn (gen_rtx_SET (t3, x));
21537 x = gen_rtx_IOR (mode, t3, t2);
21538 emit_insn (gen_rtx_SET (dest, x));
21543 /* Expand a floating-point conditional move. Return true if successful. */
21546 ix86_expand_fp_movcc (rtx operands[])
21548 machine_mode mode = GET_MODE (operands[0]);
21549 enum rtx_code code = GET_CODE (operands[1]);
21550 rtx tmp, compare_op;
21551 rtx op0 = XEXP (operands[1], 0);
21552 rtx op1 = XEXP (operands[1], 1);
21554 if (TARGET_SSE_MATH && SSE_FLOAT_MODE_P (mode))
21556 machine_mode cmode;
21558 /* Since we've no cmove for sse registers, don't force bad register
21559 allocation just to gain access to it. Deny movcc when the
21560 comparison mode doesn't match the move mode. */
21561 cmode = GET_MODE (op0);
21562 if (cmode == VOIDmode)
21563 cmode = GET_MODE (op1);
21567 code = ix86_prepare_sse_fp_compare_args (operands[0], code, &op0, &op1);
21568 if (code == UNKNOWN)
21571 if (ix86_expand_sse_fp_minmax (operands[0], code, op0, op1,
21572 operands[2], operands[3]))
21575 tmp = ix86_expand_sse_cmp (operands[0], code, op0, op1,
21576 operands[2], operands[3]);
21577 ix86_expand_sse_movcc (operands[0], tmp, operands[2], operands[3]);
21581 if (GET_MODE (op0) == TImode
21582 || (GET_MODE (op0) == DImode
21586 /* The floating point conditional move instructions don't directly
21587 support conditions resulting from a signed integer comparison. */
21589 compare_op = ix86_expand_compare (code, op0, op1);
21590 if (!fcmov_comparison_operator (compare_op, VOIDmode))
21592 tmp = gen_reg_rtx (QImode);
21593 ix86_expand_setcc (tmp, code, op0, op1);
21595 compare_op = ix86_expand_compare (NE, tmp, const0_rtx);
21598 emit_insn (gen_rtx_SET (operands[0],
21599 gen_rtx_IF_THEN_ELSE (mode, compare_op,
21600 operands[2], operands[3])));
21605 /* Expand a floating-point vector conditional move; a vcond operation
21606 rather than a movcc operation. */
21609 ix86_expand_fp_vcond (rtx operands[])
21611 enum rtx_code code = GET_CODE (operands[3]);
21614 code = ix86_prepare_sse_fp_compare_args (operands[0], code,
21615 &operands[4], &operands[5]);
21616 if (code == UNKNOWN)
21619 switch (GET_CODE (operands[3]))
21622 temp = ix86_expand_sse_cmp (operands[0], ORDERED, operands[4],
21623 operands[5], operands[0], operands[0]);
21624 cmp = ix86_expand_sse_cmp (operands[0], NE, operands[4],
21625 operands[5], operands[1], operands[2]);
21629 temp = ix86_expand_sse_cmp (operands[0], UNORDERED, operands[4],
21630 operands[5], operands[0], operands[0]);
21631 cmp = ix86_expand_sse_cmp (operands[0], EQ, operands[4],
21632 operands[5], operands[1], operands[2]);
21636 gcc_unreachable ();
21638 cmp = expand_simple_binop (GET_MODE (cmp), code, temp, cmp, cmp, 1,
21640 ix86_expand_sse_movcc (operands[0], cmp, operands[1], operands[2]);
21644 if (ix86_expand_sse_fp_minmax (operands[0], code, operands[4],
21645 operands[5], operands[1], operands[2]))
21648 cmp = ix86_expand_sse_cmp (operands[0], code, operands[4], operands[5],
21649 operands[1], operands[2]);
21650 ix86_expand_sse_movcc (operands[0], cmp, operands[1], operands[2]);
21654 /* Expand a signed/unsigned integral vector conditional move. */
21657 ix86_expand_int_vcond (rtx operands[])
21659 machine_mode data_mode = GET_MODE (operands[0]);
21660 machine_mode mode = GET_MODE (operands[4]);
21661 enum rtx_code code = GET_CODE (operands[3]);
21662 bool negate = false;
21665 cop0 = operands[4];
21666 cop1 = operands[5];
21668 /* Try to optimize x < 0 ? -1 : 0 into (signed) x >> 31
21669 and x < 0 ? 1 : 0 into (unsigned) x >> 31. */
21670 if ((code == LT || code == GE)
21671 && data_mode == mode
21672 && cop1 == CONST0_RTX (mode)
21673 && operands[1 + (code == LT)] == CONST0_RTX (data_mode)
21674 && GET_MODE_SIZE (GET_MODE_INNER (data_mode)) > 1
21675 && GET_MODE_SIZE (GET_MODE_INNER (data_mode)) <= 8
21676 && (GET_MODE_SIZE (data_mode) == 16
21677 || (TARGET_AVX2 && GET_MODE_SIZE (data_mode) == 32)))
21679 rtx negop = operands[2 - (code == LT)];
21680 int shift = GET_MODE_BITSIZE (GET_MODE_INNER (data_mode)) - 1;
21681 if (negop == CONST1_RTX (data_mode))
21683 rtx res = expand_simple_binop (mode, LSHIFTRT, cop0, GEN_INT (shift),
21684 operands[0], 1, OPTAB_DIRECT);
21685 if (res != operands[0])
21686 emit_move_insn (operands[0], res);
21689 else if (GET_MODE_INNER (data_mode) != DImode
21690 && vector_all_ones_operand (negop, data_mode))
21692 rtx res = expand_simple_binop (mode, ASHIFTRT, cop0, GEN_INT (shift),
21693 operands[0], 0, OPTAB_DIRECT);
21694 if (res != operands[0])
21695 emit_move_insn (operands[0], res);
21700 if (!nonimmediate_operand (cop1, mode))
21701 cop1 = force_reg (mode, cop1);
21702 if (!general_operand (operands[1], data_mode))
21703 operands[1] = force_reg (data_mode, operands[1]);
21704 if (!general_operand (operands[2], data_mode))
21705 operands[2] = force_reg (data_mode, operands[2]);
21707 /* XOP supports all of the comparisons on all 128-bit vector int types. */
21709 && (mode == V16QImode || mode == V8HImode
21710 || mode == V4SImode || mode == V2DImode))
21714 /* Canonicalize the comparison to EQ, GT, GTU. */
21725 code = reverse_condition (code);
21731 code = reverse_condition (code);
21737 std::swap (cop0, cop1);
21738 code = swap_condition (code);
21742 gcc_unreachable ();
21745 /* Only SSE4.1/SSE4.2 supports V2DImode. */
21746 if (mode == V2DImode)
21751 /* SSE4.1 supports EQ. */
21752 if (!TARGET_SSE4_1)
21758 /* SSE4.2 supports GT/GTU. */
21759 if (!TARGET_SSE4_2)
21764 gcc_unreachable ();
21768 /* Unsigned parallel compare is not supported by the hardware.
21769 Play some tricks to turn this into a signed comparison
21773 cop0 = force_reg (mode, cop0);
21785 rtx (*gen_sub3) (rtx, rtx, rtx);
21789 case V16SImode: gen_sub3 = gen_subv16si3; break;
21790 case V8DImode: gen_sub3 = gen_subv8di3; break;
21791 case V8SImode: gen_sub3 = gen_subv8si3; break;
21792 case V4DImode: gen_sub3 = gen_subv4di3; break;
21793 case V4SImode: gen_sub3 = gen_subv4si3; break;
21794 case V2DImode: gen_sub3 = gen_subv2di3; break;
21796 gcc_unreachable ();
21798 /* Subtract (-(INT MAX) - 1) from both operands to make
21800 mask = ix86_build_signbit_mask (mode, true, false);
21801 t1 = gen_reg_rtx (mode);
21802 emit_insn (gen_sub3 (t1, cop0, mask));
21804 t2 = gen_reg_rtx (mode);
21805 emit_insn (gen_sub3 (t2, cop1, mask));
21819 /* Perform a parallel unsigned saturating subtraction. */
21820 x = gen_reg_rtx (mode);
21821 emit_insn (gen_rtx_SET (x, gen_rtx_US_MINUS (mode, cop0, cop1)));
21824 cop1 = CONST0_RTX (mode);
21830 gcc_unreachable ();
21835 /* Allow the comparison to be done in one mode, but the movcc to
21836 happen in another mode. */
21837 if (data_mode == mode)
21839 x = ix86_expand_sse_cmp (operands[0], code, cop0, cop1,
21840 operands[1+negate], operands[2-negate]);
21844 gcc_assert (GET_MODE_SIZE (data_mode) == GET_MODE_SIZE (mode));
21845 x = ix86_expand_sse_cmp (gen_reg_rtx (mode), code, cop0, cop1,
21846 operands[1+negate], operands[2-negate]);
21847 if (GET_MODE (x) == mode)
21848 x = gen_lowpart (data_mode, x);
21851 ix86_expand_sse_movcc (operands[0], x, operands[1+negate],
21852 operands[2-negate]);
21856 /* AVX512F does support 64-byte integer vector operations,
21857 thus the longest vector we are faced with is V64QImode. */
21858 #define MAX_VECT_LEN 64
21860 struct expand_vec_perm_d
21862 rtx target, op0, op1;
21863 unsigned char perm[MAX_VECT_LEN];
21864 machine_mode vmode;
21865 unsigned char nelt;
21866 bool one_operand_p;
21871 ix86_expand_vec_perm_vpermi2 (rtx target, rtx op0, rtx mask, rtx op1,
21872 struct expand_vec_perm_d *d)
21874 /* ix86_expand_vec_perm_vpermi2 is called from both const and non-const
21875 expander, so args are either in d, or in op0, op1 etc. */
21876 machine_mode mode = GET_MODE (d ? d->op0 : op0);
21877 machine_mode maskmode = mode;
21878 rtx (*gen) (rtx, rtx, rtx, rtx) = NULL;
21883 if (TARGET_AVX512VL && TARGET_AVX512BW)
21884 gen = gen_avx512vl_vpermi2varv8hi3;
21887 if (TARGET_AVX512VL && TARGET_AVX512BW)
21888 gen = gen_avx512vl_vpermi2varv16hi3;
21891 if (TARGET_AVX512VBMI)
21892 gen = gen_avx512bw_vpermi2varv64qi3;
21895 if (TARGET_AVX512BW)
21896 gen = gen_avx512bw_vpermi2varv32hi3;
21899 if (TARGET_AVX512VL)
21900 gen = gen_avx512vl_vpermi2varv4si3;
21903 if (TARGET_AVX512VL)
21904 gen = gen_avx512vl_vpermi2varv8si3;
21907 if (TARGET_AVX512F)
21908 gen = gen_avx512f_vpermi2varv16si3;
21911 if (TARGET_AVX512VL)
21913 gen = gen_avx512vl_vpermi2varv4sf3;
21914 maskmode = V4SImode;
21918 if (TARGET_AVX512VL)
21920 gen = gen_avx512vl_vpermi2varv8sf3;
21921 maskmode = V8SImode;
21925 if (TARGET_AVX512F)
21927 gen = gen_avx512f_vpermi2varv16sf3;
21928 maskmode = V16SImode;
21932 if (TARGET_AVX512VL)
21933 gen = gen_avx512vl_vpermi2varv2di3;
21936 if (TARGET_AVX512VL)
21937 gen = gen_avx512vl_vpermi2varv4di3;
21940 if (TARGET_AVX512F)
21941 gen = gen_avx512f_vpermi2varv8di3;
21944 if (TARGET_AVX512VL)
21946 gen = gen_avx512vl_vpermi2varv2df3;
21947 maskmode = V2DImode;
21951 if (TARGET_AVX512VL)
21953 gen = gen_avx512vl_vpermi2varv4df3;
21954 maskmode = V4DImode;
21958 if (TARGET_AVX512F)
21960 gen = gen_avx512f_vpermi2varv8df3;
21961 maskmode = V8DImode;
21971 /* ix86_expand_vec_perm_vpermi2 is called from both const and non-const
21972 expander, so args are either in d, or in op0, op1 etc. */
21976 target = d->target;
21979 for (int i = 0; i < d->nelt; ++i)
21980 vec[i] = GEN_INT (d->perm[i]);
21981 mask = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (d->nelt, vec));
21984 emit_insn (gen (target, op0, force_reg (maskmode, mask), op1));
21988 /* Expand a variable vector permutation. */
21991 ix86_expand_vec_perm (rtx operands[])
21993 rtx target = operands[0];
21994 rtx op0 = operands[1];
21995 rtx op1 = operands[2];
21996 rtx mask = operands[3];
21997 rtx t1, t2, t3, t4, t5, t6, t7, t8, vt, vt2, vec[32];
21998 machine_mode mode = GET_MODE (op0);
21999 machine_mode maskmode = GET_MODE (mask);
22001 bool one_operand_shuffle = rtx_equal_p (op0, op1);
22003 /* Number of elements in the vector. */
22004 w = GET_MODE_NUNITS (mode);
22005 e = GET_MODE_UNIT_SIZE (mode);
22006 gcc_assert (w <= 64);
22008 if (ix86_expand_vec_perm_vpermi2 (target, op0, mask, op1, NULL))
22013 if (mode == V4DImode || mode == V4DFmode || mode == V16HImode)
22015 /* Unfortunately, the VPERMQ and VPERMPD instructions only support
22016 an constant shuffle operand. With a tiny bit of effort we can
22017 use VPERMD instead. A re-interpretation stall for V4DFmode is
22018 unfortunate but there's no avoiding it.
22019 Similarly for V16HImode we don't have instructions for variable
22020 shuffling, while for V32QImode we can use after preparing suitable
22021 masks vpshufb; vpshufb; vpermq; vpor. */
22023 if (mode == V16HImode)
22025 maskmode = mode = V32QImode;
22031 maskmode = mode = V8SImode;
22035 t1 = gen_reg_rtx (maskmode);
22037 /* Replicate the low bits of the V4DImode mask into V8SImode:
22039 t1 = { A A B B C C D D }. */
22040 for (i = 0; i < w / 2; ++i)
22041 vec[i*2 + 1] = vec[i*2] = GEN_INT (i * 2);
22042 vt = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (w, vec));
22043 vt = force_reg (maskmode, vt);
22044 mask = gen_lowpart (maskmode, mask);
22045 if (maskmode == V8SImode)
22046 emit_insn (gen_avx2_permvarv8si (t1, mask, vt));
22048 emit_insn (gen_avx2_pshufbv32qi3 (t1, mask, vt));
22050 /* Multiply the shuffle indicies by two. */
22051 t1 = expand_simple_binop (maskmode, PLUS, t1, t1, t1, 1,
22054 /* Add one to the odd shuffle indicies:
22055 t1 = { A*2, A*2+1, B*2, B*2+1, ... }. */
22056 for (i = 0; i < w / 2; ++i)
22058 vec[i * 2] = const0_rtx;
22059 vec[i * 2 + 1] = const1_rtx;
22061 vt = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (w, vec));
22062 vt = validize_mem (force_const_mem (maskmode, vt));
22063 t1 = expand_simple_binop (maskmode, PLUS, t1, vt, t1, 1,
22066 /* Continue as if V8SImode (resp. V32QImode) was used initially. */
22067 operands[3] = mask = t1;
22068 target = gen_reg_rtx (mode);
22069 op0 = gen_lowpart (mode, op0);
22070 op1 = gen_lowpart (mode, op1);
22076 /* The VPERMD and VPERMPS instructions already properly ignore
22077 the high bits of the shuffle elements. No need for us to
22078 perform an AND ourselves. */
22079 if (one_operand_shuffle)
22081 emit_insn (gen_avx2_permvarv8si (target, op0, mask));
22082 if (target != operands[0])
22083 emit_move_insn (operands[0],
22084 gen_lowpart (GET_MODE (operands[0]), target));
22088 t1 = gen_reg_rtx (V8SImode);
22089 t2 = gen_reg_rtx (V8SImode);
22090 emit_insn (gen_avx2_permvarv8si (t1, op0, mask));
22091 emit_insn (gen_avx2_permvarv8si (t2, op1, mask));
22097 mask = gen_lowpart (V8SImode, mask);
22098 if (one_operand_shuffle)
22099 emit_insn (gen_avx2_permvarv8sf (target, op0, mask));
22102 t1 = gen_reg_rtx (V8SFmode);
22103 t2 = gen_reg_rtx (V8SFmode);
22104 emit_insn (gen_avx2_permvarv8sf (t1, op0, mask));
22105 emit_insn (gen_avx2_permvarv8sf (t2, op1, mask));
22111 /* By combining the two 128-bit input vectors into one 256-bit
22112 input vector, we can use VPERMD and VPERMPS for the full
22113 two-operand shuffle. */
22114 t1 = gen_reg_rtx (V8SImode);
22115 t2 = gen_reg_rtx (V8SImode);
22116 emit_insn (gen_avx_vec_concatv8si (t1, op0, op1));
22117 emit_insn (gen_avx_vec_concatv8si (t2, mask, mask));
22118 emit_insn (gen_avx2_permvarv8si (t1, t1, t2));
22119 emit_insn (gen_avx_vextractf128v8si (target, t1, const0_rtx));
22123 t1 = gen_reg_rtx (V8SFmode);
22124 t2 = gen_reg_rtx (V8SImode);
22125 mask = gen_lowpart (V4SImode, mask);
22126 emit_insn (gen_avx_vec_concatv8sf (t1, op0, op1));
22127 emit_insn (gen_avx_vec_concatv8si (t2, mask, mask));
22128 emit_insn (gen_avx2_permvarv8sf (t1, t1, t2));
22129 emit_insn (gen_avx_vextractf128v8sf (target, t1, const0_rtx));
22133 t1 = gen_reg_rtx (V32QImode);
22134 t2 = gen_reg_rtx (V32QImode);
22135 t3 = gen_reg_rtx (V32QImode);
22136 vt2 = GEN_INT (-128);
22137 for (i = 0; i < 32; i++)
22139 vt = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, vec));
22140 vt = force_reg (V32QImode, vt);
22141 for (i = 0; i < 32; i++)
22142 vec[i] = i < 16 ? vt2 : const0_rtx;
22143 vt2 = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, vec));
22144 vt2 = force_reg (V32QImode, vt2);
22145 /* From mask create two adjusted masks, which contain the same
22146 bits as mask in the low 7 bits of each vector element.
22147 The first mask will have the most significant bit clear
22148 if it requests element from the same 128-bit lane
22149 and MSB set if it requests element from the other 128-bit lane.
22150 The second mask will have the opposite values of the MSB,
22151 and additionally will have its 128-bit lanes swapped.
22152 E.g. { 07 12 1e 09 ... | 17 19 05 1f ... } mask vector will have
22153 t1 { 07 92 9e 09 ... | 17 19 85 1f ... } and
22154 t3 { 97 99 05 9f ... | 87 12 1e 89 ... } where each ...
22155 stands for other 12 bytes. */
22156 /* The bit whether element is from the same lane or the other
22157 lane is bit 4, so shift it up by 3 to the MSB position. */
22158 t5 = gen_reg_rtx (V4DImode);
22159 emit_insn (gen_ashlv4di3 (t5, gen_lowpart (V4DImode, mask),
22161 /* Clear MSB bits from the mask just in case it had them set. */
22162 emit_insn (gen_avx2_andnotv32qi3 (t2, vt, mask));
22163 /* After this t1 will have MSB set for elements from other lane. */
22164 emit_insn (gen_xorv32qi3 (t1, gen_lowpart (V32QImode, t5), vt2));
22165 /* Clear bits other than MSB. */
22166 emit_insn (gen_andv32qi3 (t1, t1, vt));
22167 /* Or in the lower bits from mask into t3. */
22168 emit_insn (gen_iorv32qi3 (t3, t1, t2));
22169 /* And invert MSB bits in t1, so MSB is set for elements from the same
22171 emit_insn (gen_xorv32qi3 (t1, t1, vt));
22172 /* Swap 128-bit lanes in t3. */
22173 t6 = gen_reg_rtx (V4DImode);
22174 emit_insn (gen_avx2_permv4di_1 (t6, gen_lowpart (V4DImode, t3),
22175 const2_rtx, GEN_INT (3),
22176 const0_rtx, const1_rtx));
22177 /* And or in the lower bits from mask into t1. */
22178 emit_insn (gen_iorv32qi3 (t1, t1, t2));
22179 if (one_operand_shuffle)
22181 /* Each of these shuffles will put 0s in places where
22182 element from the other 128-bit lane is needed, otherwise
22183 will shuffle in the requested value. */
22184 emit_insn (gen_avx2_pshufbv32qi3 (t3, op0,
22185 gen_lowpart (V32QImode, t6)));
22186 emit_insn (gen_avx2_pshufbv32qi3 (t1, op0, t1));
22187 /* For t3 the 128-bit lanes are swapped again. */
22188 t7 = gen_reg_rtx (V4DImode);
22189 emit_insn (gen_avx2_permv4di_1 (t7, gen_lowpart (V4DImode, t3),
22190 const2_rtx, GEN_INT (3),
22191 const0_rtx, const1_rtx));
22192 /* And oring both together leads to the result. */
22193 emit_insn (gen_iorv32qi3 (target, t1,
22194 gen_lowpart (V32QImode, t7)));
22195 if (target != operands[0])
22196 emit_move_insn (operands[0],
22197 gen_lowpart (GET_MODE (operands[0]), target));
22201 t4 = gen_reg_rtx (V32QImode);
22202 /* Similarly to the above one_operand_shuffle code,
22203 just for repeated twice for each operand. merge_two:
22204 code will merge the two results together. */
22205 emit_insn (gen_avx2_pshufbv32qi3 (t4, op0,
22206 gen_lowpart (V32QImode, t6)));
22207 emit_insn (gen_avx2_pshufbv32qi3 (t3, op1,
22208 gen_lowpart (V32QImode, t6)));
22209 emit_insn (gen_avx2_pshufbv32qi3 (t2, op0, t1));
22210 emit_insn (gen_avx2_pshufbv32qi3 (t1, op1, t1));
22211 t7 = gen_reg_rtx (V4DImode);
22212 emit_insn (gen_avx2_permv4di_1 (t7, gen_lowpart (V4DImode, t4),
22213 const2_rtx, GEN_INT (3),
22214 const0_rtx, const1_rtx));
22215 t8 = gen_reg_rtx (V4DImode);
22216 emit_insn (gen_avx2_permv4di_1 (t8, gen_lowpart (V4DImode, t3),
22217 const2_rtx, GEN_INT (3),
22218 const0_rtx, const1_rtx));
22219 emit_insn (gen_iorv32qi3 (t4, t2, gen_lowpart (V32QImode, t7)));
22220 emit_insn (gen_iorv32qi3 (t3, t1, gen_lowpart (V32QImode, t8)));
22226 gcc_assert (GET_MODE_SIZE (mode) <= 16);
22233 /* The XOP VPPERM insn supports three inputs. By ignoring the
22234 one_operand_shuffle special case, we avoid creating another
22235 set of constant vectors in memory. */
22236 one_operand_shuffle = false;
22238 /* mask = mask & {2*w-1, ...} */
22239 vt = GEN_INT (2*w - 1);
22243 /* mask = mask & {w-1, ...} */
22244 vt = GEN_INT (w - 1);
22247 for (i = 0; i < w; i++)
22249 vt = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (w, vec));
22250 mask = expand_simple_binop (maskmode, AND, mask, vt,
22251 NULL_RTX, 0, OPTAB_DIRECT);
22253 /* For non-QImode operations, convert the word permutation control
22254 into a byte permutation control. */
22255 if (mode != V16QImode)
22257 mask = expand_simple_binop (maskmode, ASHIFT, mask,
22258 GEN_INT (exact_log2 (e)),
22259 NULL_RTX, 0, OPTAB_DIRECT);
22261 /* Convert mask to vector of chars. */
22262 mask = force_reg (V16QImode, gen_lowpart (V16QImode, mask));
22264 /* Replicate each of the input bytes into byte positions:
22265 (v2di) --> {0,0,0,0,0,0,0,0, 8,8,8,8,8,8,8,8}
22266 (v4si) --> {0,0,0,0, 4,4,4,4, 8,8,8,8, 12,12,12,12}
22267 (v8hi) --> {0,0, 2,2, 4,4, 6,6, ...}. */
22268 for (i = 0; i < 16; ++i)
22269 vec[i] = GEN_INT (i/e * e);
22270 vt = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, vec));
22271 vt = validize_mem (force_const_mem (V16QImode, vt));
22273 emit_insn (gen_xop_pperm (mask, mask, mask, vt));
22275 emit_insn (gen_ssse3_pshufbv16qi3 (mask, mask, vt));
22277 /* Convert it into the byte positions by doing
22278 mask = mask + {0,1,..,16/w, 0,1,..,16/w, ...} */
22279 for (i = 0; i < 16; ++i)
22280 vec[i] = GEN_INT (i % e);
22281 vt = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, vec));
22282 vt = validize_mem (force_const_mem (V16QImode, vt));
22283 emit_insn (gen_addv16qi3 (mask, mask, vt));
22286 /* The actual shuffle operations all operate on V16QImode. */
22287 op0 = gen_lowpart (V16QImode, op0);
22288 op1 = gen_lowpart (V16QImode, op1);
22292 if (GET_MODE (target) != V16QImode)
22293 target = gen_reg_rtx (V16QImode);
22294 emit_insn (gen_xop_pperm (target, op0, op1, mask));
22295 if (target != operands[0])
22296 emit_move_insn (operands[0],
22297 gen_lowpart (GET_MODE (operands[0]), target));
22299 else if (one_operand_shuffle)
22301 if (GET_MODE (target) != V16QImode)
22302 target = gen_reg_rtx (V16QImode);
22303 emit_insn (gen_ssse3_pshufbv16qi3 (target, op0, mask));
22304 if (target != operands[0])
22305 emit_move_insn (operands[0],
22306 gen_lowpart (GET_MODE (operands[0]), target));
22313 /* Shuffle the two input vectors independently. */
22314 t1 = gen_reg_rtx (V16QImode);
22315 t2 = gen_reg_rtx (V16QImode);
22316 emit_insn (gen_ssse3_pshufbv16qi3 (t1, op0, mask));
22317 emit_insn (gen_ssse3_pshufbv16qi3 (t2, op1, mask));
22320 /* Then merge them together. The key is whether any given control
22321 element contained a bit set that indicates the second word. */
22322 mask = operands[3];
22324 if (maskmode == V2DImode && !TARGET_SSE4_1)
22326 /* Without SSE4.1, we don't have V2DImode EQ. Perform one
22327 more shuffle to convert the V2DI input mask into a V4SI
22328 input mask. At which point the masking that expand_int_vcond
22329 will work as desired. */
22330 rtx t3 = gen_reg_rtx (V4SImode);
22331 emit_insn (gen_sse2_pshufd_1 (t3, gen_lowpart (V4SImode, mask),
22332 const0_rtx, const0_rtx,
22333 const2_rtx, const2_rtx));
22335 maskmode = V4SImode;
22339 for (i = 0; i < w; i++)
22341 vt = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (w, vec));
22342 vt = force_reg (maskmode, vt);
22343 mask = expand_simple_binop (maskmode, AND, mask, vt,
22344 NULL_RTX, 0, OPTAB_DIRECT);
22346 if (GET_MODE (target) != mode)
22347 target = gen_reg_rtx (mode);
22349 xops[1] = gen_lowpart (mode, t2);
22350 xops[2] = gen_lowpart (mode, t1);
22351 xops[3] = gen_rtx_EQ (maskmode, mask, vt);
22354 ok = ix86_expand_int_vcond (xops);
22356 if (target != operands[0])
22357 emit_move_insn (operands[0],
22358 gen_lowpart (GET_MODE (operands[0]), target));
22362 /* Unpack OP[1] into the next wider integer vector type. UNSIGNED_P is
22363 true if we should do zero extension, else sign extension. HIGH_P is
22364 true if we want the N/2 high elements, else the low elements. */
22367 ix86_expand_sse_unpack (rtx dest, rtx src, bool unsigned_p, bool high_p)
22369 machine_mode imode = GET_MODE (src);
22374 rtx (*unpack)(rtx, rtx);
22375 rtx (*extract)(rtx, rtx) = NULL;
22376 machine_mode halfmode = BLKmode;
22382 unpack = gen_avx512bw_zero_extendv32qiv32hi2;
22384 unpack = gen_avx512bw_sign_extendv32qiv32hi2;
22385 halfmode = V32QImode;
22387 = high_p ? gen_vec_extract_hi_v64qi : gen_vec_extract_lo_v64qi;
22391 unpack = gen_avx2_zero_extendv16qiv16hi2;
22393 unpack = gen_avx2_sign_extendv16qiv16hi2;
22394 halfmode = V16QImode;
22396 = high_p ? gen_vec_extract_hi_v32qi : gen_vec_extract_lo_v32qi;
22400 unpack = gen_avx512f_zero_extendv16hiv16si2;
22402 unpack = gen_avx512f_sign_extendv16hiv16si2;
22403 halfmode = V16HImode;
22405 = high_p ? gen_vec_extract_hi_v32hi : gen_vec_extract_lo_v32hi;
22409 unpack = gen_avx2_zero_extendv8hiv8si2;
22411 unpack = gen_avx2_sign_extendv8hiv8si2;
22412 halfmode = V8HImode;
22414 = high_p ? gen_vec_extract_hi_v16hi : gen_vec_extract_lo_v16hi;
22418 unpack = gen_avx512f_zero_extendv8siv8di2;
22420 unpack = gen_avx512f_sign_extendv8siv8di2;
22421 halfmode = V8SImode;
22423 = high_p ? gen_vec_extract_hi_v16si : gen_vec_extract_lo_v16si;
22427 unpack = gen_avx2_zero_extendv4siv4di2;
22429 unpack = gen_avx2_sign_extendv4siv4di2;
22430 halfmode = V4SImode;
22432 = high_p ? gen_vec_extract_hi_v8si : gen_vec_extract_lo_v8si;
22436 unpack = gen_sse4_1_zero_extendv8qiv8hi2;
22438 unpack = gen_sse4_1_sign_extendv8qiv8hi2;
22442 unpack = gen_sse4_1_zero_extendv4hiv4si2;
22444 unpack = gen_sse4_1_sign_extendv4hiv4si2;
22448 unpack = gen_sse4_1_zero_extendv2siv2di2;
22450 unpack = gen_sse4_1_sign_extendv2siv2di2;
22453 gcc_unreachable ();
22456 if (GET_MODE_SIZE (imode) >= 32)
22458 tmp = gen_reg_rtx (halfmode);
22459 emit_insn (extract (tmp, src));
22463 /* Shift higher 8 bytes to lower 8 bytes. */
22464 tmp = gen_reg_rtx (V1TImode);
22465 emit_insn (gen_sse2_lshrv1ti3 (tmp, gen_lowpart (V1TImode, src),
22467 tmp = gen_lowpart (imode, tmp);
22472 emit_insn (unpack (dest, tmp));
22476 rtx (*unpack)(rtx, rtx, rtx);
22482 unpack = gen_vec_interleave_highv16qi;
22484 unpack = gen_vec_interleave_lowv16qi;
22488 unpack = gen_vec_interleave_highv8hi;
22490 unpack = gen_vec_interleave_lowv8hi;
22494 unpack = gen_vec_interleave_highv4si;
22496 unpack = gen_vec_interleave_lowv4si;
22499 gcc_unreachable ();
22503 tmp = force_reg (imode, CONST0_RTX (imode));
22505 tmp = ix86_expand_sse_cmp (gen_reg_rtx (imode), GT, CONST0_RTX (imode),
22506 src, pc_rtx, pc_rtx);
22508 rtx tmp2 = gen_reg_rtx (imode);
22509 emit_insn (unpack (tmp2, src, tmp));
22510 emit_move_insn (dest, gen_lowpart (GET_MODE (dest), tmp2));
22514 /* Expand conditional increment or decrement using adb/sbb instructions.
22515 The default case using setcc followed by the conditional move can be
22516 done by generic code. */
22518 ix86_expand_int_addcc (rtx operands[])
22520 enum rtx_code code = GET_CODE (operands[1]);
22522 rtx (*insn)(rtx, rtx, rtx, rtx, rtx);
22524 rtx val = const0_rtx;
22525 bool fpcmp = false;
22527 rtx op0 = XEXP (operands[1], 0);
22528 rtx op1 = XEXP (operands[1], 1);
22530 if (operands[3] != const1_rtx
22531 && operands[3] != constm1_rtx)
22533 if (!ix86_expand_carry_flag_compare (code, op0, op1, &compare_op))
22535 code = GET_CODE (compare_op);
22537 flags = XEXP (compare_op, 0);
22539 if (GET_MODE (flags) == CCFPmode
22540 || GET_MODE (flags) == CCFPUmode)
22543 code = ix86_fp_compare_code_to_integer (code);
22550 PUT_CODE (compare_op,
22551 reverse_condition_maybe_unordered
22552 (GET_CODE (compare_op)));
22554 PUT_CODE (compare_op, reverse_condition (GET_CODE (compare_op)));
22557 mode = GET_MODE (operands[0]);
22559 /* Construct either adc or sbb insn. */
22560 if ((code == LTU) == (operands[3] == constm1_rtx))
22565 insn = gen_subqi3_carry;
22568 insn = gen_subhi3_carry;
22571 insn = gen_subsi3_carry;
22574 insn = gen_subdi3_carry;
22577 gcc_unreachable ();
22585 insn = gen_addqi3_carry;
22588 insn = gen_addhi3_carry;
22591 insn = gen_addsi3_carry;
22594 insn = gen_adddi3_carry;
22597 gcc_unreachable ();
22600 emit_insn (insn (operands[0], operands[2], val, flags, compare_op));
22606 /* Split operands 0 and 1 into half-mode parts. Similar to split_double_mode,
22607 but works for floating pointer parameters and nonoffsetable memories.
22608 For pushes, it returns just stack offsets; the values will be saved
22609 in the right order. Maximally three parts are generated. */
22612 ix86_split_to_parts (rtx operand, rtx *parts, machine_mode mode)
22617 size = mode==XFmode ? 3 : GET_MODE_SIZE (mode) / 4;
22619 size = (GET_MODE_SIZE (mode) + 4) / 8;
22621 gcc_assert (!REG_P (operand) || !MMX_REGNO_P (REGNO (operand)));
22622 gcc_assert (size >= 2 && size <= 4);
22624 /* Optimize constant pool reference to immediates. This is used by fp
22625 moves, that force all constants to memory to allow combining. */
22626 if (MEM_P (operand) && MEM_READONLY_P (operand))
22628 rtx tmp = maybe_get_pool_constant (operand);
22633 if (MEM_P (operand) && !offsettable_memref_p (operand))
22635 /* The only non-offsetable memories we handle are pushes. */
22636 int ok = push_operand (operand, VOIDmode);
22640 operand = copy_rtx (operand);
22641 PUT_MODE (operand, word_mode);
22642 parts[0] = parts[1] = parts[2] = parts[3] = operand;
22646 if (GET_CODE (operand) == CONST_VECTOR)
22648 machine_mode imode = int_mode_for_mode (mode);
22649 /* Caution: if we looked through a constant pool memory above,
22650 the operand may actually have a different mode now. That's
22651 ok, since we want to pun this all the way back to an integer. */
22652 operand = simplify_subreg (imode, operand, GET_MODE (operand), 0);
22653 gcc_assert (operand != NULL);
22659 if (mode == DImode)
22660 split_double_mode (mode, &operand, 1, &parts[0], &parts[1]);
22665 if (REG_P (operand))
22667 gcc_assert (reload_completed);
22668 for (i = 0; i < size; i++)
22669 parts[i] = gen_rtx_REG (SImode, REGNO (operand) + i);
22671 else if (offsettable_memref_p (operand))
22673 operand = adjust_address (operand, SImode, 0);
22674 parts[0] = operand;
22675 for (i = 1; i < size; i++)
22676 parts[i] = adjust_address (operand, SImode, 4 * i);
22678 else if (CONST_DOUBLE_P (operand))
22683 REAL_VALUE_FROM_CONST_DOUBLE (r, operand);
22687 real_to_target (l, &r, mode);
22688 parts[3] = gen_int_mode (l[3], SImode);
22689 parts[2] = gen_int_mode (l[2], SImode);
22692 /* We can't use REAL_VALUE_TO_TARGET_LONG_DOUBLE since
22693 long double may not be 80-bit. */
22694 real_to_target (l, &r, mode);
22695 parts[2] = gen_int_mode (l[2], SImode);
22698 REAL_VALUE_TO_TARGET_DOUBLE (r, l);
22701 gcc_unreachable ();
22703 parts[1] = gen_int_mode (l[1], SImode);
22704 parts[0] = gen_int_mode (l[0], SImode);
22707 gcc_unreachable ();
22712 if (mode == TImode)
22713 split_double_mode (mode, &operand, 1, &parts[0], &parts[1]);
22714 if (mode == XFmode || mode == TFmode)
22716 machine_mode upper_mode = mode==XFmode ? SImode : DImode;
22717 if (REG_P (operand))
22719 gcc_assert (reload_completed);
22720 parts[0] = gen_rtx_REG (DImode, REGNO (operand) + 0);
22721 parts[1] = gen_rtx_REG (upper_mode, REGNO (operand) + 1);
22723 else if (offsettable_memref_p (operand))
22725 operand = adjust_address (operand, DImode, 0);
22726 parts[0] = operand;
22727 parts[1] = adjust_address (operand, upper_mode, 8);
22729 else if (CONST_DOUBLE_P (operand))
22734 REAL_VALUE_FROM_CONST_DOUBLE (r, operand);
22735 real_to_target (l, &r, mode);
22737 /* real_to_target puts 32-bit pieces in each long. */
22740 ((l[0] & (HOST_WIDE_INT) 0xffffffff)
22741 | ((l[1] & (HOST_WIDE_INT) 0xffffffff) << 32),
22744 if (upper_mode == SImode)
22745 parts[1] = gen_int_mode (l[2], SImode);
22749 ((l[2] & (HOST_WIDE_INT) 0xffffffff)
22750 | ((l[3] & (HOST_WIDE_INT) 0xffffffff) << 32),
22754 gcc_unreachable ();
22761 /* Emit insns to perform a move or push of DI, DF, XF, and TF values.
22762 Return false when normal moves are needed; true when all required
22763 insns have been emitted. Operands 2-4 contain the input values
22764 int the correct order; operands 5-7 contain the output values. */
22767 ix86_split_long_move (rtx operands[])
22772 int collisions = 0;
22773 machine_mode mode = GET_MODE (operands[0]);
22774 bool collisionparts[4];
22776 /* The DFmode expanders may ask us to move double.
22777 For 64bit target this is single move. By hiding the fact
22778 here we simplify i386.md splitters. */
22779 if (TARGET_64BIT && GET_MODE_SIZE (GET_MODE (operands[0])) == 8)
22781 /* Optimize constant pool reference to immediates. This is used by
22782 fp moves, that force all constants to memory to allow combining. */
22784 if (MEM_P (operands[1])
22785 && GET_CODE (XEXP (operands[1], 0)) == SYMBOL_REF
22786 && CONSTANT_POOL_ADDRESS_P (XEXP (operands[1], 0)))
22787 operands[1] = get_pool_constant (XEXP (operands[1], 0));
22788 if (push_operand (operands[0], VOIDmode))
22790 operands[0] = copy_rtx (operands[0]);
22791 PUT_MODE (operands[0], word_mode);
22794 operands[0] = gen_lowpart (DImode, operands[0]);
22795 operands[1] = gen_lowpart (DImode, operands[1]);
22796 emit_move_insn (operands[0], operands[1]);
22800 /* The only non-offsettable memory we handle is push. */
22801 if (push_operand (operands[0], VOIDmode))
22804 gcc_assert (!MEM_P (operands[0])
22805 || offsettable_memref_p (operands[0]));
22807 nparts = ix86_split_to_parts (operands[1], part[1], GET_MODE (operands[0]));
22808 ix86_split_to_parts (operands[0], part[0], GET_MODE (operands[0]));
22810 /* When emitting push, take care for source operands on the stack. */
22811 if (push && MEM_P (operands[1])
22812 && reg_overlap_mentioned_p (stack_pointer_rtx, operands[1]))
22814 rtx src_base = XEXP (part[1][nparts - 1], 0);
22816 /* Compensate for the stack decrement by 4. */
22817 if (!TARGET_64BIT && nparts == 3
22818 && mode == XFmode && TARGET_128BIT_LONG_DOUBLE)
22819 src_base = plus_constant (Pmode, src_base, 4);
22821 /* src_base refers to the stack pointer and is
22822 automatically decreased by emitted push. */
22823 for (i = 0; i < nparts; i++)
22824 part[1][i] = change_address (part[1][i],
22825 GET_MODE (part[1][i]), src_base);
22828 /* We need to do copy in the right order in case an address register
22829 of the source overlaps the destination. */
22830 if (REG_P (part[0][0]) && MEM_P (part[1][0]))
22834 for (i = 0; i < nparts; i++)
22837 = reg_overlap_mentioned_p (part[0][i], XEXP (part[1][0], 0));
22838 if (collisionparts[i])
22842 /* Collision in the middle part can be handled by reordering. */
22843 if (collisions == 1 && nparts == 3 && collisionparts [1])
22845 std::swap (part[0][1], part[0][2]);
22846 std::swap (part[1][1], part[1][2]);
22848 else if (collisions == 1
22850 && (collisionparts [1] || collisionparts [2]))
22852 if (collisionparts [1])
22854 std::swap (part[0][1], part[0][2]);
22855 std::swap (part[1][1], part[1][2]);
22859 std::swap (part[0][2], part[0][3]);
22860 std::swap (part[1][2], part[1][3]);
22864 /* If there are more collisions, we can't handle it by reordering.
22865 Do an lea to the last part and use only one colliding move. */
22866 else if (collisions > 1)
22868 rtx base, addr, tls_base = NULL_RTX;
22872 base = part[0][nparts - 1];
22874 /* Handle the case when the last part isn't valid for lea.
22875 Happens in 64-bit mode storing the 12-byte XFmode. */
22876 if (GET_MODE (base) != Pmode)
22877 base = gen_rtx_REG (Pmode, REGNO (base));
22879 addr = XEXP (part[1][0], 0);
22880 if (TARGET_TLS_DIRECT_SEG_REFS)
22882 struct ix86_address parts;
22883 int ok = ix86_decompose_address (addr, &parts);
22885 if (parts.seg == DEFAULT_TLS_SEG_REG)
22887 /* It is not valid to use %gs: or %fs: in
22888 lea though, so we need to remove it from the
22889 address used for lea and add it to each individual
22890 memory loads instead. */
22891 addr = copy_rtx (addr);
22893 while (GET_CODE (*x) == PLUS)
22895 for (i = 0; i < 2; i++)
22897 rtx u = XEXP (*x, i);
22898 if (GET_CODE (u) == ZERO_EXTEND)
22900 if (GET_CODE (u) == UNSPEC
22901 && XINT (u, 1) == UNSPEC_TP)
22903 tls_base = XEXP (*x, i);
22904 *x = XEXP (*x, 1 - i);
22912 gcc_assert (tls_base);
22915 emit_insn (gen_rtx_SET (base, addr));
22917 base = gen_rtx_PLUS (GET_MODE (base), base, tls_base);
22918 part[1][0] = replace_equiv_address (part[1][0], base);
22919 for (i = 1; i < nparts; i++)
22922 base = copy_rtx (base);
22923 tmp = plus_constant (Pmode, base, UNITS_PER_WORD * i);
22924 part[1][i] = replace_equiv_address (part[1][i], tmp);
22935 if (TARGET_128BIT_LONG_DOUBLE && mode == XFmode)
22936 emit_insn (ix86_gen_add3 (stack_pointer_rtx,
22937 stack_pointer_rtx, GEN_INT (-4)));
22938 emit_move_insn (part[0][2], part[1][2]);
22940 else if (nparts == 4)
22942 emit_move_insn (part[0][3], part[1][3]);
22943 emit_move_insn (part[0][2], part[1][2]);
22948 /* In 64bit mode we don't have 32bit push available. In case this is
22949 register, it is OK - we will just use larger counterpart. We also
22950 retype memory - these comes from attempt to avoid REX prefix on
22951 moving of second half of TFmode value. */
22952 if (GET_MODE (part[1][1]) == SImode)
22954 switch (GET_CODE (part[1][1]))
22957 part[1][1] = adjust_address (part[1][1], DImode, 0);
22961 part[1][1] = gen_rtx_REG (DImode, REGNO (part[1][1]));
22965 gcc_unreachable ();
22968 if (GET_MODE (part[1][0]) == SImode)
22969 part[1][0] = part[1][1];
22972 emit_move_insn (part[0][1], part[1][1]);
22973 emit_move_insn (part[0][0], part[1][0]);
22977 /* Choose correct order to not overwrite the source before it is copied. */
22978 if ((REG_P (part[0][0])
22979 && REG_P (part[1][1])
22980 && (REGNO (part[0][0]) == REGNO (part[1][1])
22982 && REGNO (part[0][0]) == REGNO (part[1][2]))
22984 && REGNO (part[0][0]) == REGNO (part[1][3]))))
22986 && reg_overlap_mentioned_p (part[0][0], XEXP (part[1][0], 0))))
22988 for (i = 0, j = nparts - 1; i < nparts; i++, j--)
22990 operands[2 + i] = part[0][j];
22991 operands[6 + i] = part[1][j];
22996 for (i = 0; i < nparts; i++)
22998 operands[2 + i] = part[0][i];
22999 operands[6 + i] = part[1][i];
23003 /* If optimizing for size, attempt to locally unCSE nonzero constants. */
23004 if (optimize_insn_for_size_p ())
23006 for (j = 0; j < nparts - 1; j++)
23007 if (CONST_INT_P (operands[6 + j])
23008 && operands[6 + j] != const0_rtx
23009 && REG_P (operands[2 + j]))
23010 for (i = j; i < nparts - 1; i++)
23011 if (CONST_INT_P (operands[7 + i])
23012 && INTVAL (operands[7 + i]) == INTVAL (operands[6 + j]))
23013 operands[7 + i] = operands[2 + j];
23016 for (i = 0; i < nparts; i++)
23017 emit_move_insn (operands[2 + i], operands[6 + i]);
23022 /* Helper function of ix86_split_ashl used to generate an SImode/DImode
23023 left shift by a constant, either using a single shift or
23024 a sequence of add instructions. */
23027 ix86_expand_ashl_const (rtx operand, int count, machine_mode mode)
23029 rtx (*insn)(rtx, rtx, rtx);
23032 || (count * ix86_cost->add <= ix86_cost->shift_const
23033 && !optimize_insn_for_size_p ()))
23035 insn = mode == DImode ? gen_addsi3 : gen_adddi3;
23036 while (count-- > 0)
23037 emit_insn (insn (operand, operand, operand));
23041 insn = mode == DImode ? gen_ashlsi3 : gen_ashldi3;
23042 emit_insn (insn (operand, operand, GEN_INT (count)));
23047 ix86_split_ashl (rtx *operands, rtx scratch, machine_mode mode)
23049 rtx (*gen_ashl3)(rtx, rtx, rtx);
23050 rtx (*gen_shld)(rtx, rtx, rtx);
23051 int half_width = GET_MODE_BITSIZE (mode) >> 1;
23053 rtx low[2], high[2];
23056 if (CONST_INT_P (operands[2]))
23058 split_double_mode (mode, operands, 2, low, high);
23059 count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (mode) - 1);
23061 if (count >= half_width)
23063 emit_move_insn (high[0], low[1]);
23064 emit_move_insn (low[0], const0_rtx);
23066 if (count > half_width)
23067 ix86_expand_ashl_const (high[0], count - half_width, mode);
23071 gen_shld = mode == DImode ? gen_x86_shld : gen_x86_64_shld;
23073 if (!rtx_equal_p (operands[0], operands[1]))
23074 emit_move_insn (operands[0], operands[1]);
23076 emit_insn (gen_shld (high[0], low[0], GEN_INT (count)));
23077 ix86_expand_ashl_const (low[0], count, mode);
23082 split_double_mode (mode, operands, 1, low, high);
23084 gen_ashl3 = mode == DImode ? gen_ashlsi3 : gen_ashldi3;
23086 if (operands[1] == const1_rtx)
23088 /* Assuming we've chosen a QImode capable registers, then 1 << N
23089 can be done with two 32/64-bit shifts, no branches, no cmoves. */
23090 if (ANY_QI_REG_P (low[0]) && ANY_QI_REG_P (high[0]))
23092 rtx s, d, flags = gen_rtx_REG (CCZmode, FLAGS_REG);
23094 ix86_expand_clear (low[0]);
23095 ix86_expand_clear (high[0]);
23096 emit_insn (gen_testqi_ccz_1 (operands[2], GEN_INT (half_width)));
23098 d = gen_lowpart (QImode, low[0]);
23099 d = gen_rtx_STRICT_LOW_PART (VOIDmode, d);
23100 s = gen_rtx_EQ (QImode, flags, const0_rtx);
23101 emit_insn (gen_rtx_SET (d, s));
23103 d = gen_lowpart (QImode, high[0]);
23104 d = gen_rtx_STRICT_LOW_PART (VOIDmode, d);
23105 s = gen_rtx_NE (QImode, flags, const0_rtx);
23106 emit_insn (gen_rtx_SET (d, s));
23109 /* Otherwise, we can get the same results by manually performing
23110 a bit extract operation on bit 5/6, and then performing the two
23111 shifts. The two methods of getting 0/1 into low/high are exactly
23112 the same size. Avoiding the shift in the bit extract case helps
23113 pentium4 a bit; no one else seems to care much either way. */
23116 machine_mode half_mode;
23117 rtx (*gen_lshr3)(rtx, rtx, rtx);
23118 rtx (*gen_and3)(rtx, rtx, rtx);
23119 rtx (*gen_xor3)(rtx, rtx, rtx);
23120 HOST_WIDE_INT bits;
23123 if (mode == DImode)
23125 half_mode = SImode;
23126 gen_lshr3 = gen_lshrsi3;
23127 gen_and3 = gen_andsi3;
23128 gen_xor3 = gen_xorsi3;
23133 half_mode = DImode;
23134 gen_lshr3 = gen_lshrdi3;
23135 gen_and3 = gen_anddi3;
23136 gen_xor3 = gen_xordi3;
23140 if (TARGET_PARTIAL_REG_STALL && !optimize_insn_for_size_p ())
23141 x = gen_rtx_ZERO_EXTEND (half_mode, operands[2]);
23143 x = gen_lowpart (half_mode, operands[2]);
23144 emit_insn (gen_rtx_SET (high[0], x));
23146 emit_insn (gen_lshr3 (high[0], high[0], GEN_INT (bits)));
23147 emit_insn (gen_and3 (high[0], high[0], const1_rtx));
23148 emit_move_insn (low[0], high[0]);
23149 emit_insn (gen_xor3 (low[0], low[0], const1_rtx));
23152 emit_insn (gen_ashl3 (low[0], low[0], operands[2]));
23153 emit_insn (gen_ashl3 (high[0], high[0], operands[2]));
23157 if (operands[1] == constm1_rtx)
23159 /* For -1 << N, we can avoid the shld instruction, because we
23160 know that we're shifting 0...31/63 ones into a -1. */
23161 emit_move_insn (low[0], constm1_rtx);
23162 if (optimize_insn_for_size_p ())
23163 emit_move_insn (high[0], low[0]);
23165 emit_move_insn (high[0], constm1_rtx);
23169 gen_shld = mode == DImode ? gen_x86_shld : gen_x86_64_shld;
23171 if (!rtx_equal_p (operands[0], operands[1]))
23172 emit_move_insn (operands[0], operands[1]);
23174 split_double_mode (mode, operands, 1, low, high);
23175 emit_insn (gen_shld (high[0], low[0], operands[2]));
23178 emit_insn (gen_ashl3 (low[0], low[0], operands[2]));
23180 if (TARGET_CMOVE && scratch)
23182 rtx (*gen_x86_shift_adj_1)(rtx, rtx, rtx, rtx)
23183 = mode == DImode ? gen_x86_shiftsi_adj_1 : gen_x86_shiftdi_adj_1;
23185 ix86_expand_clear (scratch);
23186 emit_insn (gen_x86_shift_adj_1 (high[0], low[0], operands[2], scratch));
23190 rtx (*gen_x86_shift_adj_2)(rtx, rtx, rtx)
23191 = mode == DImode ? gen_x86_shiftsi_adj_2 : gen_x86_shiftdi_adj_2;
23193 emit_insn (gen_x86_shift_adj_2 (high[0], low[0], operands[2]));
23198 ix86_split_ashr (rtx *operands, rtx scratch, machine_mode mode)
23200 rtx (*gen_ashr3)(rtx, rtx, rtx)
23201 = mode == DImode ? gen_ashrsi3 : gen_ashrdi3;
23202 rtx (*gen_shrd)(rtx, rtx, rtx);
23203 int half_width = GET_MODE_BITSIZE (mode) >> 1;
23205 rtx low[2], high[2];
23208 if (CONST_INT_P (operands[2]))
23210 split_double_mode (mode, operands, 2, low, high);
23211 count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (mode) - 1);
23213 if (count == GET_MODE_BITSIZE (mode) - 1)
23215 emit_move_insn (high[0], high[1]);
23216 emit_insn (gen_ashr3 (high[0], high[0],
23217 GEN_INT (half_width - 1)));
23218 emit_move_insn (low[0], high[0]);
23221 else if (count >= half_width)
23223 emit_move_insn (low[0], high[1]);
23224 emit_move_insn (high[0], low[0]);
23225 emit_insn (gen_ashr3 (high[0], high[0],
23226 GEN_INT (half_width - 1)));
23228 if (count > half_width)
23229 emit_insn (gen_ashr3 (low[0], low[0],
23230 GEN_INT (count - half_width)));
23234 gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd;
23236 if (!rtx_equal_p (operands[0], operands[1]))
23237 emit_move_insn (operands[0], operands[1]);
23239 emit_insn (gen_shrd (low[0], high[0], GEN_INT (count)));
23240 emit_insn (gen_ashr3 (high[0], high[0], GEN_INT (count)));
23245 gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd;
23247 if (!rtx_equal_p (operands[0], operands[1]))
23248 emit_move_insn (operands[0], operands[1]);
23250 split_double_mode (mode, operands, 1, low, high);
23252 emit_insn (gen_shrd (low[0], high[0], operands[2]));
23253 emit_insn (gen_ashr3 (high[0], high[0], operands[2]));
23255 if (TARGET_CMOVE && scratch)
23257 rtx (*gen_x86_shift_adj_1)(rtx, rtx, rtx, rtx)
23258 = mode == DImode ? gen_x86_shiftsi_adj_1 : gen_x86_shiftdi_adj_1;
23260 emit_move_insn (scratch, high[0]);
23261 emit_insn (gen_ashr3 (scratch, scratch,
23262 GEN_INT (half_width - 1)));
23263 emit_insn (gen_x86_shift_adj_1 (low[0], high[0], operands[2],
23268 rtx (*gen_x86_shift_adj_3)(rtx, rtx, rtx)
23269 = mode == DImode ? gen_x86_shiftsi_adj_3 : gen_x86_shiftdi_adj_3;
23271 emit_insn (gen_x86_shift_adj_3 (low[0], high[0], operands[2]));
23277 ix86_split_lshr (rtx *operands, rtx scratch, machine_mode mode)
23279 rtx (*gen_lshr3)(rtx, rtx, rtx)
23280 = mode == DImode ? gen_lshrsi3 : gen_lshrdi3;
23281 rtx (*gen_shrd)(rtx, rtx, rtx);
23282 int half_width = GET_MODE_BITSIZE (mode) >> 1;
23284 rtx low[2], high[2];
23287 if (CONST_INT_P (operands[2]))
23289 split_double_mode (mode, operands, 2, low, high);
23290 count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (mode) - 1);
23292 if (count >= half_width)
23294 emit_move_insn (low[0], high[1]);
23295 ix86_expand_clear (high[0]);
23297 if (count > half_width)
23298 emit_insn (gen_lshr3 (low[0], low[0],
23299 GEN_INT (count - half_width)));
23303 gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd;
23305 if (!rtx_equal_p (operands[0], operands[1]))
23306 emit_move_insn (operands[0], operands[1]);
23308 emit_insn (gen_shrd (low[0], high[0], GEN_INT (count)));
23309 emit_insn (gen_lshr3 (high[0], high[0], GEN_INT (count)));
23314 gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd;
23316 if (!rtx_equal_p (operands[0], operands[1]))
23317 emit_move_insn (operands[0], operands[1]);
23319 split_double_mode (mode, operands, 1, low, high);
23321 emit_insn (gen_shrd (low[0], high[0], operands[2]));
23322 emit_insn (gen_lshr3 (high[0], high[0], operands[2]));
23324 if (TARGET_CMOVE && scratch)
23326 rtx (*gen_x86_shift_adj_1)(rtx, rtx, rtx, rtx)
23327 = mode == DImode ? gen_x86_shiftsi_adj_1 : gen_x86_shiftdi_adj_1;
23329 ix86_expand_clear (scratch);
23330 emit_insn (gen_x86_shift_adj_1 (low[0], high[0], operands[2],
23335 rtx (*gen_x86_shift_adj_2)(rtx, rtx, rtx)
23336 = mode == DImode ? gen_x86_shiftsi_adj_2 : gen_x86_shiftdi_adj_2;
23338 emit_insn (gen_x86_shift_adj_2 (low[0], high[0], operands[2]));
23343 /* Predict just emitted jump instruction to be taken with probability PROB. */
23345 predict_jump (int prob)
23347 rtx insn = get_last_insn ();
23348 gcc_assert (JUMP_P (insn));
23349 add_int_reg_note (insn, REG_BR_PROB, prob);
23352 /* Helper function for the string operations below. Dest VARIABLE whether
23353 it is aligned to VALUE bytes. If true, jump to the label. */
23354 static rtx_code_label *
23355 ix86_expand_aligntest (rtx variable, int value, bool epilogue)
23357 rtx_code_label *label = gen_label_rtx ();
23358 rtx tmpcount = gen_reg_rtx (GET_MODE (variable));
23359 if (GET_MODE (variable) == DImode)
23360 emit_insn (gen_anddi3 (tmpcount, variable, GEN_INT (value)));
23362 emit_insn (gen_andsi3 (tmpcount, variable, GEN_INT (value)));
23363 emit_cmp_and_jump_insns (tmpcount, const0_rtx, EQ, 0, GET_MODE (variable),
23366 predict_jump (REG_BR_PROB_BASE * 50 / 100);
23368 predict_jump (REG_BR_PROB_BASE * 90 / 100);
23372 /* Adjust COUNTER by the VALUE. */
23374 ix86_adjust_counter (rtx countreg, HOST_WIDE_INT value)
23376 rtx (*gen_add)(rtx, rtx, rtx)
23377 = GET_MODE (countreg) == DImode ? gen_adddi3 : gen_addsi3;
23379 emit_insn (gen_add (countreg, countreg, GEN_INT (-value)));
23382 /* Zero extend possibly SImode EXP to Pmode register. */
23384 ix86_zero_extend_to_Pmode (rtx exp)
23386 return force_reg (Pmode, convert_to_mode (Pmode, exp, 1));
23389 /* Divide COUNTREG by SCALE. */
23391 scale_counter (rtx countreg, int scale)
23397 if (CONST_INT_P (countreg))
23398 return GEN_INT (INTVAL (countreg) / scale);
23399 gcc_assert (REG_P (countreg));
23401 sc = expand_simple_binop (GET_MODE (countreg), LSHIFTRT, countreg,
23402 GEN_INT (exact_log2 (scale)),
23403 NULL, 1, OPTAB_DIRECT);
23407 /* Return mode for the memcpy/memset loop counter. Prefer SImode over
23408 DImode for constant loop counts. */
23410 static machine_mode
23411 counter_mode (rtx count_exp)
23413 if (GET_MODE (count_exp) != VOIDmode)
23414 return GET_MODE (count_exp);
23415 if (!CONST_INT_P (count_exp))
23417 if (TARGET_64BIT && (INTVAL (count_exp) & ~0xffffffff))
23422 /* Copy the address to a Pmode register. This is used for x32 to
23423 truncate DImode TLS address to a SImode register. */
23426 ix86_copy_addr_to_reg (rtx addr)
23429 if (GET_MODE (addr) == Pmode || GET_MODE (addr) == VOIDmode)
23431 reg = copy_addr_to_reg (addr);
23432 REG_POINTER (reg) = 1;
23437 gcc_assert (GET_MODE (addr) == DImode && Pmode == SImode);
23438 reg = copy_to_mode_reg (DImode, addr);
23439 REG_POINTER (reg) = 1;
23440 return gen_rtx_SUBREG (SImode, reg, 0);
23444 /* When ISSETMEM is FALSE, output simple loop to move memory pointer to SRCPTR
23445 to DESTPTR via chunks of MODE unrolled UNROLL times, overall size is COUNT
23446 specified in bytes. When ISSETMEM is TRUE, output the equivalent loop to set
23447 memory by VALUE (supposed to be in MODE).
23449 The size is rounded down to whole number of chunk size moved at once.
23450 SRCMEM and DESTMEM provide MEMrtx to feed proper aliasing info. */
23454 expand_set_or_movmem_via_loop (rtx destmem, rtx srcmem,
23455 rtx destptr, rtx srcptr, rtx value,
23456 rtx count, machine_mode mode, int unroll,
23457 int expected_size, bool issetmem)
23459 rtx_code_label *out_label, *top_label;
23461 machine_mode iter_mode = counter_mode (count);
23462 int piece_size_n = GET_MODE_SIZE (mode) * unroll;
23463 rtx piece_size = GEN_INT (piece_size_n);
23464 rtx piece_size_mask = GEN_INT (~((GET_MODE_SIZE (mode) * unroll) - 1));
23468 top_label = gen_label_rtx ();
23469 out_label = gen_label_rtx ();
23470 iter = gen_reg_rtx (iter_mode);
23472 size = expand_simple_binop (iter_mode, AND, count, piece_size_mask,
23473 NULL, 1, OPTAB_DIRECT);
23474 /* Those two should combine. */
23475 if (piece_size == const1_rtx)
23477 emit_cmp_and_jump_insns (size, const0_rtx, EQ, NULL_RTX, iter_mode,
23479 predict_jump (REG_BR_PROB_BASE * 10 / 100);
23481 emit_move_insn (iter, const0_rtx);
23483 emit_label (top_label);
23485 tmp = convert_modes (Pmode, iter_mode, iter, true);
23487 /* This assert could be relaxed - in this case we'll need to compute
23488 smallest power of two, containing in PIECE_SIZE_N and pass it to
23490 gcc_assert ((piece_size_n & (piece_size_n - 1)) == 0);
23491 destmem = offset_address (destmem, tmp, piece_size_n);
23492 destmem = adjust_address (destmem, mode, 0);
23496 srcmem = offset_address (srcmem, copy_rtx (tmp), piece_size_n);
23497 srcmem = adjust_address (srcmem, mode, 0);
23499 /* When unrolling for chips that reorder memory reads and writes,
23500 we can save registers by using single temporary.
23501 Also using 4 temporaries is overkill in 32bit mode. */
23502 if (!TARGET_64BIT && 0)
23504 for (i = 0; i < unroll; i++)
23509 adjust_address (copy_rtx (destmem), mode, GET_MODE_SIZE (mode));
23511 adjust_address (copy_rtx (srcmem), mode, GET_MODE_SIZE (mode));
23513 emit_move_insn (destmem, srcmem);
23519 gcc_assert (unroll <= 4);
23520 for (i = 0; i < unroll; i++)
23522 tmpreg[i] = gen_reg_rtx (mode);
23526 adjust_address (copy_rtx (srcmem), mode, GET_MODE_SIZE (mode));
23528 emit_move_insn (tmpreg[i], srcmem);
23530 for (i = 0; i < unroll; i++)
23535 adjust_address (copy_rtx (destmem), mode, GET_MODE_SIZE (mode));
23537 emit_move_insn (destmem, tmpreg[i]);
23542 for (i = 0; i < unroll; i++)
23546 adjust_address (copy_rtx (destmem), mode, GET_MODE_SIZE (mode));
23547 emit_move_insn (destmem, value);
23550 tmp = expand_simple_binop (iter_mode, PLUS, iter, piece_size, iter,
23551 true, OPTAB_LIB_WIDEN);
23553 emit_move_insn (iter, tmp);
23555 emit_cmp_and_jump_insns (iter, size, LT, NULL_RTX, iter_mode,
23557 if (expected_size != -1)
23559 expected_size /= GET_MODE_SIZE (mode) * unroll;
23560 if (expected_size == 0)
23562 else if (expected_size > REG_BR_PROB_BASE)
23563 predict_jump (REG_BR_PROB_BASE - 1);
23565 predict_jump (REG_BR_PROB_BASE - (REG_BR_PROB_BASE + expected_size / 2) / expected_size);
23568 predict_jump (REG_BR_PROB_BASE * 80 / 100);
23569 iter = ix86_zero_extend_to_Pmode (iter);
23570 tmp = expand_simple_binop (Pmode, PLUS, destptr, iter, destptr,
23571 true, OPTAB_LIB_WIDEN);
23572 if (tmp != destptr)
23573 emit_move_insn (destptr, tmp);
23576 tmp = expand_simple_binop (Pmode, PLUS, srcptr, iter, srcptr,
23577 true, OPTAB_LIB_WIDEN);
23579 emit_move_insn (srcptr, tmp);
23581 emit_label (out_label);
23584 /* Output "rep; mov" or "rep; stos" instruction depending on ISSETMEM argument.
23585 When ISSETMEM is true, arguments SRCMEM and SRCPTR are ignored.
23586 When ISSETMEM is false, arguments VALUE and ORIG_VALUE are ignored.
23587 For setmem case, VALUE is a promoted to a wider size ORIG_VALUE.
23588 ORIG_VALUE is the original value passed to memset to fill the memory with.
23589 Other arguments have same meaning as for previous function. */
23592 expand_set_or_movmem_via_rep (rtx destmem, rtx srcmem,
23593 rtx destptr, rtx srcptr, rtx value, rtx orig_value,
23595 machine_mode mode, bool issetmem)
23600 HOST_WIDE_INT rounded_count;
23602 /* If possible, it is shorter to use rep movs.
23603 TODO: Maybe it is better to move this logic to decide_alg. */
23604 if (mode == QImode && CONST_INT_P (count) && !(INTVAL (count) & 3)
23605 && (!issetmem || orig_value == const0_rtx))
23608 if (destptr != XEXP (destmem, 0) || GET_MODE (destmem) != BLKmode)
23609 destmem = adjust_automodify_address_nv (destmem, BLKmode, destptr, 0);
23611 countreg = ix86_zero_extend_to_Pmode (scale_counter (count,
23612 GET_MODE_SIZE (mode)));
23613 if (mode != QImode)
23615 destexp = gen_rtx_ASHIFT (Pmode, countreg,
23616 GEN_INT (exact_log2 (GET_MODE_SIZE (mode))));
23617 destexp = gen_rtx_PLUS (Pmode, destexp, destptr);
23620 destexp = gen_rtx_PLUS (Pmode, destptr, countreg);
23621 if ((!issetmem || orig_value == const0_rtx) && CONST_INT_P (count))
23623 rounded_count = (INTVAL (count)
23624 & ~((HOST_WIDE_INT) GET_MODE_SIZE (mode) - 1));
23625 destmem = shallow_copy_rtx (destmem);
23626 set_mem_size (destmem, rounded_count);
23628 else if (MEM_SIZE_KNOWN_P (destmem))
23629 clear_mem_size (destmem);
23633 value = force_reg (mode, gen_lowpart (mode, value));
23634 emit_insn (gen_rep_stos (destptr, countreg, destmem, value, destexp));
23638 if (srcptr != XEXP (srcmem, 0) || GET_MODE (srcmem) != BLKmode)
23639 srcmem = adjust_automodify_address_nv (srcmem, BLKmode, srcptr, 0);
23640 if (mode != QImode)
23642 srcexp = gen_rtx_ASHIFT (Pmode, countreg,
23643 GEN_INT (exact_log2 (GET_MODE_SIZE (mode))));
23644 srcexp = gen_rtx_PLUS (Pmode, srcexp, srcptr);
23647 srcexp = gen_rtx_PLUS (Pmode, srcptr, countreg);
23648 if (CONST_INT_P (count))
23650 rounded_count = (INTVAL (count)
23651 & ~((HOST_WIDE_INT) GET_MODE_SIZE (mode) - 1));
23652 srcmem = shallow_copy_rtx (srcmem);
23653 set_mem_size (srcmem, rounded_count);
23657 if (MEM_SIZE_KNOWN_P (srcmem))
23658 clear_mem_size (srcmem);
23660 emit_insn (gen_rep_mov (destptr, destmem, srcptr, srcmem, countreg,
23665 /* This function emits moves to copy SIZE_TO_MOVE bytes from SRCMEM to
23667 SRC is passed by pointer to be updated on return.
23668 Return value is updated DST. */
23670 emit_memmov (rtx destmem, rtx *srcmem, rtx destptr, rtx srcptr,
23671 HOST_WIDE_INT size_to_move)
23673 rtx dst = destmem, src = *srcmem, adjust, tempreg;
23674 enum insn_code code;
23675 machine_mode move_mode;
23678 /* Find the widest mode in which we could perform moves.
23679 Start with the biggest power of 2 less than SIZE_TO_MOVE and half
23680 it until move of such size is supported. */
23681 piece_size = 1 << floor_log2 (size_to_move);
23682 move_mode = mode_for_size (piece_size * BITS_PER_UNIT, MODE_INT, 0);
23683 code = optab_handler (mov_optab, move_mode);
23684 while (code == CODE_FOR_nothing && piece_size > 1)
23687 move_mode = mode_for_size (piece_size * BITS_PER_UNIT, MODE_INT, 0);
23688 code = optab_handler (mov_optab, move_mode);
23691 /* Find the corresponding vector mode with the same size as MOVE_MODE.
23692 MOVE_MODE is an integer mode at the moment (SI, DI, TI, etc.). */
23693 if (GET_MODE_SIZE (move_mode) > GET_MODE_SIZE (word_mode))
23695 int nunits = GET_MODE_SIZE (move_mode) / GET_MODE_SIZE (word_mode);
23696 move_mode = mode_for_vector (word_mode, nunits);
23697 code = optab_handler (mov_optab, move_mode);
23698 if (code == CODE_FOR_nothing)
23700 move_mode = word_mode;
23701 piece_size = GET_MODE_SIZE (move_mode);
23702 code = optab_handler (mov_optab, move_mode);
23705 gcc_assert (code != CODE_FOR_nothing);
23707 dst = adjust_automodify_address_nv (dst, move_mode, destptr, 0);
23708 src = adjust_automodify_address_nv (src, move_mode, srcptr, 0);
23710 /* Emit moves. We'll need SIZE_TO_MOVE/PIECE_SIZES moves. */
23711 gcc_assert (size_to_move % piece_size == 0);
23712 adjust = GEN_INT (piece_size);
23713 for (i = 0; i < size_to_move; i += piece_size)
23715 /* We move from memory to memory, so we'll need to do it via
23716 a temporary register. */
23717 tempreg = gen_reg_rtx (move_mode);
23718 emit_insn (GEN_FCN (code) (tempreg, src));
23719 emit_insn (GEN_FCN (code) (dst, tempreg));
23721 emit_move_insn (destptr,
23722 gen_rtx_PLUS (Pmode, copy_rtx (destptr), adjust));
23723 emit_move_insn (srcptr,
23724 gen_rtx_PLUS (Pmode, copy_rtx (srcptr), adjust));
23726 dst = adjust_automodify_address_nv (dst, move_mode, destptr,
23728 src = adjust_automodify_address_nv (src, move_mode, srcptr,
23732 /* Update DST and SRC rtx. */
23737 /* Output code to copy at most count & (max_size - 1) bytes from SRC to DEST. */
23739 expand_movmem_epilogue (rtx destmem, rtx srcmem,
23740 rtx destptr, rtx srcptr, rtx count, int max_size)
23743 if (CONST_INT_P (count))
23745 HOST_WIDE_INT countval = INTVAL (count);
23746 HOST_WIDE_INT epilogue_size = countval % max_size;
23749 /* For now MAX_SIZE should be a power of 2. This assert could be
23750 relaxed, but it'll require a bit more complicated epilogue
23752 gcc_assert ((max_size & (max_size - 1)) == 0);
23753 for (i = max_size; i >= 1; i >>= 1)
23755 if (epilogue_size & i)
23756 destmem = emit_memmov (destmem, &srcmem, destptr, srcptr, i);
23762 count = expand_simple_binop (GET_MODE (count), AND, count, GEN_INT (max_size - 1),
23763 count, 1, OPTAB_DIRECT);
23764 expand_set_or_movmem_via_loop (destmem, srcmem, destptr, srcptr, NULL,
23765 count, QImode, 1, 4, false);
23769 /* When there are stringops, we can cheaply increase dest and src pointers.
23770 Otherwise we save code size by maintaining offset (zero is readily
23771 available from preceding rep operation) and using x86 addressing modes.
23773 if (TARGET_SINGLE_STRINGOP)
23777 rtx_code_label *label = ix86_expand_aligntest (count, 4, true);
23778 src = change_address (srcmem, SImode, srcptr);
23779 dest = change_address (destmem, SImode, destptr);
23780 emit_insn (gen_strmov (destptr, dest, srcptr, src));
23781 emit_label (label);
23782 LABEL_NUSES (label) = 1;
23786 rtx_code_label *label = ix86_expand_aligntest (count, 2, true);
23787 src = change_address (srcmem, HImode, srcptr);
23788 dest = change_address (destmem, HImode, destptr);
23789 emit_insn (gen_strmov (destptr, dest, srcptr, src));
23790 emit_label (label);
23791 LABEL_NUSES (label) = 1;
23795 rtx_code_label *label = ix86_expand_aligntest (count, 1, true);
23796 src = change_address (srcmem, QImode, srcptr);
23797 dest = change_address (destmem, QImode, destptr);
23798 emit_insn (gen_strmov (destptr, dest, srcptr, src));
23799 emit_label (label);
23800 LABEL_NUSES (label) = 1;
23805 rtx offset = force_reg (Pmode, const0_rtx);
23810 rtx_code_label *label = ix86_expand_aligntest (count, 4, true);
23811 src = change_address (srcmem, SImode, srcptr);
23812 dest = change_address (destmem, SImode, destptr);
23813 emit_move_insn (dest, src);
23814 tmp = expand_simple_binop (Pmode, PLUS, offset, GEN_INT (4), NULL,
23815 true, OPTAB_LIB_WIDEN);
23817 emit_move_insn (offset, tmp);
23818 emit_label (label);
23819 LABEL_NUSES (label) = 1;
23823 rtx_code_label *label = ix86_expand_aligntest (count, 2, true);
23824 tmp = gen_rtx_PLUS (Pmode, srcptr, offset);
23825 src = change_address (srcmem, HImode, tmp);
23826 tmp = gen_rtx_PLUS (Pmode, destptr, offset);
23827 dest = change_address (destmem, HImode, tmp);
23828 emit_move_insn (dest, src);
23829 tmp = expand_simple_binop (Pmode, PLUS, offset, GEN_INT (2), tmp,
23830 true, OPTAB_LIB_WIDEN);
23832 emit_move_insn (offset, tmp);
23833 emit_label (label);
23834 LABEL_NUSES (label) = 1;
23838 rtx_code_label *label = ix86_expand_aligntest (count, 1, true);
23839 tmp = gen_rtx_PLUS (Pmode, srcptr, offset);
23840 src = change_address (srcmem, QImode, tmp);
23841 tmp = gen_rtx_PLUS (Pmode, destptr, offset);
23842 dest = change_address (destmem, QImode, tmp);
23843 emit_move_insn (dest, src);
23844 emit_label (label);
23845 LABEL_NUSES (label) = 1;
23850 /* This function emits moves to fill SIZE_TO_MOVE bytes starting from DESTMEM
23851 with value PROMOTED_VAL.
23852 SRC is passed by pointer to be updated on return.
23853 Return value is updated DST. */
23855 emit_memset (rtx destmem, rtx destptr, rtx promoted_val,
23856 HOST_WIDE_INT size_to_move)
23858 rtx dst = destmem, adjust;
23859 enum insn_code code;
23860 machine_mode move_mode;
23863 /* Find the widest mode in which we could perform moves.
23864 Start with the biggest power of 2 less than SIZE_TO_MOVE and half
23865 it until move of such size is supported. */
23866 move_mode = GET_MODE (promoted_val);
23867 if (move_mode == VOIDmode)
23868 move_mode = QImode;
23869 if (size_to_move < GET_MODE_SIZE (move_mode))
23871 move_mode = mode_for_size (size_to_move * BITS_PER_UNIT, MODE_INT, 0);
23872 promoted_val = gen_lowpart (move_mode, promoted_val);
23874 piece_size = GET_MODE_SIZE (move_mode);
23875 code = optab_handler (mov_optab, move_mode);
23876 gcc_assert (code != CODE_FOR_nothing && promoted_val != NULL_RTX);
23878 dst = adjust_automodify_address_nv (dst, move_mode, destptr, 0);
23880 /* Emit moves. We'll need SIZE_TO_MOVE/PIECE_SIZES moves. */
23881 gcc_assert (size_to_move % piece_size == 0);
23882 adjust = GEN_INT (piece_size);
23883 for (i = 0; i < size_to_move; i += piece_size)
23885 if (piece_size <= GET_MODE_SIZE (word_mode))
23887 emit_insn (gen_strset (destptr, dst, promoted_val));
23888 dst = adjust_automodify_address_nv (dst, move_mode, destptr,
23893 emit_insn (GEN_FCN (code) (dst, promoted_val));
23895 emit_move_insn (destptr,
23896 gen_rtx_PLUS (Pmode, copy_rtx (destptr), adjust));
23898 dst = adjust_automodify_address_nv (dst, move_mode, destptr,
23902 /* Update DST rtx. */
23905 /* Output code to set at most count & (max_size - 1) bytes starting by DEST. */
23907 expand_setmem_epilogue_via_loop (rtx destmem, rtx destptr, rtx value,
23908 rtx count, int max_size)
23911 expand_simple_binop (counter_mode (count), AND, count,
23912 GEN_INT (max_size - 1), count, 1, OPTAB_DIRECT);
23913 expand_set_or_movmem_via_loop (destmem, NULL, destptr, NULL,
23914 gen_lowpart (QImode, value), count, QImode,
23915 1, max_size / 2, true);
23918 /* Output code to set at most count & (max_size - 1) bytes starting by DEST. */
23920 expand_setmem_epilogue (rtx destmem, rtx destptr, rtx value, rtx vec_value,
23921 rtx count, int max_size)
23925 if (CONST_INT_P (count))
23927 HOST_WIDE_INT countval = INTVAL (count);
23928 HOST_WIDE_INT epilogue_size = countval % max_size;
23931 /* For now MAX_SIZE should be a power of 2. This assert could be
23932 relaxed, but it'll require a bit more complicated epilogue
23934 gcc_assert ((max_size & (max_size - 1)) == 0);
23935 for (i = max_size; i >= 1; i >>= 1)
23937 if (epilogue_size & i)
23939 if (vec_value && i > GET_MODE_SIZE (GET_MODE (value)))
23940 destmem = emit_memset (destmem, destptr, vec_value, i);
23942 destmem = emit_memset (destmem, destptr, value, i);
23949 expand_setmem_epilogue_via_loop (destmem, destptr, value, count, max_size);
23954 rtx_code_label *label = ix86_expand_aligntest (count, 16, true);
23957 dest = change_address (destmem, DImode, destptr);
23958 emit_insn (gen_strset (destptr, dest, value));
23959 dest = adjust_automodify_address_nv (dest, DImode, destptr, 8);
23960 emit_insn (gen_strset (destptr, dest, value));
23964 dest = change_address (destmem, SImode, destptr);
23965 emit_insn (gen_strset (destptr, dest, value));
23966 dest = adjust_automodify_address_nv (dest, SImode, destptr, 4);
23967 emit_insn (gen_strset (destptr, dest, value));
23968 dest = adjust_automodify_address_nv (dest, SImode, destptr, 8);
23969 emit_insn (gen_strset (destptr, dest, value));
23970 dest = adjust_automodify_address_nv (dest, SImode, destptr, 12);
23971 emit_insn (gen_strset (destptr, dest, value));
23973 emit_label (label);
23974 LABEL_NUSES (label) = 1;
23978 rtx_code_label *label = ix86_expand_aligntest (count, 8, true);
23981 dest = change_address (destmem, DImode, destptr);
23982 emit_insn (gen_strset (destptr, dest, value));
23986 dest = change_address (destmem, SImode, destptr);
23987 emit_insn (gen_strset (destptr, dest, value));
23988 dest = adjust_automodify_address_nv (dest, SImode, destptr, 4);
23989 emit_insn (gen_strset (destptr, dest, value));
23991 emit_label (label);
23992 LABEL_NUSES (label) = 1;
23996 rtx_code_label *label = ix86_expand_aligntest (count, 4, true);
23997 dest = change_address (destmem, SImode, destptr);
23998 emit_insn (gen_strset (destptr, dest, gen_lowpart (SImode, value)));
23999 emit_label (label);
24000 LABEL_NUSES (label) = 1;
24004 rtx_code_label *label = ix86_expand_aligntest (count, 2, true);
24005 dest = change_address (destmem, HImode, destptr);
24006 emit_insn (gen_strset (destptr, dest, gen_lowpart (HImode, value)));
24007 emit_label (label);
24008 LABEL_NUSES (label) = 1;
24012 rtx_code_label *label = ix86_expand_aligntest (count, 1, true);
24013 dest = change_address (destmem, QImode, destptr);
24014 emit_insn (gen_strset (destptr, dest, gen_lowpart (QImode, value)));
24015 emit_label (label);
24016 LABEL_NUSES (label) = 1;
24020 /* Depending on ISSETMEM, copy enough from SRCMEM to DESTMEM or set enough to
24021 DESTMEM to align it to DESIRED_ALIGNMENT. Original alignment is ALIGN.
24022 Depending on ISSETMEM, either arguments SRCMEM/SRCPTR or VALUE/VEC_VALUE are
24024 Return value is updated DESTMEM. */
24026 expand_set_or_movmem_prologue (rtx destmem, rtx srcmem,
24027 rtx destptr, rtx srcptr, rtx value,
24028 rtx vec_value, rtx count, int align,
24029 int desired_alignment, bool issetmem)
24032 for (i = 1; i < desired_alignment; i <<= 1)
24036 rtx_code_label *label = ix86_expand_aligntest (destptr, i, false);
24039 if (vec_value && i > GET_MODE_SIZE (GET_MODE (value)))
24040 destmem = emit_memset (destmem, destptr, vec_value, i);
24042 destmem = emit_memset (destmem, destptr, value, i);
24045 destmem = emit_memmov (destmem, &srcmem, destptr, srcptr, i);
24046 ix86_adjust_counter (count, i);
24047 emit_label (label);
24048 LABEL_NUSES (label) = 1;
24049 set_mem_align (destmem, i * 2 * BITS_PER_UNIT);
24055 /* Test if COUNT&SIZE is nonzero and if so, expand movme
24056 or setmem sequence that is valid for SIZE..2*SIZE-1 bytes
24057 and jump to DONE_LABEL. */
24059 expand_small_movmem_or_setmem (rtx destmem, rtx srcmem,
24060 rtx destptr, rtx srcptr,
24061 rtx value, rtx vec_value,
24062 rtx count, int size,
24063 rtx done_label, bool issetmem)
24065 rtx_code_label *label = ix86_expand_aligntest (count, size, false);
24066 machine_mode mode = mode_for_size (size * BITS_PER_UNIT, MODE_INT, 1);
24070 /* If we do not have vector value to copy, we must reduce size. */
24075 if (GET_MODE (value) == VOIDmode && size > 8)
24077 else if (GET_MODE_SIZE (mode) > GET_MODE_SIZE (GET_MODE (value)))
24078 mode = GET_MODE (value);
24081 mode = GET_MODE (vec_value), value = vec_value;
24085 /* Choose appropriate vector mode. */
24087 mode = TARGET_AVX ? V32QImode : TARGET_SSE ? V16QImode : DImode;
24088 else if (size >= 16)
24089 mode = TARGET_SSE ? V16QImode : DImode;
24090 srcmem = change_address (srcmem, mode, srcptr);
24092 destmem = change_address (destmem, mode, destptr);
24093 modesize = GEN_INT (GET_MODE_SIZE (mode));
24094 gcc_assert (GET_MODE_SIZE (mode) <= size);
24095 for (n = 0; n * GET_MODE_SIZE (mode) < size; n++)
24098 emit_move_insn (destmem, gen_lowpart (mode, value));
24101 emit_move_insn (destmem, srcmem);
24102 srcmem = offset_address (srcmem, modesize, GET_MODE_SIZE (mode));
24104 destmem = offset_address (destmem, modesize, GET_MODE_SIZE (mode));
24107 destmem = offset_address (destmem, count, 1);
24108 destmem = offset_address (destmem, GEN_INT (-2 * size),
24109 GET_MODE_SIZE (mode));
24112 srcmem = offset_address (srcmem, count, 1);
24113 srcmem = offset_address (srcmem, GEN_INT (-2 * size),
24114 GET_MODE_SIZE (mode));
24116 for (n = 0; n * GET_MODE_SIZE (mode) < size; n++)
24119 emit_move_insn (destmem, gen_lowpart (mode, value));
24122 emit_move_insn (destmem, srcmem);
24123 srcmem = offset_address (srcmem, modesize, GET_MODE_SIZE (mode));
24125 destmem = offset_address (destmem, modesize, GET_MODE_SIZE (mode));
24127 emit_jump_insn (gen_jump (done_label));
24130 emit_label (label);
24131 LABEL_NUSES (label) = 1;
24134 /* Handle small memcpy (up to SIZE that is supposed to be small power of 2.
24135 and get ready for the main memcpy loop by copying iniital DESIRED_ALIGN-ALIGN
24136 bytes and last SIZE bytes adjusitng DESTPTR/SRCPTR/COUNT in a way we can
24137 proceed with an loop copying SIZE bytes at once. Do moves in MODE.
24138 DONE_LABEL is a label after the whole copying sequence. The label is created
24139 on demand if *DONE_LABEL is NULL.
24140 MIN_SIZE is minimal size of block copied. This value gets adjusted for new
24141 bounds after the initial copies.
24143 DESTMEM/SRCMEM are memory expressions pointing to the copies block,
24144 DESTPTR/SRCPTR are pointers to the block. DYNAMIC_CHECK indicate whether
24145 we will dispatch to a library call for large blocks.
24147 In pseudocode we do:
24151 Assume that SIZE is 4. Bigger sizes are handled analogously
24154 copy 4 bytes from SRCPTR to DESTPTR
24155 copy 4 bytes from SRCPTR + COUNT - 4 to DESTPTR + COUNT - 4
24160 copy 1 byte from SRCPTR to DESTPTR
24163 copy 2 bytes from SRCPTR to DESTPTR
24164 copy 2 bytes from SRCPTR + COUNT - 2 to DESTPTR + COUNT - 2
24169 copy at least DESIRED_ALIGN-ALIGN bytes from SRCPTR to DESTPTR
24170 copy SIZE bytes from SRCPTR + COUNT - SIZE to DESTPTR + COUNT -SIZE
24172 OLD_DESPTR = DESTPTR;
24173 Align DESTPTR up to DESIRED_ALIGN
24174 SRCPTR += DESTPTR - OLD_DESTPTR
24175 COUNT -= DEST_PTR - OLD_DESTPTR
24177 Round COUNT down to multiple of SIZE
24178 << optional caller supplied zero size guard is here >>
24179 << optional caller suppplied dynamic check is here >>
24180 << caller supplied main copy loop is here >>
24185 expand_set_or_movmem_prologue_epilogue_by_misaligned_moves (rtx destmem, rtx srcmem,
24186 rtx *destptr, rtx *srcptr,
24188 rtx value, rtx vec_value,
24190 rtx_code_label **done_label,
24194 unsigned HOST_WIDE_INT *min_size,
24195 bool dynamic_check,
24198 rtx_code_label *loop_label = NULL, *label;
24201 int prolog_size = 0;
24204 /* Chose proper value to copy. */
24205 if (issetmem && VECTOR_MODE_P (mode))
24206 mode_value = vec_value;
24208 mode_value = value;
24209 gcc_assert (GET_MODE_SIZE (mode) <= size);
24211 /* See if block is big or small, handle small blocks. */
24212 if (!CONST_INT_P (*count) && *min_size < (unsigned HOST_WIDE_INT)size)
24215 loop_label = gen_label_rtx ();
24218 *done_label = gen_label_rtx ();
24220 emit_cmp_and_jump_insns (*count, GEN_INT (size2), GE, 0, GET_MODE (*count),
24224 /* Handle sizes > 3. */
24225 for (;size2 > 2; size2 >>= 1)
24226 expand_small_movmem_or_setmem (destmem, srcmem,
24230 size2, *done_label, issetmem);
24231 /* Nothing to copy? Jump to DONE_LABEL if so */
24232 emit_cmp_and_jump_insns (*count, const0_rtx, EQ, 0, GET_MODE (*count),
24235 /* Do a byte copy. */
24236 destmem = change_address (destmem, QImode, *destptr);
24238 emit_move_insn (destmem, gen_lowpart (QImode, value));
24241 srcmem = change_address (srcmem, QImode, *srcptr);
24242 emit_move_insn (destmem, srcmem);
24245 /* Handle sizes 2 and 3. */
24246 label = ix86_expand_aligntest (*count, 2, false);
24247 destmem = change_address (destmem, HImode, *destptr);
24248 destmem = offset_address (destmem, *count, 1);
24249 destmem = offset_address (destmem, GEN_INT (-2), 2);
24251 emit_move_insn (destmem, gen_lowpart (HImode, value));
24254 srcmem = change_address (srcmem, HImode, *srcptr);
24255 srcmem = offset_address (srcmem, *count, 1);
24256 srcmem = offset_address (srcmem, GEN_INT (-2), 2);
24257 emit_move_insn (destmem, srcmem);
24260 emit_label (label);
24261 LABEL_NUSES (label) = 1;
24262 emit_jump_insn (gen_jump (*done_label));
24266 gcc_assert (*min_size >= (unsigned HOST_WIDE_INT)size
24267 || UINTVAL (*count) >= (unsigned HOST_WIDE_INT)size);
24269 /* Start memcpy for COUNT >= SIZE. */
24272 emit_label (loop_label);
24273 LABEL_NUSES (loop_label) = 1;
24276 /* Copy first desired_align bytes. */
24278 srcmem = change_address (srcmem, mode, *srcptr);
24279 destmem = change_address (destmem, mode, *destptr);
24280 modesize = GEN_INT (GET_MODE_SIZE (mode));
24281 for (n = 0; prolog_size < desired_align - align; n++)
24284 emit_move_insn (destmem, mode_value);
24287 emit_move_insn (destmem, srcmem);
24288 srcmem = offset_address (srcmem, modesize, GET_MODE_SIZE (mode));
24290 destmem = offset_address (destmem, modesize, GET_MODE_SIZE (mode));
24291 prolog_size += GET_MODE_SIZE (mode);
24295 /* Copy last SIZE bytes. */
24296 destmem = offset_address (destmem, *count, 1);
24297 destmem = offset_address (destmem,
24298 GEN_INT (-size - prolog_size),
24301 emit_move_insn (destmem, mode_value);
24304 srcmem = offset_address (srcmem, *count, 1);
24305 srcmem = offset_address (srcmem,
24306 GEN_INT (-size - prolog_size),
24308 emit_move_insn (destmem, srcmem);
24310 for (n = 1; n * GET_MODE_SIZE (mode) < size; n++)
24312 destmem = offset_address (destmem, modesize, 1);
24314 emit_move_insn (destmem, mode_value);
24317 srcmem = offset_address (srcmem, modesize, 1);
24318 emit_move_insn (destmem, srcmem);
24322 /* Align destination. */
24323 if (desired_align > 1 && desired_align > align)
24325 rtx saveddest = *destptr;
24327 gcc_assert (desired_align <= size);
24328 /* Align destptr up, place it to new register. */
24329 *destptr = expand_simple_binop (GET_MODE (*destptr), PLUS, *destptr,
24330 GEN_INT (prolog_size),
24331 NULL_RTX, 1, OPTAB_DIRECT);
24332 if (REG_P (*destptr) && REG_P (saveddest) && REG_POINTER (saveddest))
24333 REG_POINTER (*destptr) = 1;
24334 *destptr = expand_simple_binop (GET_MODE (*destptr), AND, *destptr,
24335 GEN_INT (-desired_align),
24336 *destptr, 1, OPTAB_DIRECT);
24337 /* See how many bytes we skipped. */
24338 saveddest = expand_simple_binop (GET_MODE (*destptr), MINUS, saveddest,
24340 saveddest, 1, OPTAB_DIRECT);
24341 /* Adjust srcptr and count. */
24343 *srcptr = expand_simple_binop (GET_MODE (*srcptr), MINUS, *srcptr,
24344 saveddest, *srcptr, 1, OPTAB_DIRECT);
24345 *count = expand_simple_binop (GET_MODE (*count), PLUS, *count,
24346 saveddest, *count, 1, OPTAB_DIRECT);
24347 /* We copied at most size + prolog_size. */
24348 if (*min_size > (unsigned HOST_WIDE_INT)(size + prolog_size))
24349 *min_size = (*min_size - size) & ~(unsigned HOST_WIDE_INT)(size - 1);
24353 /* Our loops always round down the bock size, but for dispatch to library
24354 we need precise value. */
24356 *count = expand_simple_binop (GET_MODE (*count), AND, *count,
24357 GEN_INT (-size), *count, 1, OPTAB_DIRECT);
24361 gcc_assert (prolog_size == 0);
24362 /* Decrease count, so we won't end up copying last word twice. */
24363 if (!CONST_INT_P (*count))
24364 *count = expand_simple_binop (GET_MODE (*count), PLUS, *count,
24365 constm1_rtx, *count, 1, OPTAB_DIRECT);
24367 *count = GEN_INT ((UINTVAL (*count) - 1) & ~(unsigned HOST_WIDE_INT)(size - 1));
24369 *min_size = (*min_size - 1) & ~(unsigned HOST_WIDE_INT)(size - 1);
24374 /* This function is like the previous one, except here we know how many bytes
24375 need to be copied. That allows us to update alignment not only of DST, which
24376 is returned, but also of SRC, which is passed as a pointer for that
24379 expand_set_or_movmem_constant_prologue (rtx dst, rtx *srcp, rtx destreg,
24380 rtx srcreg, rtx value, rtx vec_value,
24381 int desired_align, int align_bytes,
24385 rtx orig_dst = dst;
24386 rtx orig_src = NULL;
24387 int piece_size = 1;
24388 int copied_bytes = 0;
24392 gcc_assert (srcp != NULL);
24397 for (piece_size = 1;
24398 piece_size <= desired_align && copied_bytes < align_bytes;
24401 if (align_bytes & piece_size)
24405 if (vec_value && piece_size > GET_MODE_SIZE (GET_MODE (value)))
24406 dst = emit_memset (dst, destreg, vec_value, piece_size);
24408 dst = emit_memset (dst, destreg, value, piece_size);
24411 dst = emit_memmov (dst, &src, destreg, srcreg, piece_size);
24412 copied_bytes += piece_size;
24415 if (MEM_ALIGN (dst) < (unsigned int) desired_align * BITS_PER_UNIT)
24416 set_mem_align (dst, desired_align * BITS_PER_UNIT);
24417 if (MEM_SIZE_KNOWN_P (orig_dst))
24418 set_mem_size (dst, MEM_SIZE (orig_dst) - align_bytes);
24422 int src_align_bytes = get_mem_align_offset (src, desired_align
24424 if (src_align_bytes >= 0)
24425 src_align_bytes = desired_align - src_align_bytes;
24426 if (src_align_bytes >= 0)
24428 unsigned int src_align;
24429 for (src_align = desired_align; src_align >= 2; src_align >>= 1)
24431 if ((src_align_bytes & (src_align - 1))
24432 == (align_bytes & (src_align - 1)))
24435 if (src_align > (unsigned int) desired_align)
24436 src_align = desired_align;
24437 if (MEM_ALIGN (src) < src_align * BITS_PER_UNIT)
24438 set_mem_align (src, src_align * BITS_PER_UNIT);
24440 if (MEM_SIZE_KNOWN_P (orig_src))
24441 set_mem_size (src, MEM_SIZE (orig_src) - align_bytes);
24448 /* Return true if ALG can be used in current context.
24449 Assume we expand memset if MEMSET is true. */
24451 alg_usable_p (enum stringop_alg alg, bool memset)
24453 if (alg == no_stringop)
24455 if (alg == vector_loop)
24456 return TARGET_SSE || TARGET_AVX;
24457 /* Algorithms using the rep prefix want at least edi and ecx;
24458 additionally, memset wants eax and memcpy wants esi. Don't
24459 consider such algorithms if the user has appropriated those
24460 registers for their own purposes. */
24461 if (alg == rep_prefix_1_byte
24462 || alg == rep_prefix_4_byte
24463 || alg == rep_prefix_8_byte)
24464 return !(fixed_regs[CX_REG] || fixed_regs[DI_REG]
24465 || (memset ? fixed_regs[AX_REG] : fixed_regs[SI_REG]));
24469 /* Given COUNT and EXPECTED_SIZE, decide on codegen of string operation. */
24470 static enum stringop_alg
24471 decide_alg (HOST_WIDE_INT count, HOST_WIDE_INT expected_size,
24472 unsigned HOST_WIDE_INT min_size, unsigned HOST_WIDE_INT max_size,
24473 bool memset, bool zero_memset, int *dynamic_check, bool *noalign)
24475 const struct stringop_algs * algs;
24476 bool optimize_for_speed;
24478 const struct processor_costs *cost;
24480 bool any_alg_usable_p = false;
24483 *dynamic_check = -1;
24485 /* Even if the string operation call is cold, we still might spend a lot
24486 of time processing large blocks. */
24487 if (optimize_function_for_size_p (cfun)
24488 || (optimize_insn_for_size_p ()
24490 || (expected_size != -1 && expected_size < 256))))
24491 optimize_for_speed = false;
24493 optimize_for_speed = true;
24495 cost = optimize_for_speed ? ix86_cost : &ix86_size_cost;
24497 algs = &cost->memset[TARGET_64BIT != 0];
24499 algs = &cost->memcpy[TARGET_64BIT != 0];
24501 /* See maximal size for user defined algorithm. */
24502 for (i = 0; i < MAX_STRINGOP_ALGS; i++)
24504 enum stringop_alg candidate = algs->size[i].alg;
24505 bool usable = alg_usable_p (candidate, memset);
24506 any_alg_usable_p |= usable;
24508 if (candidate != libcall && candidate && usable)
24509 max = algs->size[i].max;
24512 /* If expected size is not known but max size is small enough
24513 so inline version is a win, set expected size into
24515 if (((max > 1 && (unsigned HOST_WIDE_INT) max >= max_size) || max == -1)
24516 && expected_size == -1)
24517 expected_size = min_size / 2 + max_size / 2;
24519 /* If user specified the algorithm, honnor it if possible. */
24520 if (ix86_stringop_alg != no_stringop
24521 && alg_usable_p (ix86_stringop_alg, memset))
24522 return ix86_stringop_alg;
24523 /* rep; movq or rep; movl is the smallest variant. */
24524 else if (!optimize_for_speed)
24527 if (!count || (count & 3) || (memset && !zero_memset))
24528 return alg_usable_p (rep_prefix_1_byte, memset)
24529 ? rep_prefix_1_byte : loop_1_byte;
24531 return alg_usable_p (rep_prefix_4_byte, memset)
24532 ? rep_prefix_4_byte : loop;
24534 /* Very tiny blocks are best handled via the loop, REP is expensive to
24536 else if (expected_size != -1 && expected_size < 4)
24537 return loop_1_byte;
24538 else if (expected_size != -1)
24540 enum stringop_alg alg = libcall;
24541 bool alg_noalign = false;
24542 for (i = 0; i < MAX_STRINGOP_ALGS; i++)
24544 /* We get here if the algorithms that were not libcall-based
24545 were rep-prefix based and we are unable to use rep prefixes
24546 based on global register usage. Break out of the loop and
24547 use the heuristic below. */
24548 if (algs->size[i].max == 0)
24550 if (algs->size[i].max >= expected_size || algs->size[i].max == -1)
24552 enum stringop_alg candidate = algs->size[i].alg;
24554 if (candidate != libcall && alg_usable_p (candidate, memset))
24557 alg_noalign = algs->size[i].noalign;
24559 /* Honor TARGET_INLINE_ALL_STRINGOPS by picking
24560 last non-libcall inline algorithm. */
24561 if (TARGET_INLINE_ALL_STRINGOPS)
24563 /* When the current size is best to be copied by a libcall,
24564 but we are still forced to inline, run the heuristic below
24565 that will pick code for medium sized blocks. */
24566 if (alg != libcall)
24568 *noalign = alg_noalign;
24571 else if (!any_alg_usable_p)
24574 else if (alg_usable_p (candidate, memset))
24576 *noalign = algs->size[i].noalign;
24582 /* When asked to inline the call anyway, try to pick meaningful choice.
24583 We look for maximal size of block that is faster to copy by hand and
24584 take blocks of at most of that size guessing that average size will
24585 be roughly half of the block.
24587 If this turns out to be bad, we might simply specify the preferred
24588 choice in ix86_costs. */
24589 if ((TARGET_INLINE_ALL_STRINGOPS || TARGET_INLINE_STRINGOPS_DYNAMICALLY)
24590 && (algs->unknown_size == libcall
24591 || !alg_usable_p (algs->unknown_size, memset)))
24593 enum stringop_alg alg;
24595 /* If there aren't any usable algorithms, then recursing on
24596 smaller sizes isn't going to find anything. Just return the
24597 simple byte-at-a-time copy loop. */
24598 if (!any_alg_usable_p)
24600 /* Pick something reasonable. */
24601 if (TARGET_INLINE_STRINGOPS_DYNAMICALLY)
24602 *dynamic_check = 128;
24603 return loop_1_byte;
24607 alg = decide_alg (count, max / 2, min_size, max_size, memset,
24608 zero_memset, dynamic_check, noalign);
24609 gcc_assert (*dynamic_check == -1);
24610 if (TARGET_INLINE_STRINGOPS_DYNAMICALLY)
24611 *dynamic_check = max;
24613 gcc_assert (alg != libcall);
24616 return (alg_usable_p (algs->unknown_size, memset)
24617 ? algs->unknown_size : libcall);
24620 /* Decide on alignment. We know that the operand is already aligned to ALIGN
24621 (ALIGN can be based on profile feedback and thus it is not 100% guaranteed). */
24623 decide_alignment (int align,
24624 enum stringop_alg alg,
24626 machine_mode move_mode)
24628 int desired_align = 0;
24630 gcc_assert (alg != no_stringop);
24632 if (alg == libcall)
24634 if (move_mode == VOIDmode)
24637 desired_align = GET_MODE_SIZE (move_mode);
24638 /* PentiumPro has special logic triggering for 8 byte aligned blocks.
24639 copying whole cacheline at once. */
24640 if (TARGET_PENTIUMPRO
24641 && (alg == rep_prefix_4_byte || alg == rep_prefix_1_byte))
24646 if (desired_align < align)
24647 desired_align = align;
24648 if (expected_size != -1 && expected_size < 4)
24649 desired_align = align;
24651 return desired_align;
24655 /* Helper function for memcpy. For QImode value 0xXY produce
24656 0xXYXYXYXY of wide specified by MODE. This is essentially
24657 a * 0x10101010, but we can do slightly better than
24658 synth_mult by unwinding the sequence by hand on CPUs with
24661 promote_duplicated_reg (machine_mode mode, rtx val)
24663 machine_mode valmode = GET_MODE (val);
24665 int nops = mode == DImode ? 3 : 2;
24667 gcc_assert (mode == SImode || mode == DImode || val == const0_rtx);
24668 if (val == const0_rtx)
24669 return copy_to_mode_reg (mode, CONST0_RTX (mode));
24670 if (CONST_INT_P (val))
24672 HOST_WIDE_INT v = INTVAL (val) & 255;
24676 if (mode == DImode)
24677 v |= (v << 16) << 16;
24678 return copy_to_mode_reg (mode, gen_int_mode (v, mode));
24681 if (valmode == VOIDmode)
24683 if (valmode != QImode)
24684 val = gen_lowpart (QImode, val);
24685 if (mode == QImode)
24687 if (!TARGET_PARTIAL_REG_STALL)
24689 if (ix86_cost->mult_init[mode == DImode ? 3 : 2]
24690 + ix86_cost->mult_bit * (mode == DImode ? 8 : 4)
24691 <= (ix86_cost->shift_const + ix86_cost->add) * nops
24692 + (COSTS_N_INSNS (TARGET_PARTIAL_REG_STALL == 0)))
24694 rtx reg = convert_modes (mode, QImode, val, true);
24695 tmp = promote_duplicated_reg (mode, const1_rtx);
24696 return expand_simple_binop (mode, MULT, reg, tmp, NULL, 1,
24701 rtx reg = convert_modes (mode, QImode, val, true);
24703 if (!TARGET_PARTIAL_REG_STALL)
24704 if (mode == SImode)
24705 emit_insn (gen_movsi_insv_1 (reg, reg));
24707 emit_insn (gen_movdi_insv_1 (reg, reg));
24710 tmp = expand_simple_binop (mode, ASHIFT, reg, GEN_INT (8),
24711 NULL, 1, OPTAB_DIRECT);
24713 expand_simple_binop (mode, IOR, reg, tmp, reg, 1, OPTAB_DIRECT);
24715 tmp = expand_simple_binop (mode, ASHIFT, reg, GEN_INT (16),
24716 NULL, 1, OPTAB_DIRECT);
24717 reg = expand_simple_binop (mode, IOR, reg, tmp, reg, 1, OPTAB_DIRECT);
24718 if (mode == SImode)
24720 tmp = expand_simple_binop (mode, ASHIFT, reg, GEN_INT (32),
24721 NULL, 1, OPTAB_DIRECT);
24722 reg = expand_simple_binop (mode, IOR, reg, tmp, reg, 1, OPTAB_DIRECT);
24727 /* Duplicate value VAL using promote_duplicated_reg into maximal size that will
24728 be needed by main loop copying SIZE_NEEDED chunks and prologue getting
24729 alignment from ALIGN to DESIRED_ALIGN. */
24731 promote_duplicated_reg_to_size (rtx val, int size_needed, int desired_align,
24737 && (size_needed > 4 || (desired_align > align && desired_align > 4)))
24738 promoted_val = promote_duplicated_reg (DImode, val);
24739 else if (size_needed > 2 || (desired_align > align && desired_align > 2))
24740 promoted_val = promote_duplicated_reg (SImode, val);
24741 else if (size_needed > 1 || (desired_align > align && desired_align > 1))
24742 promoted_val = promote_duplicated_reg (HImode, val);
24744 promoted_val = val;
24746 return promoted_val;
24749 /* Expand string move (memcpy) ot store (memset) operation. Use i386 string
24750 operations when profitable. The code depends upon architecture, block size
24751 and alignment, but always has one of the following overall structures:
24753 Aligned move sequence:
24755 1) Prologue guard: Conditional that jumps up to epilogues for small
24756 blocks that can be handled by epilogue alone. This is faster
24757 but also needed for correctness, since prologue assume the block
24758 is larger than the desired alignment.
24760 Optional dynamic check for size and libcall for large
24761 blocks is emitted here too, with -minline-stringops-dynamically.
24763 2) Prologue: copy first few bytes in order to get destination
24764 aligned to DESIRED_ALIGN. It is emitted only when ALIGN is less
24765 than DESIRED_ALIGN and up to DESIRED_ALIGN - ALIGN bytes can be
24766 copied. We emit either a jump tree on power of two sized
24767 blocks, or a byte loop.
24769 3) Main body: the copying loop itself, copying in SIZE_NEEDED chunks
24770 with specified algorithm.
24772 4) Epilogue: code copying tail of the block that is too small to be
24773 handled by main body (or up to size guarded by prologue guard).
24775 Misaligned move sequence
24777 1) missaligned move prologue/epilogue containing:
24778 a) Prologue handling small memory blocks and jumping to done_label
24779 (skipped if blocks are known to be large enough)
24780 b) Signle move copying first DESIRED_ALIGN-ALIGN bytes if alignment is
24781 needed by single possibly misaligned move
24782 (skipped if alignment is not needed)
24783 c) Copy of last SIZE_NEEDED bytes by possibly misaligned moves
24785 2) Zero size guard dispatching to done_label, if needed
24787 3) dispatch to library call, if needed,
24789 3) Main body: the copying loop itself, copying in SIZE_NEEDED chunks
24790 with specified algorithm. */
24792 ix86_expand_set_or_movmem (rtx dst, rtx src, rtx count_exp, rtx val_exp,
24793 rtx align_exp, rtx expected_align_exp,
24794 rtx expected_size_exp, rtx min_size_exp,
24795 rtx max_size_exp, rtx probable_max_size_exp,
24800 rtx_code_label *label = NULL;
24802 rtx_code_label *jump_around_label = NULL;
24803 HOST_WIDE_INT align = 1;
24804 unsigned HOST_WIDE_INT count = 0;
24805 HOST_WIDE_INT expected_size = -1;
24806 int size_needed = 0, epilogue_size_needed;
24807 int desired_align = 0, align_bytes = 0;
24808 enum stringop_alg alg;
24809 rtx promoted_val = NULL;
24810 rtx vec_promoted_val = NULL;
24811 bool force_loopy_epilogue = false;
24813 bool need_zero_guard = false;
24815 machine_mode move_mode = VOIDmode;
24816 int unroll_factor = 1;
24817 /* TODO: Once value ranges are available, fill in proper data. */
24818 unsigned HOST_WIDE_INT min_size = 0;
24819 unsigned HOST_WIDE_INT max_size = -1;
24820 unsigned HOST_WIDE_INT probable_max_size = -1;
24821 bool misaligned_prologue_used = false;
24823 if (CONST_INT_P (align_exp))
24824 align = INTVAL (align_exp);
24825 /* i386 can do misaligned access on reasonably increased cost. */
24826 if (CONST_INT_P (expected_align_exp)
24827 && INTVAL (expected_align_exp) > align)
24828 align = INTVAL (expected_align_exp);
24829 /* ALIGN is the minimum of destination and source alignment, but we care here
24830 just about destination alignment. */
24832 && MEM_ALIGN (dst) > (unsigned HOST_WIDE_INT) align * BITS_PER_UNIT)
24833 align = MEM_ALIGN (dst) / BITS_PER_UNIT;
24835 if (CONST_INT_P (count_exp))
24837 min_size = max_size = probable_max_size = count = expected_size
24838 = INTVAL (count_exp);
24839 /* When COUNT is 0, there is nothing to do. */
24846 min_size = INTVAL (min_size_exp);
24848 max_size = INTVAL (max_size_exp);
24849 if (probable_max_size_exp)
24850 probable_max_size = INTVAL (probable_max_size_exp);
24851 if (CONST_INT_P (expected_size_exp))
24852 expected_size = INTVAL (expected_size_exp);
24855 /* Make sure we don't need to care about overflow later on. */
24856 if (count > (HOST_WIDE_INT_1U << 30))
24859 /* Step 0: Decide on preferred algorithm, desired alignment and
24860 size of chunks to be copied by main loop. */
24861 alg = decide_alg (count, expected_size, min_size, probable_max_size,
24863 issetmem && val_exp == const0_rtx,
24864 &dynamic_check, &noalign);
24865 if (alg == libcall)
24867 gcc_assert (alg != no_stringop);
24869 /* For now vector-version of memset is generated only for memory zeroing, as
24870 creating of promoted vector value is very cheap in this case. */
24871 if (issetmem && alg == vector_loop && val_exp != const0_rtx)
24872 alg = unrolled_loop;
24875 count_exp = copy_to_mode_reg (GET_MODE (count_exp), count_exp);
24876 destreg = ix86_copy_addr_to_reg (XEXP (dst, 0));
24878 srcreg = ix86_copy_addr_to_reg (XEXP (src, 0));
24881 move_mode = word_mode;
24887 gcc_unreachable ();
24889 need_zero_guard = true;
24890 move_mode = QImode;
24893 need_zero_guard = true;
24895 case unrolled_loop:
24896 need_zero_guard = true;
24897 unroll_factor = (TARGET_64BIT ? 4 : 2);
24900 need_zero_guard = true;
24902 /* Find the widest supported mode. */
24903 move_mode = word_mode;
24904 while (optab_handler (mov_optab, GET_MODE_WIDER_MODE (move_mode))
24905 != CODE_FOR_nothing)
24906 move_mode = GET_MODE_WIDER_MODE (move_mode);
24908 /* Find the corresponding vector mode with the same size as MOVE_MODE.
24909 MOVE_MODE is an integer mode at the moment (SI, DI, TI, etc.). */
24910 if (GET_MODE_SIZE (move_mode) > GET_MODE_SIZE (word_mode))
24912 int nunits = GET_MODE_SIZE (move_mode) / GET_MODE_SIZE (word_mode);
24913 move_mode = mode_for_vector (word_mode, nunits);
24914 if (optab_handler (mov_optab, move_mode) == CODE_FOR_nothing)
24915 move_mode = word_mode;
24917 gcc_assert (optab_handler (mov_optab, move_mode) != CODE_FOR_nothing);
24919 case rep_prefix_8_byte:
24920 move_mode = DImode;
24922 case rep_prefix_4_byte:
24923 move_mode = SImode;
24925 case rep_prefix_1_byte:
24926 move_mode = QImode;
24929 size_needed = GET_MODE_SIZE (move_mode) * unroll_factor;
24930 epilogue_size_needed = size_needed;
24932 desired_align = decide_alignment (align, alg, expected_size, move_mode);
24933 if (!TARGET_ALIGN_STRINGOPS || noalign)
24934 align = desired_align;
24936 /* Step 1: Prologue guard. */
24938 /* Alignment code needs count to be in register. */
24939 if (CONST_INT_P (count_exp) && desired_align > align)
24941 if (INTVAL (count_exp) > desired_align
24942 && INTVAL (count_exp) > size_needed)
24945 = get_mem_align_offset (dst, desired_align * BITS_PER_UNIT);
24946 if (align_bytes <= 0)
24949 align_bytes = desired_align - align_bytes;
24951 if (align_bytes == 0)
24952 count_exp = force_reg (counter_mode (count_exp), count_exp);
24954 gcc_assert (desired_align >= 1 && align >= 1);
24956 /* Misaligned move sequences handle both prologue and epilogue at once.
24957 Default code generation results in a smaller code for large alignments
24958 and also avoids redundant job when sizes are known precisely. */
24959 misaligned_prologue_used
24960 = (TARGET_MISALIGNED_MOVE_STRING_PRO_EPILOGUES
24961 && MAX (desired_align, epilogue_size_needed) <= 32
24962 && desired_align <= epilogue_size_needed
24963 && ((desired_align > align && !align_bytes)
24964 || (!count && epilogue_size_needed > 1)));
24966 /* Do the cheap promotion to allow better CSE across the
24967 main loop and epilogue (ie one load of the big constant in the
24969 For now the misaligned move sequences do not have fast path
24970 without broadcasting. */
24971 if (issetmem && ((CONST_INT_P (val_exp) || misaligned_prologue_used)))
24973 if (alg == vector_loop)
24975 gcc_assert (val_exp == const0_rtx);
24976 vec_promoted_val = promote_duplicated_reg (move_mode, val_exp);
24977 promoted_val = promote_duplicated_reg_to_size (val_exp,
24978 GET_MODE_SIZE (word_mode),
24979 desired_align, align);
24983 promoted_val = promote_duplicated_reg_to_size (val_exp, size_needed,
24984 desired_align, align);
24987 /* Misaligned move sequences handles both prologues and epilogues at once.
24988 Default code generation results in smaller code for large alignments and
24989 also avoids redundant job when sizes are known precisely. */
24990 if (misaligned_prologue_used)
24992 /* Misaligned move prologue handled small blocks by itself. */
24993 expand_set_or_movmem_prologue_epilogue_by_misaligned_moves
24994 (dst, src, &destreg, &srcreg,
24995 move_mode, promoted_val, vec_promoted_val,
24997 &jump_around_label,
24998 desired_align < align
24999 ? MAX (desired_align, epilogue_size_needed) : epilogue_size_needed,
25000 desired_align, align, &min_size, dynamic_check, issetmem);
25002 src = change_address (src, BLKmode, srcreg);
25003 dst = change_address (dst, BLKmode, destreg);
25004 set_mem_align (dst, desired_align * BITS_PER_UNIT);
25005 epilogue_size_needed = 0;
25006 if (need_zero_guard && !min_size)
25008 /* It is possible that we copied enough so the main loop will not
25010 gcc_assert (size_needed > 1);
25011 if (jump_around_label == NULL_RTX)
25012 jump_around_label = gen_label_rtx ();
25013 emit_cmp_and_jump_insns (count_exp,
25014 GEN_INT (size_needed),
25015 LTU, 0, counter_mode (count_exp), 1, jump_around_label);
25016 if (expected_size == -1
25017 || expected_size < (desired_align - align) / 2 + size_needed)
25018 predict_jump (REG_BR_PROB_BASE * 20 / 100);
25020 predict_jump (REG_BR_PROB_BASE * 60 / 100);
25023 /* Ensure that alignment prologue won't copy past end of block. */
25024 else if (size_needed > 1 || (desired_align > 1 && desired_align > align))
25026 epilogue_size_needed = MAX (size_needed - 1, desired_align - align);
25027 /* Epilogue always copies COUNT_EXP & EPILOGUE_SIZE_NEEDED bytes.
25028 Make sure it is power of 2. */
25029 epilogue_size_needed = 1 << (floor_log2 (epilogue_size_needed) + 1);
25031 /* To improve performance of small blocks, we jump around the VAL
25032 promoting mode. This mean that if the promoted VAL is not constant,
25033 we might not use it in the epilogue and have to use byte
25035 if (issetmem && epilogue_size_needed > 2 && !promoted_val)
25036 force_loopy_epilogue = true;
25037 if ((count && count < (unsigned HOST_WIDE_INT) epilogue_size_needed)
25038 || max_size < (unsigned HOST_WIDE_INT) epilogue_size_needed)
25040 /* If main algorithm works on QImode, no epilogue is needed.
25041 For small sizes just don't align anything. */
25042 if (size_needed == 1)
25043 desired_align = align;
25048 && min_size < (unsigned HOST_WIDE_INT) epilogue_size_needed)
25050 label = gen_label_rtx ();
25051 emit_cmp_and_jump_insns (count_exp,
25052 GEN_INT (epilogue_size_needed),
25053 LTU, 0, counter_mode (count_exp), 1, label);
25054 if (expected_size == -1 || expected_size < epilogue_size_needed)
25055 predict_jump (REG_BR_PROB_BASE * 60 / 100);
25057 predict_jump (REG_BR_PROB_BASE * 20 / 100);
25061 /* Emit code to decide on runtime whether library call or inline should be
25063 if (dynamic_check != -1)
25065 if (!issetmem && CONST_INT_P (count_exp))
25067 if (UINTVAL (count_exp) >= (unsigned HOST_WIDE_INT)dynamic_check)
25069 emit_block_move_via_libcall (dst, src, count_exp, false);
25070 count_exp = const0_rtx;
25076 rtx_code_label *hot_label = gen_label_rtx ();
25077 if (jump_around_label == NULL_RTX)
25078 jump_around_label = gen_label_rtx ();
25079 emit_cmp_and_jump_insns (count_exp, GEN_INT (dynamic_check - 1),
25080 LEU, 0, counter_mode (count_exp),
25082 predict_jump (REG_BR_PROB_BASE * 90 / 100);
25084 set_storage_via_libcall (dst, count_exp, val_exp, false);
25086 emit_block_move_via_libcall (dst, src, count_exp, false);
25087 emit_jump (jump_around_label);
25088 emit_label (hot_label);
25092 /* Step 2: Alignment prologue. */
25093 /* Do the expensive promotion once we branched off the small blocks. */
25094 if (issetmem && !promoted_val)
25095 promoted_val = promote_duplicated_reg_to_size (val_exp, size_needed,
25096 desired_align, align);
25098 if (desired_align > align && !misaligned_prologue_used)
25100 if (align_bytes == 0)
25102 /* Except for the first move in prologue, we no longer know
25103 constant offset in aliasing info. It don't seems to worth
25104 the pain to maintain it for the first move, so throw away
25106 dst = change_address (dst, BLKmode, destreg);
25108 src = change_address (src, BLKmode, srcreg);
25109 dst = expand_set_or_movmem_prologue (dst, src, destreg, srcreg,
25110 promoted_val, vec_promoted_val,
25111 count_exp, align, desired_align,
25113 /* At most desired_align - align bytes are copied. */
25114 if (min_size < (unsigned)(desired_align - align))
25117 min_size -= desired_align - align;
25121 /* If we know how many bytes need to be stored before dst is
25122 sufficiently aligned, maintain aliasing info accurately. */
25123 dst = expand_set_or_movmem_constant_prologue (dst, &src, destreg,
25131 count_exp = plus_constant (counter_mode (count_exp),
25132 count_exp, -align_bytes);
25133 count -= align_bytes;
25134 min_size -= align_bytes;
25135 max_size -= align_bytes;
25137 if (need_zero_guard
25139 && (count < (unsigned HOST_WIDE_INT) size_needed
25140 || (align_bytes == 0
25141 && count < ((unsigned HOST_WIDE_INT) size_needed
25142 + desired_align - align))))
25144 /* It is possible that we copied enough so the main loop will not
25146 gcc_assert (size_needed > 1);
25147 if (label == NULL_RTX)
25148 label = gen_label_rtx ();
25149 emit_cmp_and_jump_insns (count_exp,
25150 GEN_INT (size_needed),
25151 LTU, 0, counter_mode (count_exp), 1, label);
25152 if (expected_size == -1
25153 || expected_size < (desired_align - align) / 2 + size_needed)
25154 predict_jump (REG_BR_PROB_BASE * 20 / 100);
25156 predict_jump (REG_BR_PROB_BASE * 60 / 100);
25159 if (label && size_needed == 1)
25161 emit_label (label);
25162 LABEL_NUSES (label) = 1;
25164 epilogue_size_needed = 1;
25166 promoted_val = val_exp;
25168 else if (label == NULL_RTX && !misaligned_prologue_used)
25169 epilogue_size_needed = size_needed;
25171 /* Step 3: Main loop. */
25178 gcc_unreachable ();
25181 case unrolled_loop:
25182 expand_set_or_movmem_via_loop (dst, src, destreg, srcreg, promoted_val,
25183 count_exp, move_mode, unroll_factor,
25184 expected_size, issetmem);
25187 expand_set_or_movmem_via_loop (dst, src, destreg, srcreg,
25188 vec_promoted_val, count_exp, move_mode,
25189 unroll_factor, expected_size, issetmem);
25191 case rep_prefix_8_byte:
25192 case rep_prefix_4_byte:
25193 case rep_prefix_1_byte:
25194 expand_set_or_movmem_via_rep (dst, src, destreg, srcreg, promoted_val,
25195 val_exp, count_exp, move_mode, issetmem);
25198 /* Adjust properly the offset of src and dest memory for aliasing. */
25199 if (CONST_INT_P (count_exp))
25202 src = adjust_automodify_address_nv (src, BLKmode, srcreg,
25203 (count / size_needed) * size_needed);
25204 dst = adjust_automodify_address_nv (dst, BLKmode, destreg,
25205 (count / size_needed) * size_needed);
25210 src = change_address (src, BLKmode, srcreg);
25211 dst = change_address (dst, BLKmode, destreg);
25214 /* Step 4: Epilogue to copy the remaining bytes. */
25218 /* When the main loop is done, COUNT_EXP might hold original count,
25219 while we want to copy only COUNT_EXP & SIZE_NEEDED bytes.
25220 Epilogue code will actually copy COUNT_EXP & EPILOGUE_SIZE_NEEDED
25221 bytes. Compensate if needed. */
25223 if (size_needed < epilogue_size_needed)
25226 expand_simple_binop (counter_mode (count_exp), AND, count_exp,
25227 GEN_INT (size_needed - 1), count_exp, 1,
25229 if (tmp != count_exp)
25230 emit_move_insn (count_exp, tmp);
25232 emit_label (label);
25233 LABEL_NUSES (label) = 1;
25236 if (count_exp != const0_rtx && epilogue_size_needed > 1)
25238 if (force_loopy_epilogue)
25239 expand_setmem_epilogue_via_loop (dst, destreg, val_exp, count_exp,
25240 epilogue_size_needed);
25244 expand_setmem_epilogue (dst, destreg, promoted_val,
25245 vec_promoted_val, count_exp,
25246 epilogue_size_needed);
25248 expand_movmem_epilogue (dst, src, destreg, srcreg, count_exp,
25249 epilogue_size_needed);
25252 if (jump_around_label)
25253 emit_label (jump_around_label);
25258 /* Expand the appropriate insns for doing strlen if not just doing
25261 out = result, initialized with the start address
25262 align_rtx = alignment of the address.
25263 scratch = scratch register, initialized with the startaddress when
25264 not aligned, otherwise undefined
25266 This is just the body. It needs the initializations mentioned above and
25267 some address computing at the end. These things are done in i386.md. */
25270 ix86_expand_strlensi_unroll_1 (rtx out, rtx src, rtx align_rtx)
25274 rtx_code_label *align_2_label = NULL;
25275 rtx_code_label *align_3_label = NULL;
25276 rtx_code_label *align_4_label = gen_label_rtx ();
25277 rtx_code_label *end_0_label = gen_label_rtx ();
25279 rtx tmpreg = gen_reg_rtx (SImode);
25280 rtx scratch = gen_reg_rtx (SImode);
25284 if (CONST_INT_P (align_rtx))
25285 align = INTVAL (align_rtx);
25287 /* Loop to check 1..3 bytes for null to get an aligned pointer. */
25289 /* Is there a known alignment and is it less than 4? */
25292 rtx scratch1 = gen_reg_rtx (Pmode);
25293 emit_move_insn (scratch1, out);
25294 /* Is there a known alignment and is it not 2? */
25297 align_3_label = gen_label_rtx (); /* Label when aligned to 3-byte */
25298 align_2_label = gen_label_rtx (); /* Label when aligned to 2-byte */
25300 /* Leave just the 3 lower bits. */
25301 align_rtx = expand_binop (Pmode, and_optab, scratch1, GEN_INT (3),
25302 NULL_RTX, 0, OPTAB_WIDEN);
25304 emit_cmp_and_jump_insns (align_rtx, const0_rtx, EQ, NULL,
25305 Pmode, 1, align_4_label);
25306 emit_cmp_and_jump_insns (align_rtx, const2_rtx, EQ, NULL,
25307 Pmode, 1, align_2_label);
25308 emit_cmp_and_jump_insns (align_rtx, const2_rtx, GTU, NULL,
25309 Pmode, 1, align_3_label);
25313 /* Since the alignment is 2, we have to check 2 or 0 bytes;
25314 check if is aligned to 4 - byte. */
25316 align_rtx = expand_binop (Pmode, and_optab, scratch1, const2_rtx,
25317 NULL_RTX, 0, OPTAB_WIDEN);
25319 emit_cmp_and_jump_insns (align_rtx, const0_rtx, EQ, NULL,
25320 Pmode, 1, align_4_label);
25323 mem = change_address (src, QImode, out);
25325 /* Now compare the bytes. */
25327 /* Compare the first n unaligned byte on a byte per byte basis. */
25328 emit_cmp_and_jump_insns (mem, const0_rtx, EQ, NULL,
25329 QImode, 1, end_0_label);
25331 /* Increment the address. */
25332 emit_insn (ix86_gen_add3 (out, out, const1_rtx));
25334 /* Not needed with an alignment of 2 */
25337 emit_label (align_2_label);
25339 emit_cmp_and_jump_insns (mem, const0_rtx, EQ, NULL, QImode, 1,
25342 emit_insn (ix86_gen_add3 (out, out, const1_rtx));
25344 emit_label (align_3_label);
25347 emit_cmp_and_jump_insns (mem, const0_rtx, EQ, NULL, QImode, 1,
25350 emit_insn (ix86_gen_add3 (out, out, const1_rtx));
25353 /* Generate loop to check 4 bytes at a time. It is not a good idea to
25354 align this loop. It gives only huge programs, but does not help to
25356 emit_label (align_4_label);
25358 mem = change_address (src, SImode, out);
25359 emit_move_insn (scratch, mem);
25360 emit_insn (ix86_gen_add3 (out, out, GEN_INT (4)));
25362 /* This formula yields a nonzero result iff one of the bytes is zero.
25363 This saves three branches inside loop and many cycles. */
25365 emit_insn (gen_addsi3 (tmpreg, scratch, GEN_INT (-0x01010101)));
25366 emit_insn (gen_one_cmplsi2 (scratch, scratch));
25367 emit_insn (gen_andsi3 (tmpreg, tmpreg, scratch));
25368 emit_insn (gen_andsi3 (tmpreg, tmpreg,
25369 gen_int_mode (0x80808080, SImode)));
25370 emit_cmp_and_jump_insns (tmpreg, const0_rtx, EQ, 0, SImode, 1,
25375 rtx reg = gen_reg_rtx (SImode);
25376 rtx reg2 = gen_reg_rtx (Pmode);
25377 emit_move_insn (reg, tmpreg);
25378 emit_insn (gen_lshrsi3 (reg, reg, GEN_INT (16)));
25380 /* If zero is not in the first two bytes, move two bytes forward. */
25381 emit_insn (gen_testsi_ccno_1 (tmpreg, GEN_INT (0x8080)));
25382 tmp = gen_rtx_REG (CCNOmode, FLAGS_REG);
25383 tmp = gen_rtx_EQ (VOIDmode, tmp, const0_rtx);
25384 emit_insn (gen_rtx_SET (tmpreg,
25385 gen_rtx_IF_THEN_ELSE (SImode, tmp,
25388 /* Emit lea manually to avoid clobbering of flags. */
25389 emit_insn (gen_rtx_SET (reg2, gen_rtx_PLUS (Pmode, out, const2_rtx)));
25391 tmp = gen_rtx_REG (CCNOmode, FLAGS_REG);
25392 tmp = gen_rtx_EQ (VOIDmode, tmp, const0_rtx);
25393 emit_insn (gen_rtx_SET (out,
25394 gen_rtx_IF_THEN_ELSE (Pmode, tmp,
25400 rtx_code_label *end_2_label = gen_label_rtx ();
25401 /* Is zero in the first two bytes? */
25403 emit_insn (gen_testsi_ccno_1 (tmpreg, GEN_INT (0x8080)));
25404 tmp = gen_rtx_REG (CCNOmode, FLAGS_REG);
25405 tmp = gen_rtx_NE (VOIDmode, tmp, const0_rtx);
25406 tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
25407 gen_rtx_LABEL_REF (VOIDmode, end_2_label),
25409 tmp = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
25410 JUMP_LABEL (tmp) = end_2_label;
25412 /* Not in the first two. Move two bytes forward. */
25413 emit_insn (gen_lshrsi3 (tmpreg, tmpreg, GEN_INT (16)));
25414 emit_insn (ix86_gen_add3 (out, out, const2_rtx));
25416 emit_label (end_2_label);
25420 /* Avoid branch in fixing the byte. */
25421 tmpreg = gen_lowpart (QImode, tmpreg);
25422 emit_insn (gen_addqi3_cc (tmpreg, tmpreg, tmpreg));
25423 tmp = gen_rtx_REG (CCmode, FLAGS_REG);
25424 cmp = gen_rtx_LTU (VOIDmode, tmp, const0_rtx);
25425 emit_insn (ix86_gen_sub3_carry (out, out, GEN_INT (3), tmp, cmp));
25427 emit_label (end_0_label);
25430 /* Expand strlen. */
25433 ix86_expand_strlen (rtx out, rtx src, rtx eoschar, rtx align)
25435 rtx addr, scratch1, scratch2, scratch3, scratch4;
25437 /* The generic case of strlen expander is long. Avoid it's
25438 expanding unless TARGET_INLINE_ALL_STRINGOPS. */
25440 if (TARGET_UNROLL_STRLEN && eoschar == const0_rtx && optimize > 1
25441 && !TARGET_INLINE_ALL_STRINGOPS
25442 && !optimize_insn_for_size_p ()
25443 && (!CONST_INT_P (align) || INTVAL (align) < 4))
25446 addr = force_reg (Pmode, XEXP (src, 0));
25447 scratch1 = gen_reg_rtx (Pmode);
25449 if (TARGET_UNROLL_STRLEN && eoschar == const0_rtx && optimize > 1
25450 && !optimize_insn_for_size_p ())
25452 /* Well it seems that some optimizer does not combine a call like
25453 foo(strlen(bar), strlen(bar));
25454 when the move and the subtraction is done here. It does calculate
25455 the length just once when these instructions are done inside of
25456 output_strlen_unroll(). But I think since &bar[strlen(bar)] is
25457 often used and I use one fewer register for the lifetime of
25458 output_strlen_unroll() this is better. */
25460 emit_move_insn (out, addr);
25462 ix86_expand_strlensi_unroll_1 (out, src, align);
25464 /* strlensi_unroll_1 returns the address of the zero at the end of
25465 the string, like memchr(), so compute the length by subtracting
25466 the start address. */
25467 emit_insn (ix86_gen_sub3 (out, out, addr));
25473 /* Can't use this if the user has appropriated eax, ecx, or edi. */
25474 if (fixed_regs[AX_REG] || fixed_regs[CX_REG] || fixed_regs[DI_REG])
25477 scratch2 = gen_reg_rtx (Pmode);
25478 scratch3 = gen_reg_rtx (Pmode);
25479 scratch4 = force_reg (Pmode, constm1_rtx);
25481 emit_move_insn (scratch3, addr);
25482 eoschar = force_reg (QImode, eoschar);
25484 src = replace_equiv_address_nv (src, scratch3);
25486 /* If .md starts supporting :P, this can be done in .md. */
25487 unspec = gen_rtx_UNSPEC (Pmode, gen_rtvec (4, src, eoschar, align,
25488 scratch4), UNSPEC_SCAS);
25489 emit_insn (gen_strlenqi_1 (scratch1, scratch3, unspec));
25490 emit_insn (ix86_gen_one_cmpl2 (scratch2, scratch1));
25491 emit_insn (ix86_gen_add3 (out, scratch2, constm1_rtx));
25496 /* For given symbol (function) construct code to compute address of it's PLT
25497 entry in large x86-64 PIC model. */
25499 construct_plt_address (rtx symbol)
25503 gcc_assert (GET_CODE (symbol) == SYMBOL_REF);
25504 gcc_assert (ix86_cmodel == CM_LARGE_PIC && !TARGET_PECOFF);
25505 gcc_assert (Pmode == DImode);
25507 tmp = gen_reg_rtx (Pmode);
25508 unspec = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, symbol), UNSPEC_PLTOFF);
25510 emit_move_insn (tmp, gen_rtx_CONST (Pmode, unspec));
25511 emit_insn (ix86_gen_add3 (tmp, tmp, pic_offset_table_rtx));
25516 ix86_expand_call (rtx retval, rtx fnaddr, rtx callarg1,
25518 rtx pop, bool sibcall)
25521 rtx use = NULL, call;
25522 unsigned int vec_len = 0;
25524 if (pop == const0_rtx)
25526 gcc_assert (!TARGET_64BIT || !pop);
25528 if (TARGET_MACHO && !TARGET_64BIT)
25531 if (flag_pic && GET_CODE (XEXP (fnaddr, 0)) == SYMBOL_REF)
25532 fnaddr = machopic_indirect_call_target (fnaddr);
25537 /* Static functions and indirect calls don't need the pic register. Also,
25538 check if PLT was explicitly avoided via no-plt or "noplt" attribute, making
25539 it an indirect call. */
25542 || (ix86_cmodel == CM_LARGE_PIC
25543 && DEFAULT_ABI != MS_ABI))
25544 && GET_CODE (XEXP (fnaddr, 0)) == SYMBOL_REF
25545 && !SYMBOL_REF_LOCAL_P (XEXP (fnaddr, 0))
25547 && (SYMBOL_REF_DECL ((XEXP (fnaddr, 0))) == NULL_TREE
25548 || !lookup_attribute ("noplt",
25549 DECL_ATTRIBUTES (SYMBOL_REF_DECL (XEXP (fnaddr, 0))))))
25551 use_reg (&use, gen_rtx_REG (Pmode, REAL_PIC_OFFSET_TABLE_REGNUM));
25552 if (ix86_use_pseudo_pic_reg ())
25553 emit_move_insn (gen_rtx_REG (Pmode, REAL_PIC_OFFSET_TABLE_REGNUM),
25554 pic_offset_table_rtx);
25558 /* Skip setting up RAX register for -mskip-rax-setup when there are no
25559 parameters passed in vector registers. */
25561 && (INTVAL (callarg2) > 0
25562 || (INTVAL (callarg2) == 0
25563 && (TARGET_SSE || !flag_skip_rax_setup))))
25565 rtx al = gen_rtx_REG (QImode, AX_REG);
25566 emit_move_insn (al, callarg2);
25567 use_reg (&use, al);
25570 if (ix86_cmodel == CM_LARGE_PIC
25573 && GET_CODE (XEXP (fnaddr, 0)) == SYMBOL_REF
25574 && !local_symbolic_operand (XEXP (fnaddr, 0), VOIDmode))
25575 fnaddr = gen_rtx_MEM (QImode, construct_plt_address (XEXP (fnaddr, 0)));
25577 ? !sibcall_insn_operand (XEXP (fnaddr, 0), word_mode)
25578 : !call_insn_operand (XEXP (fnaddr, 0), word_mode))
25580 fnaddr = convert_to_mode (word_mode, XEXP (fnaddr, 0), 1);
25581 fnaddr = gen_rtx_MEM (QImode, copy_to_mode_reg (word_mode, fnaddr));
25584 call = gen_rtx_CALL (VOIDmode, fnaddr, callarg1);
25588 /* We should add bounds as destination register in case
25589 pointer with bounds may be returned. */
25590 if (TARGET_MPX && SCALAR_INT_MODE_P (GET_MODE (retval)))
25592 rtx b0 = gen_rtx_REG (BND64mode, FIRST_BND_REG);
25593 rtx b1 = gen_rtx_REG (BND64mode, FIRST_BND_REG + 1);
25594 if (GET_CODE (retval) == PARALLEL)
25596 b0 = gen_rtx_EXPR_LIST (VOIDmode, b0, const0_rtx);
25597 b1 = gen_rtx_EXPR_LIST (VOIDmode, b1, const0_rtx);
25598 rtx par = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, b0, b1));
25599 retval = chkp_join_splitted_slot (retval, par);
25603 retval = gen_rtx_PARALLEL (VOIDmode,
25604 gen_rtvec (3, retval, b0, b1));
25605 chkp_put_regs_to_expr_list (retval);
25609 call = gen_rtx_SET (retval, call);
25611 vec[vec_len++] = call;
25615 pop = gen_rtx_PLUS (Pmode, stack_pointer_rtx, pop);
25616 pop = gen_rtx_SET (stack_pointer_rtx, pop);
25617 vec[vec_len++] = pop;
25620 if (TARGET_64BIT_MS_ABI
25621 && (!callarg2 || INTVAL (callarg2) != -2))
25623 int const cregs_size
25624 = ARRAY_SIZE (x86_64_ms_sysv_extra_clobbered_registers);
25627 for (i = 0; i < cregs_size; i++)
25629 int regno = x86_64_ms_sysv_extra_clobbered_registers[i];
25630 machine_mode mode = SSE_REGNO_P (regno) ? TImode : DImode;
25632 clobber_reg (&use, gen_rtx_REG (mode, regno));
25637 call = gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (vec_len, vec));
25638 call = emit_call_insn (call);
25640 CALL_INSN_FUNCTION_USAGE (call) = use;
25645 /* Return true if the function being called was marked with attribute "noplt"
25646 or using -fno-plt and we are compiling for non-PIC and x86_64. We need to
25647 handle the non-PIC case in the backend because there is no easy interface
25648 for the front-end to force non-PLT calls to use the GOT. This is currently
25649 used only with 64-bit ELF targets to call the function marked "noplt"
25653 ix86_nopic_noplt_attribute_p (rtx call_op)
25655 if (flag_pic || ix86_cmodel == CM_LARGE
25656 || !TARGET_64BIT || TARGET_MACHO || TARGET_SEH || TARGET_PECOFF
25657 || SYMBOL_REF_LOCAL_P (call_op))
25660 tree symbol_decl = SYMBOL_REF_DECL (call_op);
25663 || (symbol_decl != NULL_TREE
25664 && lookup_attribute ("noplt", DECL_ATTRIBUTES (symbol_decl))))
25670 /* Output the assembly for a call instruction. */
25673 ix86_output_call_insn (rtx_insn *insn, rtx call_op)
25675 bool direct_p = constant_call_address_operand (call_op, VOIDmode);
25676 bool seh_nop_p = false;
25679 if (SIBLING_CALL_P (insn))
25681 if (direct_p && ix86_nopic_noplt_attribute_p (call_op))
25682 xasm = "%!jmp\t*%p0@GOTPCREL(%%rip)";
25684 xasm = "%!jmp\t%P0";
25685 /* SEH epilogue detection requires the indirect branch case
25686 to include REX.W. */
25687 else if (TARGET_SEH)
25688 xasm = "%!rex.W jmp %A0";
25690 xasm = "%!jmp\t%A0";
25692 output_asm_insn (xasm, &call_op);
25696 /* SEH unwinding can require an extra nop to be emitted in several
25697 circumstances. Determine if we have one of those. */
25702 for (i = NEXT_INSN (insn); i ; i = NEXT_INSN (i))
25704 /* If we get to another real insn, we don't need the nop. */
25708 /* If we get to the epilogue note, prevent a catch region from
25709 being adjacent to the standard epilogue sequence. If non-
25710 call-exceptions, we'll have done this during epilogue emission. */
25711 if (NOTE_P (i) && NOTE_KIND (i) == NOTE_INSN_EPILOGUE_BEG
25712 && !flag_non_call_exceptions
25713 && !can_throw_internal (insn))
25720 /* If we didn't find a real insn following the call, prevent the
25721 unwinder from looking into the next function. */
25726 if (direct_p && ix86_nopic_noplt_attribute_p (call_op))
25727 xasm = "%!call\t*%p0@GOTPCREL(%%rip)";
25729 xasm = "%!call\t%P0";
25731 xasm = "%!call\t%A0";
25733 output_asm_insn (xasm, &call_op);
25741 /* Clear stack slot assignments remembered from previous functions.
25742 This is called from INIT_EXPANDERS once before RTL is emitted for each
25745 static struct machine_function *
25746 ix86_init_machine_status (void)
25748 struct machine_function *f;
25750 f = ggc_cleared_alloc<machine_function> ();
25751 f->use_fast_prologue_epilogue_nregs = -1;
25752 f->call_abi = ix86_abi;
25757 /* Return a MEM corresponding to a stack slot with mode MODE.
25758 Allocate a new slot if necessary.
25760 The RTL for a function can have several slots available: N is
25761 which slot to use. */
25764 assign_386_stack_local (machine_mode mode, enum ix86_stack_slot n)
25766 struct stack_local_entry *s;
25768 gcc_assert (n < MAX_386_STACK_LOCALS);
25770 for (s = ix86_stack_locals; s; s = s->next)
25771 if (s->mode == mode && s->n == n)
25772 return validize_mem (copy_rtx (s->rtl));
25774 s = ggc_alloc<stack_local_entry> ();
25777 s->rtl = assign_stack_local (mode, GET_MODE_SIZE (mode), 0);
25779 s->next = ix86_stack_locals;
25780 ix86_stack_locals = s;
25781 return validize_mem (copy_rtx (s->rtl));
25785 ix86_instantiate_decls (void)
25787 struct stack_local_entry *s;
25789 for (s = ix86_stack_locals; s; s = s->next)
25790 if (s->rtl != NULL_RTX)
25791 instantiate_decl_rtl (s->rtl);
25794 /* Check whether x86 address PARTS is a pc-relative address. */
25797 rip_relative_addr_p (struct ix86_address *parts)
25799 rtx base, index, disp;
25801 base = parts->base;
25802 index = parts->index;
25803 disp = parts->disp;
25805 if (disp && !base && !index)
25811 if (GET_CODE (disp) == CONST)
25812 symbol = XEXP (disp, 0);
25813 if (GET_CODE (symbol) == PLUS
25814 && CONST_INT_P (XEXP (symbol, 1)))
25815 symbol = XEXP (symbol, 0);
25817 if (GET_CODE (symbol) == LABEL_REF
25818 || (GET_CODE (symbol) == SYMBOL_REF
25819 && SYMBOL_REF_TLS_MODEL (symbol) == 0)
25820 || (GET_CODE (symbol) == UNSPEC
25821 && (XINT (symbol, 1) == UNSPEC_GOTPCREL
25822 || XINT (symbol, 1) == UNSPEC_PCREL
25823 || XINT (symbol, 1) == UNSPEC_GOTNTPOFF)))
25830 /* Calculate the length of the memory address in the instruction encoding.
25831 Includes addr32 prefix, does not include the one-byte modrm, opcode,
25832 or other prefixes. We never generate addr32 prefix for LEA insn. */
25835 memory_address_length (rtx addr, bool lea)
25837 struct ix86_address parts;
25838 rtx base, index, disp;
25842 if (GET_CODE (addr) == PRE_DEC
25843 || GET_CODE (addr) == POST_INC
25844 || GET_CODE (addr) == PRE_MODIFY
25845 || GET_CODE (addr) == POST_MODIFY)
25848 ok = ix86_decompose_address (addr, &parts);
25851 len = (parts.seg == SEG_DEFAULT) ? 0 : 1;
25853 /* If this is not LEA instruction, add the length of addr32 prefix. */
25854 if (TARGET_64BIT && !lea
25855 && (SImode_address_operand (addr, VOIDmode)
25856 || (parts.base && GET_MODE (parts.base) == SImode)
25857 || (parts.index && GET_MODE (parts.index) == SImode)))
25861 index = parts.index;
25864 if (base && GET_CODE (base) == SUBREG)
25865 base = SUBREG_REG (base);
25866 if (index && GET_CODE (index) == SUBREG)
25867 index = SUBREG_REG (index);
25869 gcc_assert (base == NULL_RTX || REG_P (base));
25870 gcc_assert (index == NULL_RTX || REG_P (index));
25873 - esp as the base always wants an index,
25874 - ebp as the base always wants a displacement,
25875 - r12 as the base always wants an index,
25876 - r13 as the base always wants a displacement. */
25878 /* Register Indirect. */
25879 if (base && !index && !disp)
25881 /* esp (for its index) and ebp (for its displacement) need
25882 the two-byte modrm form. Similarly for r12 and r13 in 64-bit
25884 if (base == arg_pointer_rtx
25885 || base == frame_pointer_rtx
25886 || REGNO (base) == SP_REG
25887 || REGNO (base) == BP_REG
25888 || REGNO (base) == R12_REG
25889 || REGNO (base) == R13_REG)
25893 /* Direct Addressing. In 64-bit mode mod 00 r/m 5
25894 is not disp32, but disp32(%rip), so for disp32
25895 SIB byte is needed, unless print_operand_address
25896 optimizes it into disp32(%rip) or (%rip) is implied
25898 else if (disp && !base && !index)
25901 if (rip_relative_addr_p (&parts))
25906 /* Find the length of the displacement constant. */
25909 if (base && satisfies_constraint_K (disp))
25914 /* ebp always wants a displacement. Similarly r13. */
25915 else if (base && (REGNO (base) == BP_REG || REGNO (base) == R13_REG))
25918 /* An index requires the two-byte modrm form.... */
25920 /* ...like esp (or r12), which always wants an index. */
25921 || base == arg_pointer_rtx
25922 || base == frame_pointer_rtx
25923 || (base && (REGNO (base) == SP_REG || REGNO (base) == R12_REG)))
25930 /* Compute default value for "length_immediate" attribute. When SHORTFORM
25931 is set, expect that insn have 8bit immediate alternative. */
25933 ix86_attr_length_immediate_default (rtx_insn *insn, bool shortform)
25937 extract_insn_cached (insn);
25938 for (i = recog_data.n_operands - 1; i >= 0; --i)
25939 if (CONSTANT_P (recog_data.operand[i]))
25941 enum attr_mode mode = get_attr_mode (insn);
25944 if (shortform && CONST_INT_P (recog_data.operand[i]))
25946 HOST_WIDE_INT ival = INTVAL (recog_data.operand[i]);
25953 ival = trunc_int_for_mode (ival, HImode);
25956 ival = trunc_int_for_mode (ival, SImode);
25961 if (IN_RANGE (ival, -128, 127))
25978 /* Immediates for DImode instructions are encoded
25979 as 32bit sign extended values. */
25984 fatal_insn ("unknown insn mode", insn);
25990 /* Compute default value for "length_address" attribute. */
25992 ix86_attr_length_address_default (rtx_insn *insn)
25996 if (get_attr_type (insn) == TYPE_LEA)
25998 rtx set = PATTERN (insn), addr;
26000 if (GET_CODE (set) == PARALLEL)
26001 set = XVECEXP (set, 0, 0);
26003 gcc_assert (GET_CODE (set) == SET);
26005 addr = SET_SRC (set);
26007 return memory_address_length (addr, true);
26010 extract_insn_cached (insn);
26011 for (i = recog_data.n_operands - 1; i >= 0; --i)
26012 if (MEM_P (recog_data.operand[i]))
26014 constrain_operands_cached (insn, reload_completed);
26015 if (which_alternative != -1)
26017 const char *constraints = recog_data.constraints[i];
26018 int alt = which_alternative;
26020 while (*constraints == '=' || *constraints == '+')
26023 while (*constraints++ != ',')
26025 /* Skip ignored operands. */
26026 if (*constraints == 'X')
26029 return memory_address_length (XEXP (recog_data.operand[i], 0), false);
26034 /* Compute default value for "length_vex" attribute. It includes
26035 2 or 3 byte VEX prefix and 1 opcode byte. */
26038 ix86_attr_length_vex_default (rtx_insn *insn, bool has_0f_opcode,
26043 /* Only 0f opcode can use 2 byte VEX prefix and VEX W bit uses 3
26044 byte VEX prefix. */
26045 if (!has_0f_opcode || has_vex_w)
26048 /* We can always use 2 byte VEX prefix in 32bit. */
26052 extract_insn_cached (insn);
26054 for (i = recog_data.n_operands - 1; i >= 0; --i)
26055 if (REG_P (recog_data.operand[i]))
26057 /* REX.W bit uses 3 byte VEX prefix. */
26058 if (GET_MODE (recog_data.operand[i]) == DImode
26059 && GENERAL_REG_P (recog_data.operand[i]))
26064 /* REX.X or REX.B bits use 3 byte VEX prefix. */
26065 if (MEM_P (recog_data.operand[i])
26066 && x86_extended_reg_mentioned_p (recog_data.operand[i]))
26073 /* Return the maximum number of instructions a cpu can issue. */
26076 ix86_issue_rate (void)
26080 case PROCESSOR_PENTIUM:
26081 case PROCESSOR_BONNELL:
26082 case PROCESSOR_SILVERMONT:
26083 case PROCESSOR_KNL:
26084 case PROCESSOR_INTEL:
26086 case PROCESSOR_BTVER2:
26087 case PROCESSOR_PENTIUM4:
26088 case PROCESSOR_NOCONA:
26091 case PROCESSOR_PENTIUMPRO:
26092 case PROCESSOR_ATHLON:
26094 case PROCESSOR_AMDFAM10:
26095 case PROCESSOR_GENERIC:
26096 case PROCESSOR_BTVER1:
26099 case PROCESSOR_BDVER1:
26100 case PROCESSOR_BDVER2:
26101 case PROCESSOR_BDVER3:
26102 case PROCESSOR_BDVER4:
26103 case PROCESSOR_CORE2:
26104 case PROCESSOR_NEHALEM:
26105 case PROCESSOR_SANDYBRIDGE:
26106 case PROCESSOR_HASWELL:
26114 /* A subroutine of ix86_adjust_cost -- return TRUE iff INSN reads flags set
26115 by DEP_INSN and nothing set by DEP_INSN. */
26118 ix86_flags_dependent (rtx_insn *insn, rtx_insn *dep_insn, enum attr_type insn_type)
26122 /* Simplify the test for uninteresting insns. */
26123 if (insn_type != TYPE_SETCC
26124 && insn_type != TYPE_ICMOV
26125 && insn_type != TYPE_FCMOV
26126 && insn_type != TYPE_IBR)
26129 if ((set = single_set (dep_insn)) != 0)
26131 set = SET_DEST (set);
26134 else if (GET_CODE (PATTERN (dep_insn)) == PARALLEL
26135 && XVECLEN (PATTERN (dep_insn), 0) == 2
26136 && GET_CODE (XVECEXP (PATTERN (dep_insn), 0, 0)) == SET
26137 && GET_CODE (XVECEXP (PATTERN (dep_insn), 0, 1)) == SET)
26139 set = SET_DEST (XVECEXP (PATTERN (dep_insn), 0, 0));
26140 set2 = SET_DEST (XVECEXP (PATTERN (dep_insn), 0, 0));
26145 if (!REG_P (set) || REGNO (set) != FLAGS_REG)
26148 /* This test is true if the dependent insn reads the flags but
26149 not any other potentially set register. */
26150 if (!reg_overlap_mentioned_p (set, PATTERN (insn)))
26153 if (set2 && reg_overlap_mentioned_p (set2, PATTERN (insn)))
26159 /* Return true iff USE_INSN has a memory address with operands set by
26163 ix86_agi_dependent (rtx_insn *set_insn, rtx_insn *use_insn)
26166 extract_insn_cached (use_insn);
26167 for (i = recog_data.n_operands - 1; i >= 0; --i)
26168 if (MEM_P (recog_data.operand[i]))
26170 rtx addr = XEXP (recog_data.operand[i], 0);
26171 return modified_in_p (addr, set_insn) != 0;
26176 /* Helper function for exact_store_load_dependency.
26177 Return true if addr is found in insn. */
26179 exact_dependency_1 (rtx addr, rtx insn)
26181 enum rtx_code code;
26182 const char *format_ptr;
26185 code = GET_CODE (insn);
26189 if (rtx_equal_p (addr, insn))
26204 format_ptr = GET_RTX_FORMAT (code);
26205 for (i = 0; i < GET_RTX_LENGTH (code); i++)
26207 switch (*format_ptr++)
26210 if (exact_dependency_1 (addr, XEXP (insn, i)))
26214 for (j = 0; j < XVECLEN (insn, i); j++)
26215 if (exact_dependency_1 (addr, XVECEXP (insn, i, j)))
26223 /* Return true if there exists exact dependency for store & load, i.e.
26224 the same memory address is used in them. */
26226 exact_store_load_dependency (rtx_insn *store, rtx_insn *load)
26230 set1 = single_set (store);
26233 if (!MEM_P (SET_DEST (set1)))
26235 set2 = single_set (load);
26238 if (exact_dependency_1 (SET_DEST (set1), SET_SRC (set2)))
26244 ix86_adjust_cost (rtx_insn *insn, rtx link, rtx_insn *dep_insn, int cost)
26246 enum attr_type insn_type, dep_insn_type;
26247 enum attr_memory memory;
26249 int dep_insn_code_number;
26251 /* Anti and output dependencies have zero cost on all CPUs. */
26252 if (REG_NOTE_KIND (link) != 0)
26255 dep_insn_code_number = recog_memoized (dep_insn);
26257 /* If we can't recognize the insns, we can't really do anything. */
26258 if (dep_insn_code_number < 0 || recog_memoized (insn) < 0)
26261 insn_type = get_attr_type (insn);
26262 dep_insn_type = get_attr_type (dep_insn);
26266 case PROCESSOR_PENTIUM:
26267 /* Address Generation Interlock adds a cycle of latency. */
26268 if (insn_type == TYPE_LEA)
26270 rtx addr = PATTERN (insn);
26272 if (GET_CODE (addr) == PARALLEL)
26273 addr = XVECEXP (addr, 0, 0);
26275 gcc_assert (GET_CODE (addr) == SET);
26277 addr = SET_SRC (addr);
26278 if (modified_in_p (addr, dep_insn))
26281 else if (ix86_agi_dependent (dep_insn, insn))
26284 /* ??? Compares pair with jump/setcc. */
26285 if (ix86_flags_dependent (insn, dep_insn, insn_type))
26288 /* Floating point stores require value to be ready one cycle earlier. */
26289 if (insn_type == TYPE_FMOV
26290 && get_attr_memory (insn) == MEMORY_STORE
26291 && !ix86_agi_dependent (dep_insn, insn))
26295 case PROCESSOR_PENTIUMPRO:
26296 /* INT->FP conversion is expensive. */
26297 if (get_attr_fp_int_src (dep_insn))
26300 /* There is one cycle extra latency between an FP op and a store. */
26301 if (insn_type == TYPE_FMOV
26302 && (set = single_set (dep_insn)) != NULL_RTX
26303 && (set2 = single_set (insn)) != NULL_RTX
26304 && rtx_equal_p (SET_DEST (set), SET_SRC (set2))
26305 && MEM_P (SET_DEST (set2)))
26308 memory = get_attr_memory (insn);
26310 /* Show ability of reorder buffer to hide latency of load by executing
26311 in parallel with previous instruction in case
26312 previous instruction is not needed to compute the address. */
26313 if ((memory == MEMORY_LOAD || memory == MEMORY_BOTH)
26314 && !ix86_agi_dependent (dep_insn, insn))
26316 /* Claim moves to take one cycle, as core can issue one load
26317 at time and the next load can start cycle later. */
26318 if (dep_insn_type == TYPE_IMOV
26319 || dep_insn_type == TYPE_FMOV)
26327 /* The esp dependency is resolved before
26328 the instruction is really finished. */
26329 if ((insn_type == TYPE_PUSH || insn_type == TYPE_POP)
26330 && (dep_insn_type == TYPE_PUSH || dep_insn_type == TYPE_POP))
26333 /* INT->FP conversion is expensive. */
26334 if (get_attr_fp_int_src (dep_insn))
26337 memory = get_attr_memory (insn);
26339 /* Show ability of reorder buffer to hide latency of load by executing
26340 in parallel with previous instruction in case
26341 previous instruction is not needed to compute the address. */
26342 if ((memory == MEMORY_LOAD || memory == MEMORY_BOTH)
26343 && !ix86_agi_dependent (dep_insn, insn))
26345 /* Claim moves to take one cycle, as core can issue one load
26346 at time and the next load can start cycle later. */
26347 if (dep_insn_type == TYPE_IMOV
26348 || dep_insn_type == TYPE_FMOV)
26357 case PROCESSOR_AMDFAM10:
26358 case PROCESSOR_BDVER1:
26359 case PROCESSOR_BDVER2:
26360 case PROCESSOR_BDVER3:
26361 case PROCESSOR_BDVER4:
26362 case PROCESSOR_BTVER1:
26363 case PROCESSOR_BTVER2:
26364 case PROCESSOR_GENERIC:
26365 /* Stack engine allows to execute push&pop instructions in parall. */
26366 if ((insn_type == TYPE_PUSH || insn_type == TYPE_POP)
26367 && (dep_insn_type == TYPE_PUSH || dep_insn_type == TYPE_POP))
26371 case PROCESSOR_ATHLON:
26373 memory = get_attr_memory (insn);
26375 /* Show ability of reorder buffer to hide latency of load by executing
26376 in parallel with previous instruction in case
26377 previous instruction is not needed to compute the address. */
26378 if ((memory == MEMORY_LOAD || memory == MEMORY_BOTH)
26379 && !ix86_agi_dependent (dep_insn, insn))
26381 enum attr_unit unit = get_attr_unit (insn);
26384 /* Because of the difference between the length of integer and
26385 floating unit pipeline preparation stages, the memory operands
26386 for floating point are cheaper.
26388 ??? For Athlon it the difference is most probably 2. */
26389 if (unit == UNIT_INTEGER || unit == UNIT_UNKNOWN)
26392 loadcost = TARGET_ATHLON ? 2 : 0;
26394 if (cost >= loadcost)
26401 case PROCESSOR_CORE2:
26402 case PROCESSOR_NEHALEM:
26403 case PROCESSOR_SANDYBRIDGE:
26404 case PROCESSOR_HASWELL:
26405 /* Stack engine allows to execute push&pop instructions in parall. */
26406 if ((insn_type == TYPE_PUSH || insn_type == TYPE_POP)
26407 && (dep_insn_type == TYPE_PUSH || dep_insn_type == TYPE_POP))
26410 memory = get_attr_memory (insn);
26412 /* Show ability of reorder buffer to hide latency of load by executing
26413 in parallel with previous instruction in case
26414 previous instruction is not needed to compute the address. */
26415 if ((memory == MEMORY_LOAD || memory == MEMORY_BOTH)
26416 && !ix86_agi_dependent (dep_insn, insn))
26425 case PROCESSOR_SILVERMONT:
26426 case PROCESSOR_KNL:
26427 case PROCESSOR_INTEL:
26428 if (!reload_completed)
26431 /* Increase cost of integer loads. */
26432 memory = get_attr_memory (dep_insn);
26433 if (memory == MEMORY_LOAD || memory == MEMORY_BOTH)
26435 enum attr_unit unit = get_attr_unit (dep_insn);
26436 if (unit == UNIT_INTEGER && cost == 1)
26438 if (memory == MEMORY_LOAD)
26442 /* Increase cost of ld/st for short int types only
26443 because of store forwarding issue. */
26444 rtx set = single_set (dep_insn);
26445 if (set && (GET_MODE (SET_DEST (set)) == QImode
26446 || GET_MODE (SET_DEST (set)) == HImode))
26448 /* Increase cost of store/load insn if exact
26449 dependence exists and it is load insn. */
26450 enum attr_memory insn_memory = get_attr_memory (insn);
26451 if (insn_memory == MEMORY_LOAD
26452 && exact_store_load_dependency (dep_insn, insn))
26466 /* How many alternative schedules to try. This should be as wide as the
26467 scheduling freedom in the DFA, but no wider. Making this value too
26468 large results extra work for the scheduler. */
26471 ia32_multipass_dfa_lookahead (void)
26475 case PROCESSOR_PENTIUM:
26478 case PROCESSOR_PENTIUMPRO:
26482 case PROCESSOR_BDVER1:
26483 case PROCESSOR_BDVER2:
26484 case PROCESSOR_BDVER3:
26485 case PROCESSOR_BDVER4:
26486 /* We use lookahead value 4 for BD both before and after reload
26487 schedules. Plan is to have value 8 included for O3. */
26490 case PROCESSOR_CORE2:
26491 case PROCESSOR_NEHALEM:
26492 case PROCESSOR_SANDYBRIDGE:
26493 case PROCESSOR_HASWELL:
26494 case PROCESSOR_BONNELL:
26495 case PROCESSOR_SILVERMONT:
26496 case PROCESSOR_KNL:
26497 case PROCESSOR_INTEL:
26498 /* Generally, we want haifa-sched:max_issue() to look ahead as far
26499 as many instructions can be executed on a cycle, i.e.,
26500 issue_rate. I wonder why tuning for many CPUs does not do this. */
26501 if (reload_completed)
26502 return ix86_issue_rate ();
26503 /* Don't use lookahead for pre-reload schedule to save compile time. */
26511 /* Return true if target platform supports macro-fusion. */
26514 ix86_macro_fusion_p ()
26516 return TARGET_FUSE_CMP_AND_BRANCH;
26519 /* Check whether current microarchitecture support macro fusion
26520 for insn pair "CONDGEN + CONDJMP". Refer to
26521 "Intel Architectures Optimization Reference Manual". */
26524 ix86_macro_fusion_pair_p (rtx_insn *condgen, rtx_insn *condjmp)
26527 enum rtx_code ccode;
26528 rtx compare_set = NULL_RTX, test_if, cond;
26529 rtx alu_set = NULL_RTX, addr = NULL_RTX;
26531 if (!any_condjump_p (condjmp))
26534 if (get_attr_type (condgen) != TYPE_TEST
26535 && get_attr_type (condgen) != TYPE_ICMP
26536 && get_attr_type (condgen) != TYPE_INCDEC
26537 && get_attr_type (condgen) != TYPE_ALU)
26540 compare_set = single_set (condgen);
26541 if (compare_set == NULL_RTX
26542 && !TARGET_FUSE_ALU_AND_BRANCH)
26545 if (compare_set == NULL_RTX)
26548 rtx pat = PATTERN (condgen);
26549 for (i = 0; i < XVECLEN (pat, 0); i++)
26550 if (GET_CODE (XVECEXP (pat, 0, i)) == SET)
26552 rtx set_src = SET_SRC (XVECEXP (pat, 0, i));
26553 if (GET_CODE (set_src) == COMPARE)
26554 compare_set = XVECEXP (pat, 0, i);
26556 alu_set = XVECEXP (pat, 0, i);
26559 if (compare_set == NULL_RTX)
26561 src = SET_SRC (compare_set);
26562 if (GET_CODE (src) != COMPARE)
26565 /* Macro-fusion for cmp/test MEM-IMM + conditional jmp is not
26567 if ((MEM_P (XEXP (src, 0))
26568 && CONST_INT_P (XEXP (src, 1)))
26569 || (MEM_P (XEXP (src, 1))
26570 && CONST_INT_P (XEXP (src, 0))))
26573 /* No fusion for RIP-relative address. */
26574 if (MEM_P (XEXP (src, 0)))
26575 addr = XEXP (XEXP (src, 0), 0);
26576 else if (MEM_P (XEXP (src, 1)))
26577 addr = XEXP (XEXP (src, 1), 0);
26580 ix86_address parts;
26581 int ok = ix86_decompose_address (addr, &parts);
26584 if (rip_relative_addr_p (&parts))
26588 test_if = SET_SRC (pc_set (condjmp));
26589 cond = XEXP (test_if, 0);
26590 ccode = GET_CODE (cond);
26591 /* Check whether conditional jump use Sign or Overflow Flags. */
26592 if (!TARGET_FUSE_CMP_AND_BRANCH_SOFLAGS
26599 /* Return true for TYPE_TEST and TYPE_ICMP. */
26600 if (get_attr_type (condgen) == TYPE_TEST
26601 || get_attr_type (condgen) == TYPE_ICMP)
26604 /* The following is the case that macro-fusion for alu + jmp. */
26605 if (!TARGET_FUSE_ALU_AND_BRANCH || !alu_set)
26608 /* No fusion for alu op with memory destination operand. */
26609 dest = SET_DEST (alu_set);
26613 /* Macro-fusion for inc/dec + unsigned conditional jump is not
26615 if (get_attr_type (condgen) == TYPE_INCDEC
26625 /* Try to reorder ready list to take advantage of Atom pipelined IMUL
26626 execution. It is applied if
26627 (1) IMUL instruction is on the top of list;
26628 (2) There exists the only producer of independent IMUL instruction in
26630 Return index of IMUL producer if it was found and -1 otherwise. */
26632 do_reorder_for_imul (rtx_insn **ready, int n_ready)
26635 rtx set, insn1, insn2;
26636 sd_iterator_def sd_it;
26641 if (!TARGET_BONNELL)
26644 /* Check that IMUL instruction is on the top of ready list. */
26645 insn = ready[n_ready - 1];
26646 set = single_set (insn);
26649 if (!(GET_CODE (SET_SRC (set)) == MULT
26650 && GET_MODE (SET_SRC (set)) == SImode))
26653 /* Search for producer of independent IMUL instruction. */
26654 for (i = n_ready - 2; i >= 0; i--)
26657 if (!NONDEBUG_INSN_P (insn))
26659 /* Skip IMUL instruction. */
26660 insn2 = PATTERN (insn);
26661 if (GET_CODE (insn2) == PARALLEL)
26662 insn2 = XVECEXP (insn2, 0, 0);
26663 if (GET_CODE (insn2) == SET
26664 && GET_CODE (SET_SRC (insn2)) == MULT
26665 && GET_MODE (SET_SRC (insn2)) == SImode)
26668 FOR_EACH_DEP (insn, SD_LIST_FORW, sd_it, dep)
26671 con = DEP_CON (dep);
26672 if (!NONDEBUG_INSN_P (con))
26674 insn1 = PATTERN (con);
26675 if (GET_CODE (insn1) == PARALLEL)
26676 insn1 = XVECEXP (insn1, 0, 0);
26678 if (GET_CODE (insn1) == SET
26679 && GET_CODE (SET_SRC (insn1)) == MULT
26680 && GET_MODE (SET_SRC (insn1)) == SImode)
26682 sd_iterator_def sd_it1;
26684 /* Check if there is no other dependee for IMUL. */
26686 FOR_EACH_DEP (con, SD_LIST_BACK, sd_it1, dep1)
26689 pro = DEP_PRO (dep1);
26690 if (!NONDEBUG_INSN_P (pro))
26705 /* Try to find the best candidate on the top of ready list if two insns
26706 have the same priority - candidate is best if its dependees were
26707 scheduled earlier. Applied for Silvermont only.
26708 Return true if top 2 insns must be interchanged. */
26710 swap_top_of_ready_list (rtx_insn **ready, int n_ready)
26712 rtx_insn *top = ready[n_ready - 1];
26713 rtx_insn *next = ready[n_ready - 2];
26715 sd_iterator_def sd_it;
26719 #define INSN_TICK(INSN) (HID (INSN)->tick)
26721 if (!TARGET_SILVERMONT && !TARGET_INTEL)
26724 if (!NONDEBUG_INSN_P (top))
26726 if (!NONJUMP_INSN_P (top))
26728 if (!NONDEBUG_INSN_P (next))
26730 if (!NONJUMP_INSN_P (next))
26732 set = single_set (top);
26735 set = single_set (next);
26739 if (INSN_PRIORITY_KNOWN (top) && INSN_PRIORITY_KNOWN (next))
26741 if (INSN_PRIORITY (top) != INSN_PRIORITY (next))
26743 /* Determine winner more precise. */
26744 FOR_EACH_DEP (top, SD_LIST_RES_BACK, sd_it, dep)
26747 pro = DEP_PRO (dep);
26748 if (!NONDEBUG_INSN_P (pro))
26750 if (INSN_TICK (pro) > clock1)
26751 clock1 = INSN_TICK (pro);
26753 FOR_EACH_DEP (next, SD_LIST_RES_BACK, sd_it, dep)
26756 pro = DEP_PRO (dep);
26757 if (!NONDEBUG_INSN_P (pro))
26759 if (INSN_TICK (pro) > clock2)
26760 clock2 = INSN_TICK (pro);
26763 if (clock1 == clock2)
26765 /* Determine winner - load must win. */
26766 enum attr_memory memory1, memory2;
26767 memory1 = get_attr_memory (top);
26768 memory2 = get_attr_memory (next);
26769 if (memory2 == MEMORY_LOAD && memory1 != MEMORY_LOAD)
26772 return (bool) (clock2 < clock1);
26778 /* Perform possible reodering of ready list for Atom/Silvermont only.
26779 Return issue rate. */
26781 ix86_sched_reorder (FILE *dump, int sched_verbose, rtx_insn **ready,
26782 int *pn_ready, int clock_var)
26784 int issue_rate = -1;
26785 int n_ready = *pn_ready;
26790 /* Set up issue rate. */
26791 issue_rate = ix86_issue_rate ();
26793 /* Do reodering for BONNELL/SILVERMONT only. */
26794 if (!TARGET_BONNELL && !TARGET_SILVERMONT && !TARGET_INTEL)
26797 /* Nothing to do if ready list contains only 1 instruction. */
26801 /* Do reodering for post-reload scheduler only. */
26802 if (!reload_completed)
26805 if ((index = do_reorder_for_imul (ready, n_ready)) >= 0)
26807 if (sched_verbose > 1)
26808 fprintf (dump, ";;\tatom sched_reorder: put %d insn on top\n",
26809 INSN_UID (ready[index]));
26811 /* Put IMUL producer (ready[index]) at the top of ready list. */
26812 insn = ready[index];
26813 for (i = index; i < n_ready - 1; i++)
26814 ready[i] = ready[i + 1];
26815 ready[n_ready - 1] = insn;
26819 /* Skip selective scheduling since HID is not populated in it. */
26822 && swap_top_of_ready_list (ready, n_ready))
26824 if (sched_verbose > 1)
26825 fprintf (dump, ";;\tslm sched_reorder: swap %d and %d insns\n",
26826 INSN_UID (ready[n_ready - 1]), INSN_UID (ready[n_ready - 2]));
26827 /* Swap 2 top elements of ready list. */
26828 insn = ready[n_ready - 1];
26829 ready[n_ready - 1] = ready[n_ready - 2];
26830 ready[n_ready - 2] = insn;
26836 ix86_class_likely_spilled_p (reg_class_t);
26838 /* Returns true if lhs of insn is HW function argument register and set up
26839 is_spilled to true if it is likely spilled HW register. */
26841 insn_is_function_arg (rtx insn, bool* is_spilled)
26845 if (!NONDEBUG_INSN_P (insn))
26847 /* Call instructions are not movable, ignore it. */
26850 insn = PATTERN (insn);
26851 if (GET_CODE (insn) == PARALLEL)
26852 insn = XVECEXP (insn, 0, 0);
26853 if (GET_CODE (insn) != SET)
26855 dst = SET_DEST (insn);
26856 if (REG_P (dst) && HARD_REGISTER_P (dst)
26857 && ix86_function_arg_regno_p (REGNO (dst)))
26859 /* Is it likely spilled HW register? */
26860 if (!TEST_HARD_REG_BIT (fixed_reg_set, REGNO (dst))
26861 && ix86_class_likely_spilled_p (REGNO_REG_CLASS (REGNO (dst))))
26862 *is_spilled = true;
26868 /* Add output dependencies for chain of function adjacent arguments if only
26869 there is a move to likely spilled HW register. Return first argument
26870 if at least one dependence was added or NULL otherwise. */
26872 add_parameter_dependencies (rtx_insn *call, rtx_insn *head)
26875 rtx_insn *last = call;
26876 rtx_insn *first_arg = NULL;
26877 bool is_spilled = false;
26879 head = PREV_INSN (head);
26881 /* Find nearest to call argument passing instruction. */
26884 last = PREV_INSN (last);
26887 if (!NONDEBUG_INSN_P (last))
26889 if (insn_is_function_arg (last, &is_spilled))
26897 insn = PREV_INSN (last);
26898 if (!INSN_P (insn))
26902 if (!NONDEBUG_INSN_P (insn))
26907 if (insn_is_function_arg (insn, &is_spilled))
26909 /* Add output depdendence between two function arguments if chain
26910 of output arguments contains likely spilled HW registers. */
26912 add_dependence (first_arg, insn, REG_DEP_OUTPUT);
26913 first_arg = last = insn;
26923 /* Add output or anti dependency from insn to first_arg to restrict its code
26926 avoid_func_arg_motion (rtx_insn *first_arg, rtx_insn *insn)
26931 /* Add anti dependencies for bounds stores. */
26933 && GET_CODE (PATTERN (insn)) == PARALLEL
26934 && GET_CODE (XVECEXP (PATTERN (insn), 0, 0)) == UNSPEC
26935 && XINT (XVECEXP (PATTERN (insn), 0, 0), 1) == UNSPEC_BNDSTX)
26937 add_dependence (first_arg, insn, REG_DEP_ANTI);
26941 set = single_set (insn);
26944 tmp = SET_DEST (set);
26947 /* Add output dependency to the first function argument. */
26948 add_dependence (first_arg, insn, REG_DEP_OUTPUT);
26951 /* Add anti dependency. */
26952 add_dependence (first_arg, insn, REG_DEP_ANTI);
26955 /* Avoid cross block motion of function argument through adding dependency
26956 from the first non-jump instruction in bb. */
26958 add_dependee_for_func_arg (rtx_insn *arg, basic_block bb)
26960 rtx_insn *insn = BB_END (bb);
26964 if (NONDEBUG_INSN_P (insn) && NONJUMP_INSN_P (insn))
26966 rtx set = single_set (insn);
26969 avoid_func_arg_motion (arg, insn);
26973 if (insn == BB_HEAD (bb))
26975 insn = PREV_INSN (insn);
26979 /* Hook for pre-reload schedule - avoid motion of function arguments
26980 passed in likely spilled HW registers. */
26982 ix86_dependencies_evaluation_hook (rtx_insn *head, rtx_insn *tail)
26985 rtx_insn *first_arg = NULL;
26986 if (reload_completed)
26988 while (head != tail && DEBUG_INSN_P (head))
26989 head = NEXT_INSN (head);
26990 for (insn = tail; insn != head; insn = PREV_INSN (insn))
26991 if (INSN_P (insn) && CALL_P (insn))
26993 first_arg = add_parameter_dependencies (insn, head);
26996 /* Add dependee for first argument to predecessors if only
26997 region contains more than one block. */
26998 basic_block bb = BLOCK_FOR_INSN (insn);
26999 int rgn = CONTAINING_RGN (bb->index);
27000 int nr_blks = RGN_NR_BLOCKS (rgn);
27001 /* Skip trivial regions and region head blocks that can have
27002 predecessors outside of region. */
27003 if (nr_blks > 1 && BLOCK_TO_BB (bb->index) != 0)
27008 /* Regions are SCCs with the exception of selective
27009 scheduling with pipelining of outer blocks enabled.
27010 So also check that immediate predecessors of a non-head
27011 block are in the same region. */
27012 FOR_EACH_EDGE (e, ei, bb->preds)
27014 /* Avoid creating of loop-carried dependencies through
27015 using topological ordering in the region. */
27016 if (rgn == CONTAINING_RGN (e->src->index)
27017 && BLOCK_TO_BB (bb->index) > BLOCK_TO_BB (e->src->index))
27018 add_dependee_for_func_arg (first_arg, e->src);
27026 else if (first_arg)
27027 avoid_func_arg_motion (first_arg, insn);
27030 /* Hook for pre-reload schedule - set priority of moves from likely spilled
27031 HW registers to maximum, to schedule them at soon as possible. These are
27032 moves from function argument registers at the top of the function entry
27033 and moves from function return value registers after call. */
27035 ix86_adjust_priority (rtx_insn *insn, int priority)
27039 if (reload_completed)
27042 if (!NONDEBUG_INSN_P (insn))
27045 set = single_set (insn);
27048 rtx tmp = SET_SRC (set);
27050 && HARD_REGISTER_P (tmp)
27051 && !TEST_HARD_REG_BIT (fixed_reg_set, REGNO (tmp))
27052 && ix86_class_likely_spilled_p (REGNO_REG_CLASS (REGNO (tmp))))
27053 return current_sched_info->sched_max_insns_priority;
27059 /* Model decoder of Core 2/i7.
27060 Below hooks for multipass scheduling (see haifa-sched.c:max_issue)
27061 track the instruction fetch block boundaries and make sure that long
27062 (9+ bytes) instructions are assigned to D0. */
27064 /* Maximum length of an insn that can be handled by
27065 a secondary decoder unit. '8' for Core 2/i7. */
27066 static int core2i7_secondary_decoder_max_insn_size;
27068 /* Ifetch block size, i.e., number of bytes decoder reads per cycle.
27069 '16' for Core 2/i7. */
27070 static int core2i7_ifetch_block_size;
27072 /* Maximum number of instructions decoder can handle per cycle.
27073 '6' for Core 2/i7. */
27074 static int core2i7_ifetch_block_max_insns;
27076 typedef struct ix86_first_cycle_multipass_data_ *
27077 ix86_first_cycle_multipass_data_t;
27078 typedef const struct ix86_first_cycle_multipass_data_ *
27079 const_ix86_first_cycle_multipass_data_t;
27081 /* A variable to store target state across calls to max_issue within
27083 static struct ix86_first_cycle_multipass_data_ _ix86_first_cycle_multipass_data,
27084 *ix86_first_cycle_multipass_data = &_ix86_first_cycle_multipass_data;
27086 /* Initialize DATA. */
27088 core2i7_first_cycle_multipass_init (void *_data)
27090 ix86_first_cycle_multipass_data_t data
27091 = (ix86_first_cycle_multipass_data_t) _data;
27093 data->ifetch_block_len = 0;
27094 data->ifetch_block_n_insns = 0;
27095 data->ready_try_change = NULL;
27096 data->ready_try_change_size = 0;
27099 /* Advancing the cycle; reset ifetch block counts. */
27101 core2i7_dfa_post_advance_cycle (void)
27103 ix86_first_cycle_multipass_data_t data = ix86_first_cycle_multipass_data;
27105 gcc_assert (data->ifetch_block_n_insns <= core2i7_ifetch_block_max_insns);
27107 data->ifetch_block_len = 0;
27108 data->ifetch_block_n_insns = 0;
27111 static int min_insn_size (rtx_insn *);
27113 /* Filter out insns from ready_try that the core will not be able to issue
27114 on current cycle due to decoder. */
27116 core2i7_first_cycle_multipass_filter_ready_try
27117 (const_ix86_first_cycle_multipass_data_t data,
27118 signed char *ready_try, int n_ready, bool first_cycle_insn_p)
27125 if (ready_try[n_ready])
27128 insn = get_ready_element (n_ready);
27129 insn_size = min_insn_size (insn);
27131 if (/* If this is a too long an insn for a secondary decoder ... */
27132 (!first_cycle_insn_p
27133 && insn_size > core2i7_secondary_decoder_max_insn_size)
27134 /* ... or it would not fit into the ifetch block ... */
27135 || data->ifetch_block_len + insn_size > core2i7_ifetch_block_size
27136 /* ... or the decoder is full already ... */
27137 || data->ifetch_block_n_insns + 1 > core2i7_ifetch_block_max_insns)
27138 /* ... mask the insn out. */
27140 ready_try[n_ready] = 1;
27142 if (data->ready_try_change)
27143 bitmap_set_bit (data->ready_try_change, n_ready);
27148 /* Prepare for a new round of multipass lookahead scheduling. */
27150 core2i7_first_cycle_multipass_begin (void *_data,
27151 signed char *ready_try, int n_ready,
27152 bool first_cycle_insn_p)
27154 ix86_first_cycle_multipass_data_t data
27155 = (ix86_first_cycle_multipass_data_t) _data;
27156 const_ix86_first_cycle_multipass_data_t prev_data
27157 = ix86_first_cycle_multipass_data;
27159 /* Restore the state from the end of the previous round. */
27160 data->ifetch_block_len = prev_data->ifetch_block_len;
27161 data->ifetch_block_n_insns = prev_data->ifetch_block_n_insns;
27163 /* Filter instructions that cannot be issued on current cycle due to
27164 decoder restrictions. */
27165 core2i7_first_cycle_multipass_filter_ready_try (data, ready_try, n_ready,
27166 first_cycle_insn_p);
27169 /* INSN is being issued in current solution. Account for its impact on
27170 the decoder model. */
27172 core2i7_first_cycle_multipass_issue (void *_data,
27173 signed char *ready_try, int n_ready,
27174 rtx_insn *insn, const void *_prev_data)
27176 ix86_first_cycle_multipass_data_t data
27177 = (ix86_first_cycle_multipass_data_t) _data;
27178 const_ix86_first_cycle_multipass_data_t prev_data
27179 = (const_ix86_first_cycle_multipass_data_t) _prev_data;
27181 int insn_size = min_insn_size (insn);
27183 data->ifetch_block_len = prev_data->ifetch_block_len + insn_size;
27184 data->ifetch_block_n_insns = prev_data->ifetch_block_n_insns + 1;
27185 gcc_assert (data->ifetch_block_len <= core2i7_ifetch_block_size
27186 && data->ifetch_block_n_insns <= core2i7_ifetch_block_max_insns);
27188 /* Allocate or resize the bitmap for storing INSN's effect on ready_try. */
27189 if (!data->ready_try_change)
27191 data->ready_try_change = sbitmap_alloc (n_ready);
27192 data->ready_try_change_size = n_ready;
27194 else if (data->ready_try_change_size < n_ready)
27196 data->ready_try_change = sbitmap_resize (data->ready_try_change,
27198 data->ready_try_change_size = n_ready;
27200 bitmap_clear (data->ready_try_change);
27202 /* Filter out insns from ready_try that the core will not be able to issue
27203 on current cycle due to decoder. */
27204 core2i7_first_cycle_multipass_filter_ready_try (data, ready_try, n_ready,
27208 /* Revert the effect on ready_try. */
27210 core2i7_first_cycle_multipass_backtrack (const void *_data,
27211 signed char *ready_try,
27212 int n_ready ATTRIBUTE_UNUSED)
27214 const_ix86_first_cycle_multipass_data_t data
27215 = (const_ix86_first_cycle_multipass_data_t) _data;
27216 unsigned int i = 0;
27217 sbitmap_iterator sbi;
27219 gcc_assert (bitmap_last_set_bit (data->ready_try_change) < n_ready);
27220 EXECUTE_IF_SET_IN_BITMAP (data->ready_try_change, 0, i, sbi)
27226 /* Save the result of multipass lookahead scheduling for the next round. */
27228 core2i7_first_cycle_multipass_end (const void *_data)
27230 const_ix86_first_cycle_multipass_data_t data
27231 = (const_ix86_first_cycle_multipass_data_t) _data;
27232 ix86_first_cycle_multipass_data_t next_data
27233 = ix86_first_cycle_multipass_data;
27237 next_data->ifetch_block_len = data->ifetch_block_len;
27238 next_data->ifetch_block_n_insns = data->ifetch_block_n_insns;
27242 /* Deallocate target data. */
27244 core2i7_first_cycle_multipass_fini (void *_data)
27246 ix86_first_cycle_multipass_data_t data
27247 = (ix86_first_cycle_multipass_data_t) _data;
27249 if (data->ready_try_change)
27251 sbitmap_free (data->ready_try_change);
27252 data->ready_try_change = NULL;
27253 data->ready_try_change_size = 0;
27257 /* Prepare for scheduling pass. */
27259 ix86_sched_init_global (FILE *, int, int)
27261 /* Install scheduling hooks for current CPU. Some of these hooks are used
27262 in time-critical parts of the scheduler, so we only set them up when
27263 they are actually used. */
27266 case PROCESSOR_CORE2:
27267 case PROCESSOR_NEHALEM:
27268 case PROCESSOR_SANDYBRIDGE:
27269 case PROCESSOR_HASWELL:
27270 /* Do not perform multipass scheduling for pre-reload schedule
27271 to save compile time. */
27272 if (reload_completed)
27274 targetm.sched.dfa_post_advance_cycle
27275 = core2i7_dfa_post_advance_cycle;
27276 targetm.sched.first_cycle_multipass_init
27277 = core2i7_first_cycle_multipass_init;
27278 targetm.sched.first_cycle_multipass_begin
27279 = core2i7_first_cycle_multipass_begin;
27280 targetm.sched.first_cycle_multipass_issue
27281 = core2i7_first_cycle_multipass_issue;
27282 targetm.sched.first_cycle_multipass_backtrack
27283 = core2i7_first_cycle_multipass_backtrack;
27284 targetm.sched.first_cycle_multipass_end
27285 = core2i7_first_cycle_multipass_end;
27286 targetm.sched.first_cycle_multipass_fini
27287 = core2i7_first_cycle_multipass_fini;
27289 /* Set decoder parameters. */
27290 core2i7_secondary_decoder_max_insn_size = 8;
27291 core2i7_ifetch_block_size = 16;
27292 core2i7_ifetch_block_max_insns = 6;
27295 /* ... Fall through ... */
27297 targetm.sched.dfa_post_advance_cycle = NULL;
27298 targetm.sched.first_cycle_multipass_init = NULL;
27299 targetm.sched.first_cycle_multipass_begin = NULL;
27300 targetm.sched.first_cycle_multipass_issue = NULL;
27301 targetm.sched.first_cycle_multipass_backtrack = NULL;
27302 targetm.sched.first_cycle_multipass_end = NULL;
27303 targetm.sched.first_cycle_multipass_fini = NULL;
27309 /* Compute the alignment given to a constant that is being placed in memory.
27310 EXP is the constant and ALIGN is the alignment that the object would
27312 The value of this function is used instead of that alignment to align
27316 ix86_constant_alignment (tree exp, int align)
27318 if (TREE_CODE (exp) == REAL_CST || TREE_CODE (exp) == VECTOR_CST
27319 || TREE_CODE (exp) == INTEGER_CST)
27321 if (TYPE_MODE (TREE_TYPE (exp)) == DFmode && align < 64)
27323 else if (ALIGN_MODE_128 (TYPE_MODE (TREE_TYPE (exp))) && align < 128)
27326 else if (!optimize_size && TREE_CODE (exp) == STRING_CST
27327 && TREE_STRING_LENGTH (exp) >= 31 && align < BITS_PER_WORD)
27328 return BITS_PER_WORD;
27333 /* Compute the alignment for a static variable.
27334 TYPE is the data type, and ALIGN is the alignment that
27335 the object would ordinarily have. The value of this function is used
27336 instead of that alignment to align the object. */
27339 ix86_data_alignment (tree type, int align, bool opt)
27341 /* GCC 4.8 and earlier used to incorrectly assume this alignment even
27342 for symbols from other compilation units or symbols that don't need
27343 to bind locally. In order to preserve some ABI compatibility with
27344 those compilers, ensure we don't decrease alignment from what we
27347 int max_align_compat = MIN (256, MAX_OFILE_ALIGNMENT);
27349 /* A data structure, equal or greater than the size of a cache line
27350 (64 bytes in the Pentium 4 and other recent Intel processors, including
27351 processors based on Intel Core microarchitecture) should be aligned
27352 so that its base address is a multiple of a cache line size. */
27355 = MIN ((unsigned) ix86_tune_cost->prefetch_block * 8, MAX_OFILE_ALIGNMENT);
27357 if (max_align < BITS_PER_WORD)
27358 max_align = BITS_PER_WORD;
27360 switch (ix86_align_data_type)
27362 case ix86_align_data_type_abi: opt = false; break;
27363 case ix86_align_data_type_compat: max_align = BITS_PER_WORD; break;
27364 case ix86_align_data_type_cacheline: break;
27368 && AGGREGATE_TYPE_P (type)
27369 && TYPE_SIZE (type)
27370 && TREE_CODE (TYPE_SIZE (type)) == INTEGER_CST)
27372 if (wi::geu_p (TYPE_SIZE (type), max_align_compat)
27373 && align < max_align_compat)
27374 align = max_align_compat;
27375 if (wi::geu_p (TYPE_SIZE (type), max_align)
27376 && align < max_align)
27380 /* x86-64 ABI requires arrays greater than 16 bytes to be aligned
27381 to 16byte boundary. */
27384 if ((opt ? AGGREGATE_TYPE_P (type) : TREE_CODE (type) == ARRAY_TYPE)
27385 && TYPE_SIZE (type)
27386 && TREE_CODE (TYPE_SIZE (type)) == INTEGER_CST
27387 && wi::geu_p (TYPE_SIZE (type), 128)
27395 if (TREE_CODE (type) == ARRAY_TYPE)
27397 if (TYPE_MODE (TREE_TYPE (type)) == DFmode && align < 64)
27399 if (ALIGN_MODE_128 (TYPE_MODE (TREE_TYPE (type))) && align < 128)
27402 else if (TREE_CODE (type) == COMPLEX_TYPE)
27405 if (TYPE_MODE (type) == DCmode && align < 64)
27407 if ((TYPE_MODE (type) == XCmode
27408 || TYPE_MODE (type) == TCmode) && align < 128)
27411 else if ((TREE_CODE (type) == RECORD_TYPE
27412 || TREE_CODE (type) == UNION_TYPE
27413 || TREE_CODE (type) == QUAL_UNION_TYPE)
27414 && TYPE_FIELDS (type))
27416 if (DECL_MODE (TYPE_FIELDS (type)) == DFmode && align < 64)
27418 if (ALIGN_MODE_128 (DECL_MODE (TYPE_FIELDS (type))) && align < 128)
27421 else if (TREE_CODE (type) == REAL_TYPE || TREE_CODE (type) == VECTOR_TYPE
27422 || TREE_CODE (type) == INTEGER_TYPE)
27424 if (TYPE_MODE (type) == DFmode && align < 64)
27426 if (ALIGN_MODE_128 (TYPE_MODE (type)) && align < 128)
27433 /* Compute the alignment for a local variable or a stack slot. EXP is
27434 the data type or decl itself, MODE is the widest mode available and
27435 ALIGN is the alignment that the object would ordinarily have. The
27436 value of this macro is used instead of that alignment to align the
27440 ix86_local_alignment (tree exp, machine_mode mode,
27441 unsigned int align)
27445 if (exp && DECL_P (exp))
27447 type = TREE_TYPE (exp);
27456 /* Don't do dynamic stack realignment for long long objects with
27457 -mpreferred-stack-boundary=2. */
27460 && ix86_preferred_stack_boundary < 64
27461 && (mode == DImode || (type && TYPE_MODE (type) == DImode))
27462 && (!type || !TYPE_USER_ALIGN (type))
27463 && (!decl || !DECL_USER_ALIGN (decl)))
27466 /* If TYPE is NULL, we are allocating a stack slot for caller-save
27467 register in MODE. We will return the largest alignment of XF
27471 if (mode == XFmode && align < GET_MODE_ALIGNMENT (DFmode))
27472 align = GET_MODE_ALIGNMENT (DFmode);
27476 /* x86-64 ABI requires arrays greater than 16 bytes to be aligned
27477 to 16byte boundary. Exact wording is:
27479 An array uses the same alignment as its elements, except that a local or
27480 global array variable of length at least 16 bytes or
27481 a C99 variable-length array variable always has alignment of at least 16 bytes.
27483 This was added to allow use of aligned SSE instructions at arrays. This
27484 rule is meant for static storage (where compiler can not do the analysis
27485 by itself). We follow it for automatic variables only when convenient.
27486 We fully control everything in the function compiled and functions from
27487 other unit can not rely on the alignment.
27489 Exclude va_list type. It is the common case of local array where
27490 we can not benefit from the alignment.
27492 TODO: Probably one should optimize for size only when var is not escaping. */
27493 if (TARGET_64BIT && optimize_function_for_speed_p (cfun)
27496 if (AGGREGATE_TYPE_P (type)
27497 && (va_list_type_node == NULL_TREE
27498 || (TYPE_MAIN_VARIANT (type)
27499 != TYPE_MAIN_VARIANT (va_list_type_node)))
27500 && TYPE_SIZE (type)
27501 && TREE_CODE (TYPE_SIZE (type)) == INTEGER_CST
27502 && wi::geu_p (TYPE_SIZE (type), 16)
27506 if (TREE_CODE (type) == ARRAY_TYPE)
27508 if (TYPE_MODE (TREE_TYPE (type)) == DFmode && align < 64)
27510 if (ALIGN_MODE_128 (TYPE_MODE (TREE_TYPE (type))) && align < 128)
27513 else if (TREE_CODE (type) == COMPLEX_TYPE)
27515 if (TYPE_MODE (type) == DCmode && align < 64)
27517 if ((TYPE_MODE (type) == XCmode
27518 || TYPE_MODE (type) == TCmode) && align < 128)
27521 else if ((TREE_CODE (type) == RECORD_TYPE
27522 || TREE_CODE (type) == UNION_TYPE
27523 || TREE_CODE (type) == QUAL_UNION_TYPE)
27524 && TYPE_FIELDS (type))
27526 if (DECL_MODE (TYPE_FIELDS (type)) == DFmode && align < 64)
27528 if (ALIGN_MODE_128 (DECL_MODE (TYPE_FIELDS (type))) && align < 128)
27531 else if (TREE_CODE (type) == REAL_TYPE || TREE_CODE (type) == VECTOR_TYPE
27532 || TREE_CODE (type) == INTEGER_TYPE)
27535 if (TYPE_MODE (type) == DFmode && align < 64)
27537 if (ALIGN_MODE_128 (TYPE_MODE (type)) && align < 128)
27543 /* Compute the minimum required alignment for dynamic stack realignment
27544 purposes for a local variable, parameter or a stack slot. EXP is
27545 the data type or decl itself, MODE is its mode and ALIGN is the
27546 alignment that the object would ordinarily have. */
27549 ix86_minimum_alignment (tree exp, machine_mode mode,
27550 unsigned int align)
27554 if (exp && DECL_P (exp))
27556 type = TREE_TYPE (exp);
27565 if (TARGET_64BIT || align != 64 || ix86_preferred_stack_boundary >= 64)
27568 /* Don't do dynamic stack realignment for long long objects with
27569 -mpreferred-stack-boundary=2. */
27570 if ((mode == DImode || (type && TYPE_MODE (type) == DImode))
27571 && (!type || !TYPE_USER_ALIGN (type))
27572 && (!decl || !DECL_USER_ALIGN (decl)))
27578 /* Find a location for the static chain incoming to a nested function.
27579 This is a register, unless all free registers are used by arguments. */
27582 ix86_static_chain (const_tree fndecl_or_type, bool incoming_p)
27586 /* While this function won't be called by the middle-end when a static
27587 chain isn't needed, it's also used throughout the backend so it's
27588 easiest to keep this check centralized. */
27589 if (DECL_P (fndecl_or_type) && !DECL_STATIC_CHAIN (fndecl_or_type))
27594 /* We always use R10 in 64-bit mode. */
27599 const_tree fntype, fndecl;
27602 /* By default in 32-bit mode we use ECX to pass the static chain. */
27605 if (TREE_CODE (fndecl_or_type) == FUNCTION_DECL)
27607 fntype = TREE_TYPE (fndecl_or_type);
27608 fndecl = fndecl_or_type;
27612 fntype = fndecl_or_type;
27616 ccvt = ix86_get_callcvt (fntype);
27617 if ((ccvt & IX86_CALLCVT_FASTCALL) != 0)
27619 /* Fastcall functions use ecx/edx for arguments, which leaves
27620 us with EAX for the static chain.
27621 Thiscall functions use ecx for arguments, which also
27622 leaves us with EAX for the static chain. */
27625 else if ((ccvt & IX86_CALLCVT_THISCALL) != 0)
27627 /* Thiscall functions use ecx for arguments, which leaves
27628 us with EAX and EDX for the static chain.
27629 We are using for abi-compatibility EAX. */
27632 else if (ix86_function_regparm (fntype, fndecl) == 3)
27634 /* For regparm 3, we have no free call-clobbered registers in
27635 which to store the static chain. In order to implement this,
27636 we have the trampoline push the static chain to the stack.
27637 However, we can't push a value below the return address when
27638 we call the nested function directly, so we have to use an
27639 alternate entry point. For this we use ESI, and have the
27640 alternate entry point push ESI, so that things appear the
27641 same once we're executing the nested function. */
27644 if (fndecl == current_function_decl)
27645 ix86_static_chain_on_stack = true;
27646 return gen_frame_mem (SImode,
27647 plus_constant (Pmode,
27648 arg_pointer_rtx, -8));
27654 return gen_rtx_REG (Pmode, regno);
27657 /* Emit RTL insns to initialize the variable parts of a trampoline.
27658 FNDECL is the decl of the target address; M_TRAMP is a MEM for
27659 the trampoline, and CHAIN_VALUE is an RTX for the static chain
27660 to be passed to the target function. */
27663 ix86_trampoline_init (rtx m_tramp, tree fndecl, rtx chain_value)
27669 fnaddr = XEXP (DECL_RTL (fndecl), 0);
27675 /* Load the function address to r11. Try to load address using
27676 the shorter movl instead of movabs. We may want to support
27677 movq for kernel mode, but kernel does not use trampolines at
27678 the moment. FNADDR is a 32bit address and may not be in
27679 DImode when ptr_mode == SImode. Always use movl in this
27681 if (ptr_mode == SImode
27682 || x86_64_zext_immediate_operand (fnaddr, VOIDmode))
27684 fnaddr = copy_addr_to_reg (fnaddr);
27686 mem = adjust_address (m_tramp, HImode, offset);
27687 emit_move_insn (mem, gen_int_mode (0xbb41, HImode));
27689 mem = adjust_address (m_tramp, SImode, offset + 2);
27690 emit_move_insn (mem, gen_lowpart (SImode, fnaddr));
27695 mem = adjust_address (m_tramp, HImode, offset);
27696 emit_move_insn (mem, gen_int_mode (0xbb49, HImode));
27698 mem = adjust_address (m_tramp, DImode, offset + 2);
27699 emit_move_insn (mem, fnaddr);
27703 /* Load static chain using movabs to r10. Use the shorter movl
27704 instead of movabs when ptr_mode == SImode. */
27705 if (ptr_mode == SImode)
27716 mem = adjust_address (m_tramp, HImode, offset);
27717 emit_move_insn (mem, gen_int_mode (opcode, HImode));
27719 mem = adjust_address (m_tramp, ptr_mode, offset + 2);
27720 emit_move_insn (mem, chain_value);
27723 /* Jump to r11; the last (unused) byte is a nop, only there to
27724 pad the write out to a single 32-bit store. */
27725 mem = adjust_address (m_tramp, SImode, offset);
27726 emit_move_insn (mem, gen_int_mode (0x90e3ff49, SImode));
27733 /* Depending on the static chain location, either load a register
27734 with a constant, or push the constant to the stack. All of the
27735 instructions are the same size. */
27736 chain = ix86_static_chain (fndecl, true);
27739 switch (REGNO (chain))
27742 opcode = 0xb8; break;
27744 opcode = 0xb9; break;
27746 gcc_unreachable ();
27752 mem = adjust_address (m_tramp, QImode, offset);
27753 emit_move_insn (mem, gen_int_mode (opcode, QImode));
27755 mem = adjust_address (m_tramp, SImode, offset + 1);
27756 emit_move_insn (mem, chain_value);
27759 mem = adjust_address (m_tramp, QImode, offset);
27760 emit_move_insn (mem, gen_int_mode (0xe9, QImode));
27762 mem = adjust_address (m_tramp, SImode, offset + 1);
27764 /* Compute offset from the end of the jmp to the target function.
27765 In the case in which the trampoline stores the static chain on
27766 the stack, we need to skip the first insn which pushes the
27767 (call-saved) register static chain; this push is 1 byte. */
27769 disp = expand_binop (SImode, sub_optab, fnaddr,
27770 plus_constant (Pmode, XEXP (m_tramp, 0),
27771 offset - (MEM_P (chain) ? 1 : 0)),
27772 NULL_RTX, 1, OPTAB_DIRECT);
27773 emit_move_insn (mem, disp);
27776 gcc_assert (offset <= TRAMPOLINE_SIZE);
27778 #ifdef HAVE_ENABLE_EXECUTE_STACK
27779 #ifdef CHECK_EXECUTE_STACK_ENABLED
27780 if (CHECK_EXECUTE_STACK_ENABLED)
27782 emit_library_call (gen_rtx_SYMBOL_REF (Pmode, "__enable_execute_stack"),
27783 LCT_NORMAL, VOIDmode, 1, XEXP (m_tramp, 0), Pmode);
27787 /* The following file contains several enumerations and data structures
27788 built from the definitions in i386-builtin-types.def. */
27790 #include "i386-builtin-types.inc"
27792 /* Table for the ix86 builtin non-function types. */
27793 static GTY(()) tree ix86_builtin_type_tab[(int) IX86_BT_LAST_CPTR + 1];
27795 /* Retrieve an element from the above table, building some of
27796 the types lazily. */
27799 ix86_get_builtin_type (enum ix86_builtin_type tcode)
27801 unsigned int index;
27804 gcc_assert ((unsigned)tcode < ARRAY_SIZE(ix86_builtin_type_tab));
27806 type = ix86_builtin_type_tab[(int) tcode];
27810 gcc_assert (tcode > IX86_BT_LAST_PRIM);
27811 if (tcode <= IX86_BT_LAST_VECT)
27815 index = tcode - IX86_BT_LAST_PRIM - 1;
27816 itype = ix86_get_builtin_type (ix86_builtin_type_vect_base[index]);
27817 mode = ix86_builtin_type_vect_mode[index];
27819 type = build_vector_type_for_mode (itype, mode);
27825 index = tcode - IX86_BT_LAST_VECT - 1;
27826 if (tcode <= IX86_BT_LAST_PTR)
27827 quals = TYPE_UNQUALIFIED;
27829 quals = TYPE_QUAL_CONST;
27831 itype = ix86_get_builtin_type (ix86_builtin_type_ptr_base[index]);
27832 if (quals != TYPE_UNQUALIFIED)
27833 itype = build_qualified_type (itype, quals);
27835 type = build_pointer_type (itype);
27838 ix86_builtin_type_tab[(int) tcode] = type;
27842 /* Table for the ix86 builtin function types. */
27843 static GTY(()) tree ix86_builtin_func_type_tab[(int) IX86_BT_LAST_ALIAS + 1];
27845 /* Retrieve an element from the above table, building some of
27846 the types lazily. */
27849 ix86_get_builtin_func_type (enum ix86_builtin_func_type tcode)
27853 gcc_assert ((unsigned)tcode < ARRAY_SIZE (ix86_builtin_func_type_tab));
27855 type = ix86_builtin_func_type_tab[(int) tcode];
27859 if (tcode <= IX86_BT_LAST_FUNC)
27861 unsigned start = ix86_builtin_func_start[(int) tcode];
27862 unsigned after = ix86_builtin_func_start[(int) tcode + 1];
27863 tree rtype, atype, args = void_list_node;
27866 rtype = ix86_get_builtin_type (ix86_builtin_func_args[start]);
27867 for (i = after - 1; i > start; --i)
27869 atype = ix86_get_builtin_type (ix86_builtin_func_args[i]);
27870 args = tree_cons (NULL, atype, args);
27873 type = build_function_type (rtype, args);
27877 unsigned index = tcode - IX86_BT_LAST_FUNC - 1;
27878 enum ix86_builtin_func_type icode;
27880 icode = ix86_builtin_func_alias_base[index];
27881 type = ix86_get_builtin_func_type (icode);
27884 ix86_builtin_func_type_tab[(int) tcode] = type;
27889 /* Codes for all the SSE/MMX builtins. */
27892 IX86_BUILTIN_ADDPS,
27893 IX86_BUILTIN_ADDSS,
27894 IX86_BUILTIN_DIVPS,
27895 IX86_BUILTIN_DIVSS,
27896 IX86_BUILTIN_MULPS,
27897 IX86_BUILTIN_MULSS,
27898 IX86_BUILTIN_SUBPS,
27899 IX86_BUILTIN_SUBSS,
27901 IX86_BUILTIN_CMPEQPS,
27902 IX86_BUILTIN_CMPLTPS,
27903 IX86_BUILTIN_CMPLEPS,
27904 IX86_BUILTIN_CMPGTPS,
27905 IX86_BUILTIN_CMPGEPS,
27906 IX86_BUILTIN_CMPNEQPS,
27907 IX86_BUILTIN_CMPNLTPS,
27908 IX86_BUILTIN_CMPNLEPS,
27909 IX86_BUILTIN_CMPNGTPS,
27910 IX86_BUILTIN_CMPNGEPS,
27911 IX86_BUILTIN_CMPORDPS,
27912 IX86_BUILTIN_CMPUNORDPS,
27913 IX86_BUILTIN_CMPEQSS,
27914 IX86_BUILTIN_CMPLTSS,
27915 IX86_BUILTIN_CMPLESS,
27916 IX86_BUILTIN_CMPNEQSS,
27917 IX86_BUILTIN_CMPNLTSS,
27918 IX86_BUILTIN_CMPNLESS,
27919 IX86_BUILTIN_CMPORDSS,
27920 IX86_BUILTIN_CMPUNORDSS,
27922 IX86_BUILTIN_COMIEQSS,
27923 IX86_BUILTIN_COMILTSS,
27924 IX86_BUILTIN_COMILESS,
27925 IX86_BUILTIN_COMIGTSS,
27926 IX86_BUILTIN_COMIGESS,
27927 IX86_BUILTIN_COMINEQSS,
27928 IX86_BUILTIN_UCOMIEQSS,
27929 IX86_BUILTIN_UCOMILTSS,
27930 IX86_BUILTIN_UCOMILESS,
27931 IX86_BUILTIN_UCOMIGTSS,
27932 IX86_BUILTIN_UCOMIGESS,
27933 IX86_BUILTIN_UCOMINEQSS,
27935 IX86_BUILTIN_CVTPI2PS,
27936 IX86_BUILTIN_CVTPS2PI,
27937 IX86_BUILTIN_CVTSI2SS,
27938 IX86_BUILTIN_CVTSI642SS,
27939 IX86_BUILTIN_CVTSS2SI,
27940 IX86_BUILTIN_CVTSS2SI64,
27941 IX86_BUILTIN_CVTTPS2PI,
27942 IX86_BUILTIN_CVTTSS2SI,
27943 IX86_BUILTIN_CVTTSS2SI64,
27945 IX86_BUILTIN_MAXPS,
27946 IX86_BUILTIN_MAXSS,
27947 IX86_BUILTIN_MINPS,
27948 IX86_BUILTIN_MINSS,
27950 IX86_BUILTIN_LOADUPS,
27951 IX86_BUILTIN_STOREUPS,
27952 IX86_BUILTIN_MOVSS,
27954 IX86_BUILTIN_MOVHLPS,
27955 IX86_BUILTIN_MOVLHPS,
27956 IX86_BUILTIN_LOADHPS,
27957 IX86_BUILTIN_LOADLPS,
27958 IX86_BUILTIN_STOREHPS,
27959 IX86_BUILTIN_STORELPS,
27961 IX86_BUILTIN_MASKMOVQ,
27962 IX86_BUILTIN_MOVMSKPS,
27963 IX86_BUILTIN_PMOVMSKB,
27965 IX86_BUILTIN_MOVNTPS,
27966 IX86_BUILTIN_MOVNTQ,
27968 IX86_BUILTIN_LOADDQU,
27969 IX86_BUILTIN_STOREDQU,
27971 IX86_BUILTIN_PACKSSWB,
27972 IX86_BUILTIN_PACKSSDW,
27973 IX86_BUILTIN_PACKUSWB,
27975 IX86_BUILTIN_PADDB,
27976 IX86_BUILTIN_PADDW,
27977 IX86_BUILTIN_PADDD,
27978 IX86_BUILTIN_PADDQ,
27979 IX86_BUILTIN_PADDSB,
27980 IX86_BUILTIN_PADDSW,
27981 IX86_BUILTIN_PADDUSB,
27982 IX86_BUILTIN_PADDUSW,
27983 IX86_BUILTIN_PSUBB,
27984 IX86_BUILTIN_PSUBW,
27985 IX86_BUILTIN_PSUBD,
27986 IX86_BUILTIN_PSUBQ,
27987 IX86_BUILTIN_PSUBSB,
27988 IX86_BUILTIN_PSUBSW,
27989 IX86_BUILTIN_PSUBUSB,
27990 IX86_BUILTIN_PSUBUSW,
27993 IX86_BUILTIN_PANDN,
27997 IX86_BUILTIN_PAVGB,
27998 IX86_BUILTIN_PAVGW,
28000 IX86_BUILTIN_PCMPEQB,
28001 IX86_BUILTIN_PCMPEQW,
28002 IX86_BUILTIN_PCMPEQD,
28003 IX86_BUILTIN_PCMPGTB,
28004 IX86_BUILTIN_PCMPGTW,
28005 IX86_BUILTIN_PCMPGTD,
28007 IX86_BUILTIN_PMADDWD,
28009 IX86_BUILTIN_PMAXSW,
28010 IX86_BUILTIN_PMAXUB,
28011 IX86_BUILTIN_PMINSW,
28012 IX86_BUILTIN_PMINUB,
28014 IX86_BUILTIN_PMULHUW,
28015 IX86_BUILTIN_PMULHW,
28016 IX86_BUILTIN_PMULLW,
28018 IX86_BUILTIN_PSADBW,
28019 IX86_BUILTIN_PSHUFW,
28021 IX86_BUILTIN_PSLLW,
28022 IX86_BUILTIN_PSLLD,
28023 IX86_BUILTIN_PSLLQ,
28024 IX86_BUILTIN_PSRAW,
28025 IX86_BUILTIN_PSRAD,
28026 IX86_BUILTIN_PSRLW,
28027 IX86_BUILTIN_PSRLD,
28028 IX86_BUILTIN_PSRLQ,
28029 IX86_BUILTIN_PSLLWI,
28030 IX86_BUILTIN_PSLLDI,
28031 IX86_BUILTIN_PSLLQI,
28032 IX86_BUILTIN_PSRAWI,
28033 IX86_BUILTIN_PSRADI,
28034 IX86_BUILTIN_PSRLWI,
28035 IX86_BUILTIN_PSRLDI,
28036 IX86_BUILTIN_PSRLQI,
28038 IX86_BUILTIN_PUNPCKHBW,
28039 IX86_BUILTIN_PUNPCKHWD,
28040 IX86_BUILTIN_PUNPCKHDQ,
28041 IX86_BUILTIN_PUNPCKLBW,
28042 IX86_BUILTIN_PUNPCKLWD,
28043 IX86_BUILTIN_PUNPCKLDQ,
28045 IX86_BUILTIN_SHUFPS,
28047 IX86_BUILTIN_RCPPS,
28048 IX86_BUILTIN_RCPSS,
28049 IX86_BUILTIN_RSQRTPS,
28050 IX86_BUILTIN_RSQRTPS_NR,
28051 IX86_BUILTIN_RSQRTSS,
28052 IX86_BUILTIN_RSQRTF,
28053 IX86_BUILTIN_SQRTPS,
28054 IX86_BUILTIN_SQRTPS_NR,
28055 IX86_BUILTIN_SQRTSS,
28057 IX86_BUILTIN_UNPCKHPS,
28058 IX86_BUILTIN_UNPCKLPS,
28060 IX86_BUILTIN_ANDPS,
28061 IX86_BUILTIN_ANDNPS,
28063 IX86_BUILTIN_XORPS,
28066 IX86_BUILTIN_LDMXCSR,
28067 IX86_BUILTIN_STMXCSR,
28068 IX86_BUILTIN_SFENCE,
28070 IX86_BUILTIN_FXSAVE,
28071 IX86_BUILTIN_FXRSTOR,
28072 IX86_BUILTIN_FXSAVE64,
28073 IX86_BUILTIN_FXRSTOR64,
28075 IX86_BUILTIN_XSAVE,
28076 IX86_BUILTIN_XRSTOR,
28077 IX86_BUILTIN_XSAVE64,
28078 IX86_BUILTIN_XRSTOR64,
28080 IX86_BUILTIN_XSAVEOPT,
28081 IX86_BUILTIN_XSAVEOPT64,
28083 IX86_BUILTIN_XSAVEC,
28084 IX86_BUILTIN_XSAVEC64,
28086 IX86_BUILTIN_XSAVES,
28087 IX86_BUILTIN_XRSTORS,
28088 IX86_BUILTIN_XSAVES64,
28089 IX86_BUILTIN_XRSTORS64,
28091 /* 3DNow! Original */
28092 IX86_BUILTIN_FEMMS,
28093 IX86_BUILTIN_PAVGUSB,
28094 IX86_BUILTIN_PF2ID,
28095 IX86_BUILTIN_PFACC,
28096 IX86_BUILTIN_PFADD,
28097 IX86_BUILTIN_PFCMPEQ,
28098 IX86_BUILTIN_PFCMPGE,
28099 IX86_BUILTIN_PFCMPGT,
28100 IX86_BUILTIN_PFMAX,
28101 IX86_BUILTIN_PFMIN,
28102 IX86_BUILTIN_PFMUL,
28103 IX86_BUILTIN_PFRCP,
28104 IX86_BUILTIN_PFRCPIT1,
28105 IX86_BUILTIN_PFRCPIT2,
28106 IX86_BUILTIN_PFRSQIT1,
28107 IX86_BUILTIN_PFRSQRT,
28108 IX86_BUILTIN_PFSUB,
28109 IX86_BUILTIN_PFSUBR,
28110 IX86_BUILTIN_PI2FD,
28111 IX86_BUILTIN_PMULHRW,
28113 /* 3DNow! Athlon Extensions */
28114 IX86_BUILTIN_PF2IW,
28115 IX86_BUILTIN_PFNACC,
28116 IX86_BUILTIN_PFPNACC,
28117 IX86_BUILTIN_PI2FW,
28118 IX86_BUILTIN_PSWAPDSI,
28119 IX86_BUILTIN_PSWAPDSF,
28122 IX86_BUILTIN_ADDPD,
28123 IX86_BUILTIN_ADDSD,
28124 IX86_BUILTIN_DIVPD,
28125 IX86_BUILTIN_DIVSD,
28126 IX86_BUILTIN_MULPD,
28127 IX86_BUILTIN_MULSD,
28128 IX86_BUILTIN_SUBPD,
28129 IX86_BUILTIN_SUBSD,
28131 IX86_BUILTIN_CMPEQPD,
28132 IX86_BUILTIN_CMPLTPD,
28133 IX86_BUILTIN_CMPLEPD,
28134 IX86_BUILTIN_CMPGTPD,
28135 IX86_BUILTIN_CMPGEPD,
28136 IX86_BUILTIN_CMPNEQPD,
28137 IX86_BUILTIN_CMPNLTPD,
28138 IX86_BUILTIN_CMPNLEPD,
28139 IX86_BUILTIN_CMPNGTPD,
28140 IX86_BUILTIN_CMPNGEPD,
28141 IX86_BUILTIN_CMPORDPD,
28142 IX86_BUILTIN_CMPUNORDPD,
28143 IX86_BUILTIN_CMPEQSD,
28144 IX86_BUILTIN_CMPLTSD,
28145 IX86_BUILTIN_CMPLESD,
28146 IX86_BUILTIN_CMPNEQSD,
28147 IX86_BUILTIN_CMPNLTSD,
28148 IX86_BUILTIN_CMPNLESD,
28149 IX86_BUILTIN_CMPORDSD,
28150 IX86_BUILTIN_CMPUNORDSD,
28152 IX86_BUILTIN_COMIEQSD,
28153 IX86_BUILTIN_COMILTSD,
28154 IX86_BUILTIN_COMILESD,
28155 IX86_BUILTIN_COMIGTSD,
28156 IX86_BUILTIN_COMIGESD,
28157 IX86_BUILTIN_COMINEQSD,
28158 IX86_BUILTIN_UCOMIEQSD,
28159 IX86_BUILTIN_UCOMILTSD,
28160 IX86_BUILTIN_UCOMILESD,
28161 IX86_BUILTIN_UCOMIGTSD,
28162 IX86_BUILTIN_UCOMIGESD,
28163 IX86_BUILTIN_UCOMINEQSD,
28165 IX86_BUILTIN_MAXPD,
28166 IX86_BUILTIN_MAXSD,
28167 IX86_BUILTIN_MINPD,
28168 IX86_BUILTIN_MINSD,
28170 IX86_BUILTIN_ANDPD,
28171 IX86_BUILTIN_ANDNPD,
28173 IX86_BUILTIN_XORPD,
28175 IX86_BUILTIN_SQRTPD,
28176 IX86_BUILTIN_SQRTSD,
28178 IX86_BUILTIN_UNPCKHPD,
28179 IX86_BUILTIN_UNPCKLPD,
28181 IX86_BUILTIN_SHUFPD,
28183 IX86_BUILTIN_LOADUPD,
28184 IX86_BUILTIN_STOREUPD,
28185 IX86_BUILTIN_MOVSD,
28187 IX86_BUILTIN_LOADHPD,
28188 IX86_BUILTIN_LOADLPD,
28190 IX86_BUILTIN_CVTDQ2PD,
28191 IX86_BUILTIN_CVTDQ2PS,
28193 IX86_BUILTIN_CVTPD2DQ,
28194 IX86_BUILTIN_CVTPD2PI,
28195 IX86_BUILTIN_CVTPD2PS,
28196 IX86_BUILTIN_CVTTPD2DQ,
28197 IX86_BUILTIN_CVTTPD2PI,
28199 IX86_BUILTIN_CVTPI2PD,
28200 IX86_BUILTIN_CVTSI2SD,
28201 IX86_BUILTIN_CVTSI642SD,
28203 IX86_BUILTIN_CVTSD2SI,
28204 IX86_BUILTIN_CVTSD2SI64,
28205 IX86_BUILTIN_CVTSD2SS,
28206 IX86_BUILTIN_CVTSS2SD,
28207 IX86_BUILTIN_CVTTSD2SI,
28208 IX86_BUILTIN_CVTTSD2SI64,
28210 IX86_BUILTIN_CVTPS2DQ,
28211 IX86_BUILTIN_CVTPS2PD,
28212 IX86_BUILTIN_CVTTPS2DQ,
28214 IX86_BUILTIN_MOVNTI,
28215 IX86_BUILTIN_MOVNTI64,
28216 IX86_BUILTIN_MOVNTPD,
28217 IX86_BUILTIN_MOVNTDQ,
28219 IX86_BUILTIN_MOVQ128,
28222 IX86_BUILTIN_MASKMOVDQU,
28223 IX86_BUILTIN_MOVMSKPD,
28224 IX86_BUILTIN_PMOVMSKB128,
28226 IX86_BUILTIN_PACKSSWB128,
28227 IX86_BUILTIN_PACKSSDW128,
28228 IX86_BUILTIN_PACKUSWB128,
28230 IX86_BUILTIN_PADDB128,
28231 IX86_BUILTIN_PADDW128,
28232 IX86_BUILTIN_PADDD128,
28233 IX86_BUILTIN_PADDQ128,
28234 IX86_BUILTIN_PADDSB128,
28235 IX86_BUILTIN_PADDSW128,
28236 IX86_BUILTIN_PADDUSB128,
28237 IX86_BUILTIN_PADDUSW128,
28238 IX86_BUILTIN_PSUBB128,
28239 IX86_BUILTIN_PSUBW128,
28240 IX86_BUILTIN_PSUBD128,
28241 IX86_BUILTIN_PSUBQ128,
28242 IX86_BUILTIN_PSUBSB128,
28243 IX86_BUILTIN_PSUBSW128,
28244 IX86_BUILTIN_PSUBUSB128,
28245 IX86_BUILTIN_PSUBUSW128,
28247 IX86_BUILTIN_PAND128,
28248 IX86_BUILTIN_PANDN128,
28249 IX86_BUILTIN_POR128,
28250 IX86_BUILTIN_PXOR128,
28252 IX86_BUILTIN_PAVGB128,
28253 IX86_BUILTIN_PAVGW128,
28255 IX86_BUILTIN_PCMPEQB128,
28256 IX86_BUILTIN_PCMPEQW128,
28257 IX86_BUILTIN_PCMPEQD128,
28258 IX86_BUILTIN_PCMPGTB128,
28259 IX86_BUILTIN_PCMPGTW128,
28260 IX86_BUILTIN_PCMPGTD128,
28262 IX86_BUILTIN_PMADDWD128,
28264 IX86_BUILTIN_PMAXSW128,
28265 IX86_BUILTIN_PMAXUB128,
28266 IX86_BUILTIN_PMINSW128,
28267 IX86_BUILTIN_PMINUB128,
28269 IX86_BUILTIN_PMULUDQ,
28270 IX86_BUILTIN_PMULUDQ128,
28271 IX86_BUILTIN_PMULHUW128,
28272 IX86_BUILTIN_PMULHW128,
28273 IX86_BUILTIN_PMULLW128,
28275 IX86_BUILTIN_PSADBW128,
28276 IX86_BUILTIN_PSHUFHW,
28277 IX86_BUILTIN_PSHUFLW,
28278 IX86_BUILTIN_PSHUFD,
28280 IX86_BUILTIN_PSLLDQI128,
28281 IX86_BUILTIN_PSLLWI128,
28282 IX86_BUILTIN_PSLLDI128,
28283 IX86_BUILTIN_PSLLQI128,
28284 IX86_BUILTIN_PSRAWI128,
28285 IX86_BUILTIN_PSRADI128,
28286 IX86_BUILTIN_PSRLDQI128,
28287 IX86_BUILTIN_PSRLWI128,
28288 IX86_BUILTIN_PSRLDI128,
28289 IX86_BUILTIN_PSRLQI128,
28291 IX86_BUILTIN_PSLLDQ128,
28292 IX86_BUILTIN_PSLLW128,
28293 IX86_BUILTIN_PSLLD128,
28294 IX86_BUILTIN_PSLLQ128,
28295 IX86_BUILTIN_PSRAW128,
28296 IX86_BUILTIN_PSRAD128,
28297 IX86_BUILTIN_PSRLW128,
28298 IX86_BUILTIN_PSRLD128,
28299 IX86_BUILTIN_PSRLQ128,
28301 IX86_BUILTIN_PUNPCKHBW128,
28302 IX86_BUILTIN_PUNPCKHWD128,
28303 IX86_BUILTIN_PUNPCKHDQ128,
28304 IX86_BUILTIN_PUNPCKHQDQ128,
28305 IX86_BUILTIN_PUNPCKLBW128,
28306 IX86_BUILTIN_PUNPCKLWD128,
28307 IX86_BUILTIN_PUNPCKLDQ128,
28308 IX86_BUILTIN_PUNPCKLQDQ128,
28310 IX86_BUILTIN_CLFLUSH,
28311 IX86_BUILTIN_MFENCE,
28312 IX86_BUILTIN_LFENCE,
28313 IX86_BUILTIN_PAUSE,
28315 IX86_BUILTIN_FNSTENV,
28316 IX86_BUILTIN_FLDENV,
28317 IX86_BUILTIN_FNSTSW,
28318 IX86_BUILTIN_FNCLEX,
28320 IX86_BUILTIN_BSRSI,
28321 IX86_BUILTIN_BSRDI,
28322 IX86_BUILTIN_RDPMC,
28323 IX86_BUILTIN_RDTSC,
28324 IX86_BUILTIN_RDTSCP,
28325 IX86_BUILTIN_ROLQI,
28326 IX86_BUILTIN_ROLHI,
28327 IX86_BUILTIN_RORQI,
28328 IX86_BUILTIN_RORHI,
28331 IX86_BUILTIN_ADDSUBPS,
28332 IX86_BUILTIN_HADDPS,
28333 IX86_BUILTIN_HSUBPS,
28334 IX86_BUILTIN_MOVSHDUP,
28335 IX86_BUILTIN_MOVSLDUP,
28336 IX86_BUILTIN_ADDSUBPD,
28337 IX86_BUILTIN_HADDPD,
28338 IX86_BUILTIN_HSUBPD,
28339 IX86_BUILTIN_LDDQU,
28341 IX86_BUILTIN_MONITOR,
28342 IX86_BUILTIN_MWAIT,
28345 IX86_BUILTIN_PHADDW,
28346 IX86_BUILTIN_PHADDD,
28347 IX86_BUILTIN_PHADDSW,
28348 IX86_BUILTIN_PHSUBW,
28349 IX86_BUILTIN_PHSUBD,
28350 IX86_BUILTIN_PHSUBSW,
28351 IX86_BUILTIN_PMADDUBSW,
28352 IX86_BUILTIN_PMULHRSW,
28353 IX86_BUILTIN_PSHUFB,
28354 IX86_BUILTIN_PSIGNB,
28355 IX86_BUILTIN_PSIGNW,
28356 IX86_BUILTIN_PSIGND,
28357 IX86_BUILTIN_PALIGNR,
28358 IX86_BUILTIN_PABSB,
28359 IX86_BUILTIN_PABSW,
28360 IX86_BUILTIN_PABSD,
28362 IX86_BUILTIN_PHADDW128,
28363 IX86_BUILTIN_PHADDD128,
28364 IX86_BUILTIN_PHADDSW128,
28365 IX86_BUILTIN_PHSUBW128,
28366 IX86_BUILTIN_PHSUBD128,
28367 IX86_BUILTIN_PHSUBSW128,
28368 IX86_BUILTIN_PMADDUBSW128,
28369 IX86_BUILTIN_PMULHRSW128,
28370 IX86_BUILTIN_PSHUFB128,
28371 IX86_BUILTIN_PSIGNB128,
28372 IX86_BUILTIN_PSIGNW128,
28373 IX86_BUILTIN_PSIGND128,
28374 IX86_BUILTIN_PALIGNR128,
28375 IX86_BUILTIN_PABSB128,
28376 IX86_BUILTIN_PABSW128,
28377 IX86_BUILTIN_PABSD128,
28379 /* AMDFAM10 - SSE4A New Instructions. */
28380 IX86_BUILTIN_MOVNTSD,
28381 IX86_BUILTIN_MOVNTSS,
28382 IX86_BUILTIN_EXTRQI,
28383 IX86_BUILTIN_EXTRQ,
28384 IX86_BUILTIN_INSERTQI,
28385 IX86_BUILTIN_INSERTQ,
28388 IX86_BUILTIN_BLENDPD,
28389 IX86_BUILTIN_BLENDPS,
28390 IX86_BUILTIN_BLENDVPD,
28391 IX86_BUILTIN_BLENDVPS,
28392 IX86_BUILTIN_PBLENDVB128,
28393 IX86_BUILTIN_PBLENDW128,
28398 IX86_BUILTIN_INSERTPS128,
28400 IX86_BUILTIN_MOVNTDQA,
28401 IX86_BUILTIN_MPSADBW128,
28402 IX86_BUILTIN_PACKUSDW128,
28403 IX86_BUILTIN_PCMPEQQ,
28404 IX86_BUILTIN_PHMINPOSUW128,
28406 IX86_BUILTIN_PMAXSB128,
28407 IX86_BUILTIN_PMAXSD128,
28408 IX86_BUILTIN_PMAXUD128,
28409 IX86_BUILTIN_PMAXUW128,
28411 IX86_BUILTIN_PMINSB128,
28412 IX86_BUILTIN_PMINSD128,
28413 IX86_BUILTIN_PMINUD128,
28414 IX86_BUILTIN_PMINUW128,
28416 IX86_BUILTIN_PMOVSXBW128,
28417 IX86_BUILTIN_PMOVSXBD128,
28418 IX86_BUILTIN_PMOVSXBQ128,
28419 IX86_BUILTIN_PMOVSXWD128,
28420 IX86_BUILTIN_PMOVSXWQ128,
28421 IX86_BUILTIN_PMOVSXDQ128,
28423 IX86_BUILTIN_PMOVZXBW128,
28424 IX86_BUILTIN_PMOVZXBD128,
28425 IX86_BUILTIN_PMOVZXBQ128,
28426 IX86_BUILTIN_PMOVZXWD128,
28427 IX86_BUILTIN_PMOVZXWQ128,
28428 IX86_BUILTIN_PMOVZXDQ128,
28430 IX86_BUILTIN_PMULDQ128,
28431 IX86_BUILTIN_PMULLD128,
28433 IX86_BUILTIN_ROUNDSD,
28434 IX86_BUILTIN_ROUNDSS,
28436 IX86_BUILTIN_ROUNDPD,
28437 IX86_BUILTIN_ROUNDPS,
28439 IX86_BUILTIN_FLOORPD,
28440 IX86_BUILTIN_CEILPD,
28441 IX86_BUILTIN_TRUNCPD,
28442 IX86_BUILTIN_RINTPD,
28443 IX86_BUILTIN_ROUNDPD_AZ,
28445 IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX,
28446 IX86_BUILTIN_CEILPD_VEC_PACK_SFIX,
28447 IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX,
28449 IX86_BUILTIN_FLOORPS,
28450 IX86_BUILTIN_CEILPS,
28451 IX86_BUILTIN_TRUNCPS,
28452 IX86_BUILTIN_RINTPS,
28453 IX86_BUILTIN_ROUNDPS_AZ,
28455 IX86_BUILTIN_FLOORPS_SFIX,
28456 IX86_BUILTIN_CEILPS_SFIX,
28457 IX86_BUILTIN_ROUNDPS_AZ_SFIX,
28459 IX86_BUILTIN_PTESTZ,
28460 IX86_BUILTIN_PTESTC,
28461 IX86_BUILTIN_PTESTNZC,
28463 IX86_BUILTIN_VEC_INIT_V2SI,
28464 IX86_BUILTIN_VEC_INIT_V4HI,
28465 IX86_BUILTIN_VEC_INIT_V8QI,
28466 IX86_BUILTIN_VEC_EXT_V2DF,
28467 IX86_BUILTIN_VEC_EXT_V2DI,
28468 IX86_BUILTIN_VEC_EXT_V4SF,
28469 IX86_BUILTIN_VEC_EXT_V4SI,
28470 IX86_BUILTIN_VEC_EXT_V8HI,
28471 IX86_BUILTIN_VEC_EXT_V2SI,
28472 IX86_BUILTIN_VEC_EXT_V4HI,
28473 IX86_BUILTIN_VEC_EXT_V16QI,
28474 IX86_BUILTIN_VEC_SET_V2DI,
28475 IX86_BUILTIN_VEC_SET_V4SF,
28476 IX86_BUILTIN_VEC_SET_V4SI,
28477 IX86_BUILTIN_VEC_SET_V8HI,
28478 IX86_BUILTIN_VEC_SET_V4HI,
28479 IX86_BUILTIN_VEC_SET_V16QI,
28481 IX86_BUILTIN_VEC_PACK_SFIX,
28482 IX86_BUILTIN_VEC_PACK_SFIX256,
28485 IX86_BUILTIN_CRC32QI,
28486 IX86_BUILTIN_CRC32HI,
28487 IX86_BUILTIN_CRC32SI,
28488 IX86_BUILTIN_CRC32DI,
28490 IX86_BUILTIN_PCMPESTRI128,
28491 IX86_BUILTIN_PCMPESTRM128,
28492 IX86_BUILTIN_PCMPESTRA128,
28493 IX86_BUILTIN_PCMPESTRC128,
28494 IX86_BUILTIN_PCMPESTRO128,
28495 IX86_BUILTIN_PCMPESTRS128,
28496 IX86_BUILTIN_PCMPESTRZ128,
28497 IX86_BUILTIN_PCMPISTRI128,
28498 IX86_BUILTIN_PCMPISTRM128,
28499 IX86_BUILTIN_PCMPISTRA128,
28500 IX86_BUILTIN_PCMPISTRC128,
28501 IX86_BUILTIN_PCMPISTRO128,
28502 IX86_BUILTIN_PCMPISTRS128,
28503 IX86_BUILTIN_PCMPISTRZ128,
28505 IX86_BUILTIN_PCMPGTQ,
28507 /* AES instructions */
28508 IX86_BUILTIN_AESENC128,
28509 IX86_BUILTIN_AESENCLAST128,
28510 IX86_BUILTIN_AESDEC128,
28511 IX86_BUILTIN_AESDECLAST128,
28512 IX86_BUILTIN_AESIMC128,
28513 IX86_BUILTIN_AESKEYGENASSIST128,
28515 /* PCLMUL instruction */
28516 IX86_BUILTIN_PCLMULQDQ128,
28519 IX86_BUILTIN_ADDPD256,
28520 IX86_BUILTIN_ADDPS256,
28521 IX86_BUILTIN_ADDSUBPD256,
28522 IX86_BUILTIN_ADDSUBPS256,
28523 IX86_BUILTIN_ANDPD256,
28524 IX86_BUILTIN_ANDPS256,
28525 IX86_BUILTIN_ANDNPD256,
28526 IX86_BUILTIN_ANDNPS256,
28527 IX86_BUILTIN_BLENDPD256,
28528 IX86_BUILTIN_BLENDPS256,
28529 IX86_BUILTIN_BLENDVPD256,
28530 IX86_BUILTIN_BLENDVPS256,
28531 IX86_BUILTIN_DIVPD256,
28532 IX86_BUILTIN_DIVPS256,
28533 IX86_BUILTIN_DPPS256,
28534 IX86_BUILTIN_HADDPD256,
28535 IX86_BUILTIN_HADDPS256,
28536 IX86_BUILTIN_HSUBPD256,
28537 IX86_BUILTIN_HSUBPS256,
28538 IX86_BUILTIN_MAXPD256,
28539 IX86_BUILTIN_MAXPS256,
28540 IX86_BUILTIN_MINPD256,
28541 IX86_BUILTIN_MINPS256,
28542 IX86_BUILTIN_MULPD256,
28543 IX86_BUILTIN_MULPS256,
28544 IX86_BUILTIN_ORPD256,
28545 IX86_BUILTIN_ORPS256,
28546 IX86_BUILTIN_SHUFPD256,
28547 IX86_BUILTIN_SHUFPS256,
28548 IX86_BUILTIN_SUBPD256,
28549 IX86_BUILTIN_SUBPS256,
28550 IX86_BUILTIN_XORPD256,
28551 IX86_BUILTIN_XORPS256,
28552 IX86_BUILTIN_CMPSD,
28553 IX86_BUILTIN_CMPSS,
28554 IX86_BUILTIN_CMPPD,
28555 IX86_BUILTIN_CMPPS,
28556 IX86_BUILTIN_CMPPD256,
28557 IX86_BUILTIN_CMPPS256,
28558 IX86_BUILTIN_CVTDQ2PD256,
28559 IX86_BUILTIN_CVTDQ2PS256,
28560 IX86_BUILTIN_CVTPD2PS256,
28561 IX86_BUILTIN_CVTPS2DQ256,
28562 IX86_BUILTIN_CVTPS2PD256,
28563 IX86_BUILTIN_CVTTPD2DQ256,
28564 IX86_BUILTIN_CVTPD2DQ256,
28565 IX86_BUILTIN_CVTTPS2DQ256,
28566 IX86_BUILTIN_EXTRACTF128PD256,
28567 IX86_BUILTIN_EXTRACTF128PS256,
28568 IX86_BUILTIN_EXTRACTF128SI256,
28569 IX86_BUILTIN_VZEROALL,
28570 IX86_BUILTIN_VZEROUPPER,
28571 IX86_BUILTIN_VPERMILVARPD,
28572 IX86_BUILTIN_VPERMILVARPS,
28573 IX86_BUILTIN_VPERMILVARPD256,
28574 IX86_BUILTIN_VPERMILVARPS256,
28575 IX86_BUILTIN_VPERMILPD,
28576 IX86_BUILTIN_VPERMILPS,
28577 IX86_BUILTIN_VPERMILPD256,
28578 IX86_BUILTIN_VPERMILPS256,
28579 IX86_BUILTIN_VPERMIL2PD,
28580 IX86_BUILTIN_VPERMIL2PS,
28581 IX86_BUILTIN_VPERMIL2PD256,
28582 IX86_BUILTIN_VPERMIL2PS256,
28583 IX86_BUILTIN_VPERM2F128PD256,
28584 IX86_BUILTIN_VPERM2F128PS256,
28585 IX86_BUILTIN_VPERM2F128SI256,
28586 IX86_BUILTIN_VBROADCASTSS,
28587 IX86_BUILTIN_VBROADCASTSD256,
28588 IX86_BUILTIN_VBROADCASTSS256,
28589 IX86_BUILTIN_VBROADCASTPD256,
28590 IX86_BUILTIN_VBROADCASTPS256,
28591 IX86_BUILTIN_VINSERTF128PD256,
28592 IX86_BUILTIN_VINSERTF128PS256,
28593 IX86_BUILTIN_VINSERTF128SI256,
28594 IX86_BUILTIN_LOADUPD256,
28595 IX86_BUILTIN_LOADUPS256,
28596 IX86_BUILTIN_STOREUPD256,
28597 IX86_BUILTIN_STOREUPS256,
28598 IX86_BUILTIN_LDDQU256,
28599 IX86_BUILTIN_MOVNTDQ256,
28600 IX86_BUILTIN_MOVNTPD256,
28601 IX86_BUILTIN_MOVNTPS256,
28602 IX86_BUILTIN_LOADDQU256,
28603 IX86_BUILTIN_STOREDQU256,
28604 IX86_BUILTIN_MASKLOADPD,
28605 IX86_BUILTIN_MASKLOADPS,
28606 IX86_BUILTIN_MASKSTOREPD,
28607 IX86_BUILTIN_MASKSTOREPS,
28608 IX86_BUILTIN_MASKLOADPD256,
28609 IX86_BUILTIN_MASKLOADPS256,
28610 IX86_BUILTIN_MASKSTOREPD256,
28611 IX86_BUILTIN_MASKSTOREPS256,
28612 IX86_BUILTIN_MOVSHDUP256,
28613 IX86_BUILTIN_MOVSLDUP256,
28614 IX86_BUILTIN_MOVDDUP256,
28616 IX86_BUILTIN_SQRTPD256,
28617 IX86_BUILTIN_SQRTPS256,
28618 IX86_BUILTIN_SQRTPS_NR256,
28619 IX86_BUILTIN_RSQRTPS256,
28620 IX86_BUILTIN_RSQRTPS_NR256,
28622 IX86_BUILTIN_RCPPS256,
28624 IX86_BUILTIN_ROUNDPD256,
28625 IX86_BUILTIN_ROUNDPS256,
28627 IX86_BUILTIN_FLOORPD256,
28628 IX86_BUILTIN_CEILPD256,
28629 IX86_BUILTIN_TRUNCPD256,
28630 IX86_BUILTIN_RINTPD256,
28631 IX86_BUILTIN_ROUNDPD_AZ256,
28633 IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX256,
28634 IX86_BUILTIN_CEILPD_VEC_PACK_SFIX256,
28635 IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX256,
28637 IX86_BUILTIN_FLOORPS256,
28638 IX86_BUILTIN_CEILPS256,
28639 IX86_BUILTIN_TRUNCPS256,
28640 IX86_BUILTIN_RINTPS256,
28641 IX86_BUILTIN_ROUNDPS_AZ256,
28643 IX86_BUILTIN_FLOORPS_SFIX256,
28644 IX86_BUILTIN_CEILPS_SFIX256,
28645 IX86_BUILTIN_ROUNDPS_AZ_SFIX256,
28647 IX86_BUILTIN_UNPCKHPD256,
28648 IX86_BUILTIN_UNPCKLPD256,
28649 IX86_BUILTIN_UNPCKHPS256,
28650 IX86_BUILTIN_UNPCKLPS256,
28652 IX86_BUILTIN_SI256_SI,
28653 IX86_BUILTIN_PS256_PS,
28654 IX86_BUILTIN_PD256_PD,
28655 IX86_BUILTIN_SI_SI256,
28656 IX86_BUILTIN_PS_PS256,
28657 IX86_BUILTIN_PD_PD256,
28659 IX86_BUILTIN_VTESTZPD,
28660 IX86_BUILTIN_VTESTCPD,
28661 IX86_BUILTIN_VTESTNZCPD,
28662 IX86_BUILTIN_VTESTZPS,
28663 IX86_BUILTIN_VTESTCPS,
28664 IX86_BUILTIN_VTESTNZCPS,
28665 IX86_BUILTIN_VTESTZPD256,
28666 IX86_BUILTIN_VTESTCPD256,
28667 IX86_BUILTIN_VTESTNZCPD256,
28668 IX86_BUILTIN_VTESTZPS256,
28669 IX86_BUILTIN_VTESTCPS256,
28670 IX86_BUILTIN_VTESTNZCPS256,
28671 IX86_BUILTIN_PTESTZ256,
28672 IX86_BUILTIN_PTESTC256,
28673 IX86_BUILTIN_PTESTNZC256,
28675 IX86_BUILTIN_MOVMSKPD256,
28676 IX86_BUILTIN_MOVMSKPS256,
28679 IX86_BUILTIN_MPSADBW256,
28680 IX86_BUILTIN_PABSB256,
28681 IX86_BUILTIN_PABSW256,
28682 IX86_BUILTIN_PABSD256,
28683 IX86_BUILTIN_PACKSSDW256,
28684 IX86_BUILTIN_PACKSSWB256,
28685 IX86_BUILTIN_PACKUSDW256,
28686 IX86_BUILTIN_PACKUSWB256,
28687 IX86_BUILTIN_PADDB256,
28688 IX86_BUILTIN_PADDW256,
28689 IX86_BUILTIN_PADDD256,
28690 IX86_BUILTIN_PADDQ256,
28691 IX86_BUILTIN_PADDSB256,
28692 IX86_BUILTIN_PADDSW256,
28693 IX86_BUILTIN_PADDUSB256,
28694 IX86_BUILTIN_PADDUSW256,
28695 IX86_BUILTIN_PALIGNR256,
28696 IX86_BUILTIN_AND256I,
28697 IX86_BUILTIN_ANDNOT256I,
28698 IX86_BUILTIN_PAVGB256,
28699 IX86_BUILTIN_PAVGW256,
28700 IX86_BUILTIN_PBLENDVB256,
28701 IX86_BUILTIN_PBLENDVW256,
28702 IX86_BUILTIN_PCMPEQB256,
28703 IX86_BUILTIN_PCMPEQW256,
28704 IX86_BUILTIN_PCMPEQD256,
28705 IX86_BUILTIN_PCMPEQQ256,
28706 IX86_BUILTIN_PCMPGTB256,
28707 IX86_BUILTIN_PCMPGTW256,
28708 IX86_BUILTIN_PCMPGTD256,
28709 IX86_BUILTIN_PCMPGTQ256,
28710 IX86_BUILTIN_PHADDW256,
28711 IX86_BUILTIN_PHADDD256,
28712 IX86_BUILTIN_PHADDSW256,
28713 IX86_BUILTIN_PHSUBW256,
28714 IX86_BUILTIN_PHSUBD256,
28715 IX86_BUILTIN_PHSUBSW256,
28716 IX86_BUILTIN_PMADDUBSW256,
28717 IX86_BUILTIN_PMADDWD256,
28718 IX86_BUILTIN_PMAXSB256,
28719 IX86_BUILTIN_PMAXSW256,
28720 IX86_BUILTIN_PMAXSD256,
28721 IX86_BUILTIN_PMAXUB256,
28722 IX86_BUILTIN_PMAXUW256,
28723 IX86_BUILTIN_PMAXUD256,
28724 IX86_BUILTIN_PMINSB256,
28725 IX86_BUILTIN_PMINSW256,
28726 IX86_BUILTIN_PMINSD256,
28727 IX86_BUILTIN_PMINUB256,
28728 IX86_BUILTIN_PMINUW256,
28729 IX86_BUILTIN_PMINUD256,
28730 IX86_BUILTIN_PMOVMSKB256,
28731 IX86_BUILTIN_PMOVSXBW256,
28732 IX86_BUILTIN_PMOVSXBD256,
28733 IX86_BUILTIN_PMOVSXBQ256,
28734 IX86_BUILTIN_PMOVSXWD256,
28735 IX86_BUILTIN_PMOVSXWQ256,
28736 IX86_BUILTIN_PMOVSXDQ256,
28737 IX86_BUILTIN_PMOVZXBW256,
28738 IX86_BUILTIN_PMOVZXBD256,
28739 IX86_BUILTIN_PMOVZXBQ256,
28740 IX86_BUILTIN_PMOVZXWD256,
28741 IX86_BUILTIN_PMOVZXWQ256,
28742 IX86_BUILTIN_PMOVZXDQ256,
28743 IX86_BUILTIN_PMULDQ256,
28744 IX86_BUILTIN_PMULHRSW256,
28745 IX86_BUILTIN_PMULHUW256,
28746 IX86_BUILTIN_PMULHW256,
28747 IX86_BUILTIN_PMULLW256,
28748 IX86_BUILTIN_PMULLD256,
28749 IX86_BUILTIN_PMULUDQ256,
28750 IX86_BUILTIN_POR256,
28751 IX86_BUILTIN_PSADBW256,
28752 IX86_BUILTIN_PSHUFB256,
28753 IX86_BUILTIN_PSHUFD256,
28754 IX86_BUILTIN_PSHUFHW256,
28755 IX86_BUILTIN_PSHUFLW256,
28756 IX86_BUILTIN_PSIGNB256,
28757 IX86_BUILTIN_PSIGNW256,
28758 IX86_BUILTIN_PSIGND256,
28759 IX86_BUILTIN_PSLLDQI256,
28760 IX86_BUILTIN_PSLLWI256,
28761 IX86_BUILTIN_PSLLW256,
28762 IX86_BUILTIN_PSLLDI256,
28763 IX86_BUILTIN_PSLLD256,
28764 IX86_BUILTIN_PSLLQI256,
28765 IX86_BUILTIN_PSLLQ256,
28766 IX86_BUILTIN_PSRAWI256,
28767 IX86_BUILTIN_PSRAW256,
28768 IX86_BUILTIN_PSRADI256,
28769 IX86_BUILTIN_PSRAD256,
28770 IX86_BUILTIN_PSRLDQI256,
28771 IX86_BUILTIN_PSRLWI256,
28772 IX86_BUILTIN_PSRLW256,
28773 IX86_BUILTIN_PSRLDI256,
28774 IX86_BUILTIN_PSRLD256,
28775 IX86_BUILTIN_PSRLQI256,
28776 IX86_BUILTIN_PSRLQ256,
28777 IX86_BUILTIN_PSUBB256,
28778 IX86_BUILTIN_PSUBW256,
28779 IX86_BUILTIN_PSUBD256,
28780 IX86_BUILTIN_PSUBQ256,
28781 IX86_BUILTIN_PSUBSB256,
28782 IX86_BUILTIN_PSUBSW256,
28783 IX86_BUILTIN_PSUBUSB256,
28784 IX86_BUILTIN_PSUBUSW256,
28785 IX86_BUILTIN_PUNPCKHBW256,
28786 IX86_BUILTIN_PUNPCKHWD256,
28787 IX86_BUILTIN_PUNPCKHDQ256,
28788 IX86_BUILTIN_PUNPCKHQDQ256,
28789 IX86_BUILTIN_PUNPCKLBW256,
28790 IX86_BUILTIN_PUNPCKLWD256,
28791 IX86_BUILTIN_PUNPCKLDQ256,
28792 IX86_BUILTIN_PUNPCKLQDQ256,
28793 IX86_BUILTIN_PXOR256,
28794 IX86_BUILTIN_MOVNTDQA256,
28795 IX86_BUILTIN_VBROADCASTSS_PS,
28796 IX86_BUILTIN_VBROADCASTSS_PS256,
28797 IX86_BUILTIN_VBROADCASTSD_PD256,
28798 IX86_BUILTIN_VBROADCASTSI256,
28799 IX86_BUILTIN_PBLENDD256,
28800 IX86_BUILTIN_PBLENDD128,
28801 IX86_BUILTIN_PBROADCASTB256,
28802 IX86_BUILTIN_PBROADCASTW256,
28803 IX86_BUILTIN_PBROADCASTD256,
28804 IX86_BUILTIN_PBROADCASTQ256,
28805 IX86_BUILTIN_PBROADCASTB128,
28806 IX86_BUILTIN_PBROADCASTW128,
28807 IX86_BUILTIN_PBROADCASTD128,
28808 IX86_BUILTIN_PBROADCASTQ128,
28809 IX86_BUILTIN_VPERMVARSI256,
28810 IX86_BUILTIN_VPERMDF256,
28811 IX86_BUILTIN_VPERMVARSF256,
28812 IX86_BUILTIN_VPERMDI256,
28813 IX86_BUILTIN_VPERMTI256,
28814 IX86_BUILTIN_VEXTRACT128I256,
28815 IX86_BUILTIN_VINSERT128I256,
28816 IX86_BUILTIN_MASKLOADD,
28817 IX86_BUILTIN_MASKLOADQ,
28818 IX86_BUILTIN_MASKLOADD256,
28819 IX86_BUILTIN_MASKLOADQ256,
28820 IX86_BUILTIN_MASKSTORED,
28821 IX86_BUILTIN_MASKSTOREQ,
28822 IX86_BUILTIN_MASKSTORED256,
28823 IX86_BUILTIN_MASKSTOREQ256,
28824 IX86_BUILTIN_PSLLVV4DI,
28825 IX86_BUILTIN_PSLLVV2DI,
28826 IX86_BUILTIN_PSLLVV8SI,
28827 IX86_BUILTIN_PSLLVV4SI,
28828 IX86_BUILTIN_PSRAVV8SI,
28829 IX86_BUILTIN_PSRAVV4SI,
28830 IX86_BUILTIN_PSRLVV4DI,
28831 IX86_BUILTIN_PSRLVV2DI,
28832 IX86_BUILTIN_PSRLVV8SI,
28833 IX86_BUILTIN_PSRLVV4SI,
28835 IX86_BUILTIN_GATHERSIV2DF,
28836 IX86_BUILTIN_GATHERSIV4DF,
28837 IX86_BUILTIN_GATHERDIV2DF,
28838 IX86_BUILTIN_GATHERDIV4DF,
28839 IX86_BUILTIN_GATHERSIV4SF,
28840 IX86_BUILTIN_GATHERSIV8SF,
28841 IX86_BUILTIN_GATHERDIV4SF,
28842 IX86_BUILTIN_GATHERDIV8SF,
28843 IX86_BUILTIN_GATHERSIV2DI,
28844 IX86_BUILTIN_GATHERSIV4DI,
28845 IX86_BUILTIN_GATHERDIV2DI,
28846 IX86_BUILTIN_GATHERDIV4DI,
28847 IX86_BUILTIN_GATHERSIV4SI,
28848 IX86_BUILTIN_GATHERSIV8SI,
28849 IX86_BUILTIN_GATHERDIV4SI,
28850 IX86_BUILTIN_GATHERDIV8SI,
28853 IX86_BUILTIN_SI512_SI256,
28854 IX86_BUILTIN_PD512_PD256,
28855 IX86_BUILTIN_PS512_PS256,
28856 IX86_BUILTIN_SI512_SI,
28857 IX86_BUILTIN_PD512_PD,
28858 IX86_BUILTIN_PS512_PS,
28859 IX86_BUILTIN_ADDPD512,
28860 IX86_BUILTIN_ADDPS512,
28861 IX86_BUILTIN_ADDSD_ROUND,
28862 IX86_BUILTIN_ADDSS_ROUND,
28863 IX86_BUILTIN_ALIGND512,
28864 IX86_BUILTIN_ALIGNQ512,
28865 IX86_BUILTIN_BLENDMD512,
28866 IX86_BUILTIN_BLENDMPD512,
28867 IX86_BUILTIN_BLENDMPS512,
28868 IX86_BUILTIN_BLENDMQ512,
28869 IX86_BUILTIN_BROADCASTF32X4_512,
28870 IX86_BUILTIN_BROADCASTF64X4_512,
28871 IX86_BUILTIN_BROADCASTI32X4_512,
28872 IX86_BUILTIN_BROADCASTI64X4_512,
28873 IX86_BUILTIN_BROADCASTSD512,
28874 IX86_BUILTIN_BROADCASTSS512,
28875 IX86_BUILTIN_CMPD512,
28876 IX86_BUILTIN_CMPPD512,
28877 IX86_BUILTIN_CMPPS512,
28878 IX86_BUILTIN_CMPQ512,
28879 IX86_BUILTIN_CMPSD_MASK,
28880 IX86_BUILTIN_CMPSS_MASK,
28881 IX86_BUILTIN_COMIDF,
28882 IX86_BUILTIN_COMISF,
28883 IX86_BUILTIN_COMPRESSPD512,
28884 IX86_BUILTIN_COMPRESSPDSTORE512,
28885 IX86_BUILTIN_COMPRESSPS512,
28886 IX86_BUILTIN_COMPRESSPSSTORE512,
28887 IX86_BUILTIN_CVTDQ2PD512,
28888 IX86_BUILTIN_CVTDQ2PS512,
28889 IX86_BUILTIN_CVTPD2DQ512,
28890 IX86_BUILTIN_CVTPD2PS512,
28891 IX86_BUILTIN_CVTPD2UDQ512,
28892 IX86_BUILTIN_CVTPH2PS512,
28893 IX86_BUILTIN_CVTPS2DQ512,
28894 IX86_BUILTIN_CVTPS2PD512,
28895 IX86_BUILTIN_CVTPS2PH512,
28896 IX86_BUILTIN_CVTPS2UDQ512,
28897 IX86_BUILTIN_CVTSD2SS_ROUND,
28898 IX86_BUILTIN_CVTSI2SD64,
28899 IX86_BUILTIN_CVTSI2SS32,
28900 IX86_BUILTIN_CVTSI2SS64,
28901 IX86_BUILTIN_CVTSS2SD_ROUND,
28902 IX86_BUILTIN_CVTTPD2DQ512,
28903 IX86_BUILTIN_CVTTPD2UDQ512,
28904 IX86_BUILTIN_CVTTPS2DQ512,
28905 IX86_BUILTIN_CVTTPS2UDQ512,
28906 IX86_BUILTIN_CVTUDQ2PD512,
28907 IX86_BUILTIN_CVTUDQ2PS512,
28908 IX86_BUILTIN_CVTUSI2SD32,
28909 IX86_BUILTIN_CVTUSI2SD64,
28910 IX86_BUILTIN_CVTUSI2SS32,
28911 IX86_BUILTIN_CVTUSI2SS64,
28912 IX86_BUILTIN_DIVPD512,
28913 IX86_BUILTIN_DIVPS512,
28914 IX86_BUILTIN_DIVSD_ROUND,
28915 IX86_BUILTIN_DIVSS_ROUND,
28916 IX86_BUILTIN_EXPANDPD512,
28917 IX86_BUILTIN_EXPANDPD512Z,
28918 IX86_BUILTIN_EXPANDPDLOAD512,
28919 IX86_BUILTIN_EXPANDPDLOAD512Z,
28920 IX86_BUILTIN_EXPANDPS512,
28921 IX86_BUILTIN_EXPANDPS512Z,
28922 IX86_BUILTIN_EXPANDPSLOAD512,
28923 IX86_BUILTIN_EXPANDPSLOAD512Z,
28924 IX86_BUILTIN_EXTRACTF32X4,
28925 IX86_BUILTIN_EXTRACTF64X4,
28926 IX86_BUILTIN_EXTRACTI32X4,
28927 IX86_BUILTIN_EXTRACTI64X4,
28928 IX86_BUILTIN_FIXUPIMMPD512_MASK,
28929 IX86_BUILTIN_FIXUPIMMPD512_MASKZ,
28930 IX86_BUILTIN_FIXUPIMMPS512_MASK,
28931 IX86_BUILTIN_FIXUPIMMPS512_MASKZ,
28932 IX86_BUILTIN_FIXUPIMMSD128_MASK,
28933 IX86_BUILTIN_FIXUPIMMSD128_MASKZ,
28934 IX86_BUILTIN_FIXUPIMMSS128_MASK,
28935 IX86_BUILTIN_FIXUPIMMSS128_MASKZ,
28936 IX86_BUILTIN_GETEXPPD512,
28937 IX86_BUILTIN_GETEXPPS512,
28938 IX86_BUILTIN_GETEXPSD128,
28939 IX86_BUILTIN_GETEXPSS128,
28940 IX86_BUILTIN_GETMANTPD512,
28941 IX86_BUILTIN_GETMANTPS512,
28942 IX86_BUILTIN_GETMANTSD128,
28943 IX86_BUILTIN_GETMANTSS128,
28944 IX86_BUILTIN_INSERTF32X4,
28945 IX86_BUILTIN_INSERTF64X4,
28946 IX86_BUILTIN_INSERTI32X4,
28947 IX86_BUILTIN_INSERTI64X4,
28948 IX86_BUILTIN_LOADAPD512,
28949 IX86_BUILTIN_LOADAPS512,
28950 IX86_BUILTIN_LOADDQUDI512,
28951 IX86_BUILTIN_LOADDQUSI512,
28952 IX86_BUILTIN_LOADUPD512,
28953 IX86_BUILTIN_LOADUPS512,
28954 IX86_BUILTIN_MAXPD512,
28955 IX86_BUILTIN_MAXPS512,
28956 IX86_BUILTIN_MAXSD_ROUND,
28957 IX86_BUILTIN_MAXSS_ROUND,
28958 IX86_BUILTIN_MINPD512,
28959 IX86_BUILTIN_MINPS512,
28960 IX86_BUILTIN_MINSD_ROUND,
28961 IX86_BUILTIN_MINSS_ROUND,
28962 IX86_BUILTIN_MOVAPD512,
28963 IX86_BUILTIN_MOVAPS512,
28964 IX86_BUILTIN_MOVDDUP512,
28965 IX86_BUILTIN_MOVDQA32LOAD512,
28966 IX86_BUILTIN_MOVDQA32STORE512,
28967 IX86_BUILTIN_MOVDQA32_512,
28968 IX86_BUILTIN_MOVDQA64LOAD512,
28969 IX86_BUILTIN_MOVDQA64STORE512,
28970 IX86_BUILTIN_MOVDQA64_512,
28971 IX86_BUILTIN_MOVNTDQ512,
28972 IX86_BUILTIN_MOVNTDQA512,
28973 IX86_BUILTIN_MOVNTPD512,
28974 IX86_BUILTIN_MOVNTPS512,
28975 IX86_BUILTIN_MOVSHDUP512,
28976 IX86_BUILTIN_MOVSLDUP512,
28977 IX86_BUILTIN_MULPD512,
28978 IX86_BUILTIN_MULPS512,
28979 IX86_BUILTIN_MULSD_ROUND,
28980 IX86_BUILTIN_MULSS_ROUND,
28981 IX86_BUILTIN_PABSD512,
28982 IX86_BUILTIN_PABSQ512,
28983 IX86_BUILTIN_PADDD512,
28984 IX86_BUILTIN_PADDQ512,
28985 IX86_BUILTIN_PANDD512,
28986 IX86_BUILTIN_PANDND512,
28987 IX86_BUILTIN_PANDNQ512,
28988 IX86_BUILTIN_PANDQ512,
28989 IX86_BUILTIN_PBROADCASTD512,
28990 IX86_BUILTIN_PBROADCASTD512_GPR,
28991 IX86_BUILTIN_PBROADCASTMB512,
28992 IX86_BUILTIN_PBROADCASTMW512,
28993 IX86_BUILTIN_PBROADCASTQ512,
28994 IX86_BUILTIN_PBROADCASTQ512_GPR,
28995 IX86_BUILTIN_PCMPEQD512_MASK,
28996 IX86_BUILTIN_PCMPEQQ512_MASK,
28997 IX86_BUILTIN_PCMPGTD512_MASK,
28998 IX86_BUILTIN_PCMPGTQ512_MASK,
28999 IX86_BUILTIN_PCOMPRESSD512,
29000 IX86_BUILTIN_PCOMPRESSDSTORE512,
29001 IX86_BUILTIN_PCOMPRESSQ512,
29002 IX86_BUILTIN_PCOMPRESSQSTORE512,
29003 IX86_BUILTIN_PEXPANDD512,
29004 IX86_BUILTIN_PEXPANDD512Z,
29005 IX86_BUILTIN_PEXPANDDLOAD512,
29006 IX86_BUILTIN_PEXPANDDLOAD512Z,
29007 IX86_BUILTIN_PEXPANDQ512,
29008 IX86_BUILTIN_PEXPANDQ512Z,
29009 IX86_BUILTIN_PEXPANDQLOAD512,
29010 IX86_BUILTIN_PEXPANDQLOAD512Z,
29011 IX86_BUILTIN_PMAXSD512,
29012 IX86_BUILTIN_PMAXSQ512,
29013 IX86_BUILTIN_PMAXUD512,
29014 IX86_BUILTIN_PMAXUQ512,
29015 IX86_BUILTIN_PMINSD512,
29016 IX86_BUILTIN_PMINSQ512,
29017 IX86_BUILTIN_PMINUD512,
29018 IX86_BUILTIN_PMINUQ512,
29019 IX86_BUILTIN_PMOVDB512,
29020 IX86_BUILTIN_PMOVDB512_MEM,
29021 IX86_BUILTIN_PMOVDW512,
29022 IX86_BUILTIN_PMOVDW512_MEM,
29023 IX86_BUILTIN_PMOVQB512,
29024 IX86_BUILTIN_PMOVQB512_MEM,
29025 IX86_BUILTIN_PMOVQD512,
29026 IX86_BUILTIN_PMOVQD512_MEM,
29027 IX86_BUILTIN_PMOVQW512,
29028 IX86_BUILTIN_PMOVQW512_MEM,
29029 IX86_BUILTIN_PMOVSDB512,
29030 IX86_BUILTIN_PMOVSDB512_MEM,
29031 IX86_BUILTIN_PMOVSDW512,
29032 IX86_BUILTIN_PMOVSDW512_MEM,
29033 IX86_BUILTIN_PMOVSQB512,
29034 IX86_BUILTIN_PMOVSQB512_MEM,
29035 IX86_BUILTIN_PMOVSQD512,
29036 IX86_BUILTIN_PMOVSQD512_MEM,
29037 IX86_BUILTIN_PMOVSQW512,
29038 IX86_BUILTIN_PMOVSQW512_MEM,
29039 IX86_BUILTIN_PMOVSXBD512,
29040 IX86_BUILTIN_PMOVSXBQ512,
29041 IX86_BUILTIN_PMOVSXDQ512,
29042 IX86_BUILTIN_PMOVSXWD512,
29043 IX86_BUILTIN_PMOVSXWQ512,
29044 IX86_BUILTIN_PMOVUSDB512,
29045 IX86_BUILTIN_PMOVUSDB512_MEM,
29046 IX86_BUILTIN_PMOVUSDW512,
29047 IX86_BUILTIN_PMOVUSDW512_MEM,
29048 IX86_BUILTIN_PMOVUSQB512,
29049 IX86_BUILTIN_PMOVUSQB512_MEM,
29050 IX86_BUILTIN_PMOVUSQD512,
29051 IX86_BUILTIN_PMOVUSQD512_MEM,
29052 IX86_BUILTIN_PMOVUSQW512,
29053 IX86_BUILTIN_PMOVUSQW512_MEM,
29054 IX86_BUILTIN_PMOVZXBD512,
29055 IX86_BUILTIN_PMOVZXBQ512,
29056 IX86_BUILTIN_PMOVZXDQ512,
29057 IX86_BUILTIN_PMOVZXWD512,
29058 IX86_BUILTIN_PMOVZXWQ512,
29059 IX86_BUILTIN_PMULDQ512,
29060 IX86_BUILTIN_PMULLD512,
29061 IX86_BUILTIN_PMULUDQ512,
29062 IX86_BUILTIN_PORD512,
29063 IX86_BUILTIN_PORQ512,
29064 IX86_BUILTIN_PROLD512,
29065 IX86_BUILTIN_PROLQ512,
29066 IX86_BUILTIN_PROLVD512,
29067 IX86_BUILTIN_PROLVQ512,
29068 IX86_BUILTIN_PRORD512,
29069 IX86_BUILTIN_PRORQ512,
29070 IX86_BUILTIN_PRORVD512,
29071 IX86_BUILTIN_PRORVQ512,
29072 IX86_BUILTIN_PSHUFD512,
29073 IX86_BUILTIN_PSLLD512,
29074 IX86_BUILTIN_PSLLDI512,
29075 IX86_BUILTIN_PSLLQ512,
29076 IX86_BUILTIN_PSLLQI512,
29077 IX86_BUILTIN_PSLLVV16SI,
29078 IX86_BUILTIN_PSLLVV8DI,
29079 IX86_BUILTIN_PSRAD512,
29080 IX86_BUILTIN_PSRADI512,
29081 IX86_BUILTIN_PSRAQ512,
29082 IX86_BUILTIN_PSRAQI512,
29083 IX86_BUILTIN_PSRAVV16SI,
29084 IX86_BUILTIN_PSRAVV8DI,
29085 IX86_BUILTIN_PSRLD512,
29086 IX86_BUILTIN_PSRLDI512,
29087 IX86_BUILTIN_PSRLQ512,
29088 IX86_BUILTIN_PSRLQI512,
29089 IX86_BUILTIN_PSRLVV16SI,
29090 IX86_BUILTIN_PSRLVV8DI,
29091 IX86_BUILTIN_PSUBD512,
29092 IX86_BUILTIN_PSUBQ512,
29093 IX86_BUILTIN_PTESTMD512,
29094 IX86_BUILTIN_PTESTMQ512,
29095 IX86_BUILTIN_PTESTNMD512,
29096 IX86_BUILTIN_PTESTNMQ512,
29097 IX86_BUILTIN_PUNPCKHDQ512,
29098 IX86_BUILTIN_PUNPCKHQDQ512,
29099 IX86_BUILTIN_PUNPCKLDQ512,
29100 IX86_BUILTIN_PUNPCKLQDQ512,
29101 IX86_BUILTIN_PXORD512,
29102 IX86_BUILTIN_PXORQ512,
29103 IX86_BUILTIN_RCP14PD512,
29104 IX86_BUILTIN_RCP14PS512,
29105 IX86_BUILTIN_RCP14SD,
29106 IX86_BUILTIN_RCP14SS,
29107 IX86_BUILTIN_RNDSCALEPD,
29108 IX86_BUILTIN_RNDSCALEPS,
29109 IX86_BUILTIN_RNDSCALESD,
29110 IX86_BUILTIN_RNDSCALESS,
29111 IX86_BUILTIN_RSQRT14PD512,
29112 IX86_BUILTIN_RSQRT14PS512,
29113 IX86_BUILTIN_RSQRT14SD,
29114 IX86_BUILTIN_RSQRT14SS,
29115 IX86_BUILTIN_SCALEFPD512,
29116 IX86_BUILTIN_SCALEFPS512,
29117 IX86_BUILTIN_SCALEFSD,
29118 IX86_BUILTIN_SCALEFSS,
29119 IX86_BUILTIN_SHUFPD512,
29120 IX86_BUILTIN_SHUFPS512,
29121 IX86_BUILTIN_SHUF_F32x4,
29122 IX86_BUILTIN_SHUF_F64x2,
29123 IX86_BUILTIN_SHUF_I32x4,
29124 IX86_BUILTIN_SHUF_I64x2,
29125 IX86_BUILTIN_SQRTPD512,
29126 IX86_BUILTIN_SQRTPD512_MASK,
29127 IX86_BUILTIN_SQRTPS512_MASK,
29128 IX86_BUILTIN_SQRTPS_NR512,
29129 IX86_BUILTIN_SQRTSD_ROUND,
29130 IX86_BUILTIN_SQRTSS_ROUND,
29131 IX86_BUILTIN_STOREAPD512,
29132 IX86_BUILTIN_STOREAPS512,
29133 IX86_BUILTIN_STOREDQUDI512,
29134 IX86_BUILTIN_STOREDQUSI512,
29135 IX86_BUILTIN_STOREUPD512,
29136 IX86_BUILTIN_STOREUPS512,
29137 IX86_BUILTIN_SUBPD512,
29138 IX86_BUILTIN_SUBPS512,
29139 IX86_BUILTIN_SUBSD_ROUND,
29140 IX86_BUILTIN_SUBSS_ROUND,
29141 IX86_BUILTIN_UCMPD512,
29142 IX86_BUILTIN_UCMPQ512,
29143 IX86_BUILTIN_UNPCKHPD512,
29144 IX86_BUILTIN_UNPCKHPS512,
29145 IX86_BUILTIN_UNPCKLPD512,
29146 IX86_BUILTIN_UNPCKLPS512,
29147 IX86_BUILTIN_VCVTSD2SI32,
29148 IX86_BUILTIN_VCVTSD2SI64,
29149 IX86_BUILTIN_VCVTSD2USI32,
29150 IX86_BUILTIN_VCVTSD2USI64,
29151 IX86_BUILTIN_VCVTSS2SI32,
29152 IX86_BUILTIN_VCVTSS2SI64,
29153 IX86_BUILTIN_VCVTSS2USI32,
29154 IX86_BUILTIN_VCVTSS2USI64,
29155 IX86_BUILTIN_VCVTTSD2SI32,
29156 IX86_BUILTIN_VCVTTSD2SI64,
29157 IX86_BUILTIN_VCVTTSD2USI32,
29158 IX86_BUILTIN_VCVTTSD2USI64,
29159 IX86_BUILTIN_VCVTTSS2SI32,
29160 IX86_BUILTIN_VCVTTSS2SI64,
29161 IX86_BUILTIN_VCVTTSS2USI32,
29162 IX86_BUILTIN_VCVTTSS2USI64,
29163 IX86_BUILTIN_VFMADDPD512_MASK,
29164 IX86_BUILTIN_VFMADDPD512_MASK3,
29165 IX86_BUILTIN_VFMADDPD512_MASKZ,
29166 IX86_BUILTIN_VFMADDPS512_MASK,
29167 IX86_BUILTIN_VFMADDPS512_MASK3,
29168 IX86_BUILTIN_VFMADDPS512_MASKZ,
29169 IX86_BUILTIN_VFMADDSD3_ROUND,
29170 IX86_BUILTIN_VFMADDSS3_ROUND,
29171 IX86_BUILTIN_VFMADDSUBPD512_MASK,
29172 IX86_BUILTIN_VFMADDSUBPD512_MASK3,
29173 IX86_BUILTIN_VFMADDSUBPD512_MASKZ,
29174 IX86_BUILTIN_VFMADDSUBPS512_MASK,
29175 IX86_BUILTIN_VFMADDSUBPS512_MASK3,
29176 IX86_BUILTIN_VFMADDSUBPS512_MASKZ,
29177 IX86_BUILTIN_VFMSUBADDPD512_MASK3,
29178 IX86_BUILTIN_VFMSUBADDPS512_MASK3,
29179 IX86_BUILTIN_VFMSUBPD512_MASK3,
29180 IX86_BUILTIN_VFMSUBPS512_MASK3,
29181 IX86_BUILTIN_VFMSUBSD3_MASK3,
29182 IX86_BUILTIN_VFMSUBSS3_MASK3,
29183 IX86_BUILTIN_VFNMADDPD512_MASK,
29184 IX86_BUILTIN_VFNMADDPS512_MASK,
29185 IX86_BUILTIN_VFNMSUBPD512_MASK,
29186 IX86_BUILTIN_VFNMSUBPD512_MASK3,
29187 IX86_BUILTIN_VFNMSUBPS512_MASK,
29188 IX86_BUILTIN_VFNMSUBPS512_MASK3,
29189 IX86_BUILTIN_VPCLZCNTD512,
29190 IX86_BUILTIN_VPCLZCNTQ512,
29191 IX86_BUILTIN_VPCONFLICTD512,
29192 IX86_BUILTIN_VPCONFLICTQ512,
29193 IX86_BUILTIN_VPERMDF512,
29194 IX86_BUILTIN_VPERMDI512,
29195 IX86_BUILTIN_VPERMI2VARD512,
29196 IX86_BUILTIN_VPERMI2VARPD512,
29197 IX86_BUILTIN_VPERMI2VARPS512,
29198 IX86_BUILTIN_VPERMI2VARQ512,
29199 IX86_BUILTIN_VPERMILPD512,
29200 IX86_BUILTIN_VPERMILPS512,
29201 IX86_BUILTIN_VPERMILVARPD512,
29202 IX86_BUILTIN_VPERMILVARPS512,
29203 IX86_BUILTIN_VPERMT2VARD512,
29204 IX86_BUILTIN_VPERMT2VARD512_MASKZ,
29205 IX86_BUILTIN_VPERMT2VARPD512,
29206 IX86_BUILTIN_VPERMT2VARPD512_MASKZ,
29207 IX86_BUILTIN_VPERMT2VARPS512,
29208 IX86_BUILTIN_VPERMT2VARPS512_MASKZ,
29209 IX86_BUILTIN_VPERMT2VARQ512,
29210 IX86_BUILTIN_VPERMT2VARQ512_MASKZ,
29211 IX86_BUILTIN_VPERMVARDF512,
29212 IX86_BUILTIN_VPERMVARDI512,
29213 IX86_BUILTIN_VPERMVARSF512,
29214 IX86_BUILTIN_VPERMVARSI512,
29215 IX86_BUILTIN_VTERNLOGD512_MASK,
29216 IX86_BUILTIN_VTERNLOGD512_MASKZ,
29217 IX86_BUILTIN_VTERNLOGQ512_MASK,
29218 IX86_BUILTIN_VTERNLOGQ512_MASKZ,
29220 /* Mask arithmetic operations */
29221 IX86_BUILTIN_KAND16,
29222 IX86_BUILTIN_KANDN16,
29223 IX86_BUILTIN_KNOT16,
29224 IX86_BUILTIN_KOR16,
29225 IX86_BUILTIN_KORTESTC16,
29226 IX86_BUILTIN_KORTESTZ16,
29227 IX86_BUILTIN_KUNPCKBW,
29228 IX86_BUILTIN_KXNOR16,
29229 IX86_BUILTIN_KXOR16,
29230 IX86_BUILTIN_KMOV16,
29233 IX86_BUILTIN_PMOVUSQD256_MEM,
29234 IX86_BUILTIN_PMOVUSQD128_MEM,
29235 IX86_BUILTIN_PMOVSQD256_MEM,
29236 IX86_BUILTIN_PMOVSQD128_MEM,
29237 IX86_BUILTIN_PMOVQD256_MEM,
29238 IX86_BUILTIN_PMOVQD128_MEM,
29239 IX86_BUILTIN_PMOVUSQW256_MEM,
29240 IX86_BUILTIN_PMOVUSQW128_MEM,
29241 IX86_BUILTIN_PMOVSQW256_MEM,
29242 IX86_BUILTIN_PMOVSQW128_MEM,
29243 IX86_BUILTIN_PMOVQW256_MEM,
29244 IX86_BUILTIN_PMOVQW128_MEM,
29245 IX86_BUILTIN_PMOVUSQB256_MEM,
29246 IX86_BUILTIN_PMOVUSQB128_MEM,
29247 IX86_BUILTIN_PMOVSQB256_MEM,
29248 IX86_BUILTIN_PMOVSQB128_MEM,
29249 IX86_BUILTIN_PMOVQB256_MEM,
29250 IX86_BUILTIN_PMOVQB128_MEM,
29251 IX86_BUILTIN_PMOVUSDW256_MEM,
29252 IX86_BUILTIN_PMOVUSDW128_MEM,
29253 IX86_BUILTIN_PMOVSDW256_MEM,
29254 IX86_BUILTIN_PMOVSDW128_MEM,
29255 IX86_BUILTIN_PMOVDW256_MEM,
29256 IX86_BUILTIN_PMOVDW128_MEM,
29257 IX86_BUILTIN_PMOVUSDB256_MEM,
29258 IX86_BUILTIN_PMOVUSDB128_MEM,
29259 IX86_BUILTIN_PMOVSDB256_MEM,
29260 IX86_BUILTIN_PMOVSDB128_MEM,
29261 IX86_BUILTIN_PMOVDB256_MEM,
29262 IX86_BUILTIN_PMOVDB128_MEM,
29263 IX86_BUILTIN_MOVDQA64LOAD256_MASK,
29264 IX86_BUILTIN_MOVDQA64LOAD128_MASK,
29265 IX86_BUILTIN_MOVDQA32LOAD256_MASK,
29266 IX86_BUILTIN_MOVDQA32LOAD128_MASK,
29267 IX86_BUILTIN_MOVDQA64STORE256_MASK,
29268 IX86_BUILTIN_MOVDQA64STORE128_MASK,
29269 IX86_BUILTIN_MOVDQA32STORE256_MASK,
29270 IX86_BUILTIN_MOVDQA32STORE128_MASK,
29271 IX86_BUILTIN_LOADAPD256_MASK,
29272 IX86_BUILTIN_LOADAPD128_MASK,
29273 IX86_BUILTIN_LOADAPS256_MASK,
29274 IX86_BUILTIN_LOADAPS128_MASK,
29275 IX86_BUILTIN_STOREAPD256_MASK,
29276 IX86_BUILTIN_STOREAPD128_MASK,
29277 IX86_BUILTIN_STOREAPS256_MASK,
29278 IX86_BUILTIN_STOREAPS128_MASK,
29279 IX86_BUILTIN_LOADUPD256_MASK,
29280 IX86_BUILTIN_LOADUPD128_MASK,
29281 IX86_BUILTIN_LOADUPS256_MASK,
29282 IX86_BUILTIN_LOADUPS128_MASK,
29283 IX86_BUILTIN_STOREUPD256_MASK,
29284 IX86_BUILTIN_STOREUPD128_MASK,
29285 IX86_BUILTIN_STOREUPS256_MASK,
29286 IX86_BUILTIN_STOREUPS128_MASK,
29287 IX86_BUILTIN_LOADDQUDI256_MASK,
29288 IX86_BUILTIN_LOADDQUDI128_MASK,
29289 IX86_BUILTIN_LOADDQUSI256_MASK,
29290 IX86_BUILTIN_LOADDQUSI128_MASK,
29291 IX86_BUILTIN_LOADDQUHI256_MASK,
29292 IX86_BUILTIN_LOADDQUHI128_MASK,
29293 IX86_BUILTIN_LOADDQUQI256_MASK,
29294 IX86_BUILTIN_LOADDQUQI128_MASK,
29295 IX86_BUILTIN_STOREDQUDI256_MASK,
29296 IX86_BUILTIN_STOREDQUDI128_MASK,
29297 IX86_BUILTIN_STOREDQUSI256_MASK,
29298 IX86_BUILTIN_STOREDQUSI128_MASK,
29299 IX86_BUILTIN_STOREDQUHI256_MASK,
29300 IX86_BUILTIN_STOREDQUHI128_MASK,
29301 IX86_BUILTIN_STOREDQUQI256_MASK,
29302 IX86_BUILTIN_STOREDQUQI128_MASK,
29303 IX86_BUILTIN_COMPRESSPDSTORE256,
29304 IX86_BUILTIN_COMPRESSPDSTORE128,
29305 IX86_BUILTIN_COMPRESSPSSTORE256,
29306 IX86_BUILTIN_COMPRESSPSSTORE128,
29307 IX86_BUILTIN_PCOMPRESSQSTORE256,
29308 IX86_BUILTIN_PCOMPRESSQSTORE128,
29309 IX86_BUILTIN_PCOMPRESSDSTORE256,
29310 IX86_BUILTIN_PCOMPRESSDSTORE128,
29311 IX86_BUILTIN_EXPANDPDLOAD256,
29312 IX86_BUILTIN_EXPANDPDLOAD128,
29313 IX86_BUILTIN_EXPANDPSLOAD256,
29314 IX86_BUILTIN_EXPANDPSLOAD128,
29315 IX86_BUILTIN_PEXPANDQLOAD256,
29316 IX86_BUILTIN_PEXPANDQLOAD128,
29317 IX86_BUILTIN_PEXPANDDLOAD256,
29318 IX86_BUILTIN_PEXPANDDLOAD128,
29319 IX86_BUILTIN_EXPANDPDLOAD256Z,
29320 IX86_BUILTIN_EXPANDPDLOAD128Z,
29321 IX86_BUILTIN_EXPANDPSLOAD256Z,
29322 IX86_BUILTIN_EXPANDPSLOAD128Z,
29323 IX86_BUILTIN_PEXPANDQLOAD256Z,
29324 IX86_BUILTIN_PEXPANDQLOAD128Z,
29325 IX86_BUILTIN_PEXPANDDLOAD256Z,
29326 IX86_BUILTIN_PEXPANDDLOAD128Z,
29327 IX86_BUILTIN_PALIGNR256_MASK,
29328 IX86_BUILTIN_PALIGNR128_MASK,
29329 IX86_BUILTIN_MOVDQA64_256_MASK,
29330 IX86_BUILTIN_MOVDQA64_128_MASK,
29331 IX86_BUILTIN_MOVDQA32_256_MASK,
29332 IX86_BUILTIN_MOVDQA32_128_MASK,
29333 IX86_BUILTIN_MOVAPD256_MASK,
29334 IX86_BUILTIN_MOVAPD128_MASK,
29335 IX86_BUILTIN_MOVAPS256_MASK,
29336 IX86_BUILTIN_MOVAPS128_MASK,
29337 IX86_BUILTIN_MOVDQUHI256_MASK,
29338 IX86_BUILTIN_MOVDQUHI128_MASK,
29339 IX86_BUILTIN_MOVDQUQI256_MASK,
29340 IX86_BUILTIN_MOVDQUQI128_MASK,
29341 IX86_BUILTIN_MINPS128_MASK,
29342 IX86_BUILTIN_MAXPS128_MASK,
29343 IX86_BUILTIN_MINPD128_MASK,
29344 IX86_BUILTIN_MAXPD128_MASK,
29345 IX86_BUILTIN_MAXPD256_MASK,
29346 IX86_BUILTIN_MAXPS256_MASK,
29347 IX86_BUILTIN_MINPD256_MASK,
29348 IX86_BUILTIN_MINPS256_MASK,
29349 IX86_BUILTIN_MULPS128_MASK,
29350 IX86_BUILTIN_DIVPS128_MASK,
29351 IX86_BUILTIN_MULPD128_MASK,
29352 IX86_BUILTIN_DIVPD128_MASK,
29353 IX86_BUILTIN_DIVPD256_MASK,
29354 IX86_BUILTIN_DIVPS256_MASK,
29355 IX86_BUILTIN_MULPD256_MASK,
29356 IX86_BUILTIN_MULPS256_MASK,
29357 IX86_BUILTIN_ADDPD128_MASK,
29358 IX86_BUILTIN_ADDPD256_MASK,
29359 IX86_BUILTIN_ADDPS128_MASK,
29360 IX86_BUILTIN_ADDPS256_MASK,
29361 IX86_BUILTIN_SUBPD128_MASK,
29362 IX86_BUILTIN_SUBPD256_MASK,
29363 IX86_BUILTIN_SUBPS128_MASK,
29364 IX86_BUILTIN_SUBPS256_MASK,
29365 IX86_BUILTIN_XORPD256_MASK,
29366 IX86_BUILTIN_XORPD128_MASK,
29367 IX86_BUILTIN_XORPS256_MASK,
29368 IX86_BUILTIN_XORPS128_MASK,
29369 IX86_BUILTIN_ORPD256_MASK,
29370 IX86_BUILTIN_ORPD128_MASK,
29371 IX86_BUILTIN_ORPS256_MASK,
29372 IX86_BUILTIN_ORPS128_MASK,
29373 IX86_BUILTIN_BROADCASTF32x2_256,
29374 IX86_BUILTIN_BROADCASTI32x2_256,
29375 IX86_BUILTIN_BROADCASTI32x2_128,
29376 IX86_BUILTIN_BROADCASTF64X2_256,
29377 IX86_BUILTIN_BROADCASTI64X2_256,
29378 IX86_BUILTIN_BROADCASTF32X4_256,
29379 IX86_BUILTIN_BROADCASTI32X4_256,
29380 IX86_BUILTIN_EXTRACTF32X4_256,
29381 IX86_BUILTIN_EXTRACTI32X4_256,
29382 IX86_BUILTIN_DBPSADBW256,
29383 IX86_BUILTIN_DBPSADBW128,
29384 IX86_BUILTIN_CVTTPD2QQ256,
29385 IX86_BUILTIN_CVTTPD2QQ128,
29386 IX86_BUILTIN_CVTTPD2UQQ256,
29387 IX86_BUILTIN_CVTTPD2UQQ128,
29388 IX86_BUILTIN_CVTPD2QQ256,
29389 IX86_BUILTIN_CVTPD2QQ128,
29390 IX86_BUILTIN_CVTPD2UQQ256,
29391 IX86_BUILTIN_CVTPD2UQQ128,
29392 IX86_BUILTIN_CVTPD2UDQ256_MASK,
29393 IX86_BUILTIN_CVTPD2UDQ128_MASK,
29394 IX86_BUILTIN_CVTTPS2QQ256,
29395 IX86_BUILTIN_CVTTPS2QQ128,
29396 IX86_BUILTIN_CVTTPS2UQQ256,
29397 IX86_BUILTIN_CVTTPS2UQQ128,
29398 IX86_BUILTIN_CVTTPS2DQ256_MASK,
29399 IX86_BUILTIN_CVTTPS2DQ128_MASK,
29400 IX86_BUILTIN_CVTTPS2UDQ256,
29401 IX86_BUILTIN_CVTTPS2UDQ128,
29402 IX86_BUILTIN_CVTTPD2DQ256_MASK,
29403 IX86_BUILTIN_CVTTPD2DQ128_MASK,
29404 IX86_BUILTIN_CVTTPD2UDQ256_MASK,
29405 IX86_BUILTIN_CVTTPD2UDQ128_MASK,
29406 IX86_BUILTIN_CVTPD2DQ256_MASK,
29407 IX86_BUILTIN_CVTPD2DQ128_MASK,
29408 IX86_BUILTIN_CVTDQ2PD256_MASK,
29409 IX86_BUILTIN_CVTDQ2PD128_MASK,
29410 IX86_BUILTIN_CVTUDQ2PD256_MASK,
29411 IX86_BUILTIN_CVTUDQ2PD128_MASK,
29412 IX86_BUILTIN_CVTDQ2PS256_MASK,
29413 IX86_BUILTIN_CVTDQ2PS128_MASK,
29414 IX86_BUILTIN_CVTUDQ2PS256_MASK,
29415 IX86_BUILTIN_CVTUDQ2PS128_MASK,
29416 IX86_BUILTIN_CVTPS2PD256_MASK,
29417 IX86_BUILTIN_CVTPS2PD128_MASK,
29418 IX86_BUILTIN_PBROADCASTB256_MASK,
29419 IX86_BUILTIN_PBROADCASTB256_GPR_MASK,
29420 IX86_BUILTIN_PBROADCASTB128_MASK,
29421 IX86_BUILTIN_PBROADCASTB128_GPR_MASK,
29422 IX86_BUILTIN_PBROADCASTW256_MASK,
29423 IX86_BUILTIN_PBROADCASTW256_GPR_MASK,
29424 IX86_BUILTIN_PBROADCASTW128_MASK,
29425 IX86_BUILTIN_PBROADCASTW128_GPR_MASK,
29426 IX86_BUILTIN_PBROADCASTD256_MASK,
29427 IX86_BUILTIN_PBROADCASTD256_GPR_MASK,
29428 IX86_BUILTIN_PBROADCASTD128_MASK,
29429 IX86_BUILTIN_PBROADCASTD128_GPR_MASK,
29430 IX86_BUILTIN_PBROADCASTQ256_MASK,
29431 IX86_BUILTIN_PBROADCASTQ256_GPR_MASK,
29432 IX86_BUILTIN_PBROADCASTQ128_MASK,
29433 IX86_BUILTIN_PBROADCASTQ128_GPR_MASK,
29434 IX86_BUILTIN_BROADCASTSS256,
29435 IX86_BUILTIN_BROADCASTSS128,
29436 IX86_BUILTIN_BROADCASTSD256,
29437 IX86_BUILTIN_EXTRACTF64X2_256,
29438 IX86_BUILTIN_EXTRACTI64X2_256,
29439 IX86_BUILTIN_INSERTF32X4_256,
29440 IX86_BUILTIN_INSERTI32X4_256,
29441 IX86_BUILTIN_PMOVSXBW256_MASK,
29442 IX86_BUILTIN_PMOVSXBW128_MASK,
29443 IX86_BUILTIN_PMOVSXBD256_MASK,
29444 IX86_BUILTIN_PMOVSXBD128_MASK,
29445 IX86_BUILTIN_PMOVSXBQ256_MASK,
29446 IX86_BUILTIN_PMOVSXBQ128_MASK,
29447 IX86_BUILTIN_PMOVSXWD256_MASK,
29448 IX86_BUILTIN_PMOVSXWD128_MASK,
29449 IX86_BUILTIN_PMOVSXWQ256_MASK,
29450 IX86_BUILTIN_PMOVSXWQ128_MASK,
29451 IX86_BUILTIN_PMOVSXDQ256_MASK,
29452 IX86_BUILTIN_PMOVSXDQ128_MASK,
29453 IX86_BUILTIN_PMOVZXBW256_MASK,
29454 IX86_BUILTIN_PMOVZXBW128_MASK,
29455 IX86_BUILTIN_PMOVZXBD256_MASK,
29456 IX86_BUILTIN_PMOVZXBD128_MASK,
29457 IX86_BUILTIN_PMOVZXBQ256_MASK,
29458 IX86_BUILTIN_PMOVZXBQ128_MASK,
29459 IX86_BUILTIN_PMOVZXWD256_MASK,
29460 IX86_BUILTIN_PMOVZXWD128_MASK,
29461 IX86_BUILTIN_PMOVZXWQ256_MASK,
29462 IX86_BUILTIN_PMOVZXWQ128_MASK,
29463 IX86_BUILTIN_PMOVZXDQ256_MASK,
29464 IX86_BUILTIN_PMOVZXDQ128_MASK,
29465 IX86_BUILTIN_REDUCEPD256_MASK,
29466 IX86_BUILTIN_REDUCEPD128_MASK,
29467 IX86_BUILTIN_REDUCEPS256_MASK,
29468 IX86_BUILTIN_REDUCEPS128_MASK,
29469 IX86_BUILTIN_REDUCESD_MASK,
29470 IX86_BUILTIN_REDUCESS_MASK,
29471 IX86_BUILTIN_VPERMVARHI256_MASK,
29472 IX86_BUILTIN_VPERMVARHI128_MASK,
29473 IX86_BUILTIN_VPERMT2VARHI256,
29474 IX86_BUILTIN_VPERMT2VARHI256_MASKZ,
29475 IX86_BUILTIN_VPERMT2VARHI128,
29476 IX86_BUILTIN_VPERMT2VARHI128_MASKZ,
29477 IX86_BUILTIN_VPERMI2VARHI256,
29478 IX86_BUILTIN_VPERMI2VARHI128,
29479 IX86_BUILTIN_RCP14PD256,
29480 IX86_BUILTIN_RCP14PD128,
29481 IX86_BUILTIN_RCP14PS256,
29482 IX86_BUILTIN_RCP14PS128,
29483 IX86_BUILTIN_RSQRT14PD256_MASK,
29484 IX86_BUILTIN_RSQRT14PD128_MASK,
29485 IX86_BUILTIN_RSQRT14PS256_MASK,
29486 IX86_BUILTIN_RSQRT14PS128_MASK,
29487 IX86_BUILTIN_SQRTPD256_MASK,
29488 IX86_BUILTIN_SQRTPD128_MASK,
29489 IX86_BUILTIN_SQRTPS256_MASK,
29490 IX86_BUILTIN_SQRTPS128_MASK,
29491 IX86_BUILTIN_PADDB128_MASK,
29492 IX86_BUILTIN_PADDW128_MASK,
29493 IX86_BUILTIN_PADDD128_MASK,
29494 IX86_BUILTIN_PADDQ128_MASK,
29495 IX86_BUILTIN_PSUBB128_MASK,
29496 IX86_BUILTIN_PSUBW128_MASK,
29497 IX86_BUILTIN_PSUBD128_MASK,
29498 IX86_BUILTIN_PSUBQ128_MASK,
29499 IX86_BUILTIN_PADDSB128_MASK,
29500 IX86_BUILTIN_PADDSW128_MASK,
29501 IX86_BUILTIN_PSUBSB128_MASK,
29502 IX86_BUILTIN_PSUBSW128_MASK,
29503 IX86_BUILTIN_PADDUSB128_MASK,
29504 IX86_BUILTIN_PADDUSW128_MASK,
29505 IX86_BUILTIN_PSUBUSB128_MASK,
29506 IX86_BUILTIN_PSUBUSW128_MASK,
29507 IX86_BUILTIN_PADDB256_MASK,
29508 IX86_BUILTIN_PADDW256_MASK,
29509 IX86_BUILTIN_PADDD256_MASK,
29510 IX86_BUILTIN_PADDQ256_MASK,
29511 IX86_BUILTIN_PADDSB256_MASK,
29512 IX86_BUILTIN_PADDSW256_MASK,
29513 IX86_BUILTIN_PADDUSB256_MASK,
29514 IX86_BUILTIN_PADDUSW256_MASK,
29515 IX86_BUILTIN_PSUBB256_MASK,
29516 IX86_BUILTIN_PSUBW256_MASK,
29517 IX86_BUILTIN_PSUBD256_MASK,
29518 IX86_BUILTIN_PSUBQ256_MASK,
29519 IX86_BUILTIN_PSUBSB256_MASK,
29520 IX86_BUILTIN_PSUBSW256_MASK,
29521 IX86_BUILTIN_PSUBUSB256_MASK,
29522 IX86_BUILTIN_PSUBUSW256_MASK,
29523 IX86_BUILTIN_SHUF_F64x2_256,
29524 IX86_BUILTIN_SHUF_I64x2_256,
29525 IX86_BUILTIN_SHUF_I32x4_256,
29526 IX86_BUILTIN_SHUF_F32x4_256,
29527 IX86_BUILTIN_PMOVWB128,
29528 IX86_BUILTIN_PMOVWB256,
29529 IX86_BUILTIN_PMOVSWB128,
29530 IX86_BUILTIN_PMOVSWB256,
29531 IX86_BUILTIN_PMOVUSWB128,
29532 IX86_BUILTIN_PMOVUSWB256,
29533 IX86_BUILTIN_PMOVDB128,
29534 IX86_BUILTIN_PMOVDB256,
29535 IX86_BUILTIN_PMOVSDB128,
29536 IX86_BUILTIN_PMOVSDB256,
29537 IX86_BUILTIN_PMOVUSDB128,
29538 IX86_BUILTIN_PMOVUSDB256,
29539 IX86_BUILTIN_PMOVDW128,
29540 IX86_BUILTIN_PMOVDW256,
29541 IX86_BUILTIN_PMOVSDW128,
29542 IX86_BUILTIN_PMOVSDW256,
29543 IX86_BUILTIN_PMOVUSDW128,
29544 IX86_BUILTIN_PMOVUSDW256,
29545 IX86_BUILTIN_PMOVQB128,
29546 IX86_BUILTIN_PMOVQB256,
29547 IX86_BUILTIN_PMOVSQB128,
29548 IX86_BUILTIN_PMOVSQB256,
29549 IX86_BUILTIN_PMOVUSQB128,
29550 IX86_BUILTIN_PMOVUSQB256,
29551 IX86_BUILTIN_PMOVQW128,
29552 IX86_BUILTIN_PMOVQW256,
29553 IX86_BUILTIN_PMOVSQW128,
29554 IX86_BUILTIN_PMOVSQW256,
29555 IX86_BUILTIN_PMOVUSQW128,
29556 IX86_BUILTIN_PMOVUSQW256,
29557 IX86_BUILTIN_PMOVQD128,
29558 IX86_BUILTIN_PMOVQD256,
29559 IX86_BUILTIN_PMOVSQD128,
29560 IX86_BUILTIN_PMOVSQD256,
29561 IX86_BUILTIN_PMOVUSQD128,
29562 IX86_BUILTIN_PMOVUSQD256,
29563 IX86_BUILTIN_RANGEPD256,
29564 IX86_BUILTIN_RANGEPD128,
29565 IX86_BUILTIN_RANGEPS256,
29566 IX86_BUILTIN_RANGEPS128,
29567 IX86_BUILTIN_GETEXPPS256,
29568 IX86_BUILTIN_GETEXPPD256,
29569 IX86_BUILTIN_GETEXPPS128,
29570 IX86_BUILTIN_GETEXPPD128,
29571 IX86_BUILTIN_FIXUPIMMPD256_MASK,
29572 IX86_BUILTIN_FIXUPIMMPD256_MASKZ,
29573 IX86_BUILTIN_FIXUPIMMPS256_MASK,
29574 IX86_BUILTIN_FIXUPIMMPS256_MASKZ,
29575 IX86_BUILTIN_FIXUPIMMPD128_MASK,
29576 IX86_BUILTIN_FIXUPIMMPD128_MASKZ,
29577 IX86_BUILTIN_FIXUPIMMPS128_MASK,
29578 IX86_BUILTIN_FIXUPIMMPS128_MASKZ,
29579 IX86_BUILTIN_PABSQ256,
29580 IX86_BUILTIN_PABSQ128,
29581 IX86_BUILTIN_PABSD256_MASK,
29582 IX86_BUILTIN_PABSD128_MASK,
29583 IX86_BUILTIN_PMULHRSW256_MASK,
29584 IX86_BUILTIN_PMULHRSW128_MASK,
29585 IX86_BUILTIN_PMULHUW128_MASK,
29586 IX86_BUILTIN_PMULHUW256_MASK,
29587 IX86_BUILTIN_PMULHW256_MASK,
29588 IX86_BUILTIN_PMULHW128_MASK,
29589 IX86_BUILTIN_PMULLW256_MASK,
29590 IX86_BUILTIN_PMULLW128_MASK,
29591 IX86_BUILTIN_PMULLQ256,
29592 IX86_BUILTIN_PMULLQ128,
29593 IX86_BUILTIN_ANDPD256_MASK,
29594 IX86_BUILTIN_ANDPD128_MASK,
29595 IX86_BUILTIN_ANDPS256_MASK,
29596 IX86_BUILTIN_ANDPS128_MASK,
29597 IX86_BUILTIN_ANDNPD256_MASK,
29598 IX86_BUILTIN_ANDNPD128_MASK,
29599 IX86_BUILTIN_ANDNPS256_MASK,
29600 IX86_BUILTIN_ANDNPS128_MASK,
29601 IX86_BUILTIN_PSLLWI128_MASK,
29602 IX86_BUILTIN_PSLLDI128_MASK,
29603 IX86_BUILTIN_PSLLQI128_MASK,
29604 IX86_BUILTIN_PSLLW128_MASK,
29605 IX86_BUILTIN_PSLLD128_MASK,
29606 IX86_BUILTIN_PSLLQ128_MASK,
29607 IX86_BUILTIN_PSLLWI256_MASK ,
29608 IX86_BUILTIN_PSLLW256_MASK,
29609 IX86_BUILTIN_PSLLDI256_MASK,
29610 IX86_BUILTIN_PSLLD256_MASK,
29611 IX86_BUILTIN_PSLLQI256_MASK,
29612 IX86_BUILTIN_PSLLQ256_MASK,
29613 IX86_BUILTIN_PSRADI128_MASK,
29614 IX86_BUILTIN_PSRAD128_MASK,
29615 IX86_BUILTIN_PSRADI256_MASK,
29616 IX86_BUILTIN_PSRAD256_MASK,
29617 IX86_BUILTIN_PSRAQI128_MASK,
29618 IX86_BUILTIN_PSRAQ128_MASK,
29619 IX86_BUILTIN_PSRAQI256_MASK,
29620 IX86_BUILTIN_PSRAQ256_MASK,
29621 IX86_BUILTIN_PANDD256,
29622 IX86_BUILTIN_PANDD128,
29623 IX86_BUILTIN_PSRLDI128_MASK,
29624 IX86_BUILTIN_PSRLD128_MASK,
29625 IX86_BUILTIN_PSRLDI256_MASK,
29626 IX86_BUILTIN_PSRLD256_MASK,
29627 IX86_BUILTIN_PSRLQI128_MASK,
29628 IX86_BUILTIN_PSRLQ128_MASK,
29629 IX86_BUILTIN_PSRLQI256_MASK,
29630 IX86_BUILTIN_PSRLQ256_MASK,
29631 IX86_BUILTIN_PANDQ256,
29632 IX86_BUILTIN_PANDQ128,
29633 IX86_BUILTIN_PANDND256,
29634 IX86_BUILTIN_PANDND128,
29635 IX86_BUILTIN_PANDNQ256,
29636 IX86_BUILTIN_PANDNQ128,
29637 IX86_BUILTIN_PORD256,
29638 IX86_BUILTIN_PORD128,
29639 IX86_BUILTIN_PORQ256,
29640 IX86_BUILTIN_PORQ128,
29641 IX86_BUILTIN_PXORD256,
29642 IX86_BUILTIN_PXORD128,
29643 IX86_BUILTIN_PXORQ256,
29644 IX86_BUILTIN_PXORQ128,
29645 IX86_BUILTIN_PACKSSWB256_MASK,
29646 IX86_BUILTIN_PACKSSWB128_MASK,
29647 IX86_BUILTIN_PACKUSWB256_MASK,
29648 IX86_BUILTIN_PACKUSWB128_MASK,
29649 IX86_BUILTIN_RNDSCALEPS256,
29650 IX86_BUILTIN_RNDSCALEPD256,
29651 IX86_BUILTIN_RNDSCALEPS128,
29652 IX86_BUILTIN_RNDSCALEPD128,
29653 IX86_BUILTIN_VTERNLOGQ256_MASK,
29654 IX86_BUILTIN_VTERNLOGQ256_MASKZ,
29655 IX86_BUILTIN_VTERNLOGD256_MASK,
29656 IX86_BUILTIN_VTERNLOGD256_MASKZ,
29657 IX86_BUILTIN_VTERNLOGQ128_MASK,
29658 IX86_BUILTIN_VTERNLOGQ128_MASKZ,
29659 IX86_BUILTIN_VTERNLOGD128_MASK,
29660 IX86_BUILTIN_VTERNLOGD128_MASKZ,
29661 IX86_BUILTIN_SCALEFPD256,
29662 IX86_BUILTIN_SCALEFPS256,
29663 IX86_BUILTIN_SCALEFPD128,
29664 IX86_BUILTIN_SCALEFPS128,
29665 IX86_BUILTIN_VFMADDPD256_MASK,
29666 IX86_BUILTIN_VFMADDPD256_MASK3,
29667 IX86_BUILTIN_VFMADDPD256_MASKZ,
29668 IX86_BUILTIN_VFMADDPD128_MASK,
29669 IX86_BUILTIN_VFMADDPD128_MASK3,
29670 IX86_BUILTIN_VFMADDPD128_MASKZ,
29671 IX86_BUILTIN_VFMADDPS256_MASK,
29672 IX86_BUILTIN_VFMADDPS256_MASK3,
29673 IX86_BUILTIN_VFMADDPS256_MASKZ,
29674 IX86_BUILTIN_VFMADDPS128_MASK,
29675 IX86_BUILTIN_VFMADDPS128_MASK3,
29676 IX86_BUILTIN_VFMADDPS128_MASKZ,
29677 IX86_BUILTIN_VFMSUBPD256_MASK3,
29678 IX86_BUILTIN_VFMSUBPD128_MASK3,
29679 IX86_BUILTIN_VFMSUBPS256_MASK3,
29680 IX86_BUILTIN_VFMSUBPS128_MASK3,
29681 IX86_BUILTIN_VFNMADDPD256_MASK,
29682 IX86_BUILTIN_VFNMADDPD128_MASK,
29683 IX86_BUILTIN_VFNMADDPS256_MASK,
29684 IX86_BUILTIN_VFNMADDPS128_MASK,
29685 IX86_BUILTIN_VFNMSUBPD256_MASK,
29686 IX86_BUILTIN_VFNMSUBPD256_MASK3,
29687 IX86_BUILTIN_VFNMSUBPD128_MASK,
29688 IX86_BUILTIN_VFNMSUBPD128_MASK3,
29689 IX86_BUILTIN_VFNMSUBPS256_MASK,
29690 IX86_BUILTIN_VFNMSUBPS256_MASK3,
29691 IX86_BUILTIN_VFNMSUBPS128_MASK,
29692 IX86_BUILTIN_VFNMSUBPS128_MASK3,
29693 IX86_BUILTIN_VFMADDSUBPD256_MASK,
29694 IX86_BUILTIN_VFMADDSUBPD256_MASK3,
29695 IX86_BUILTIN_VFMADDSUBPD256_MASKZ,
29696 IX86_BUILTIN_VFMADDSUBPD128_MASK,
29697 IX86_BUILTIN_VFMADDSUBPD128_MASK3,
29698 IX86_BUILTIN_VFMADDSUBPD128_MASKZ,
29699 IX86_BUILTIN_VFMADDSUBPS256_MASK,
29700 IX86_BUILTIN_VFMADDSUBPS256_MASK3,
29701 IX86_BUILTIN_VFMADDSUBPS256_MASKZ,
29702 IX86_BUILTIN_VFMADDSUBPS128_MASK,
29703 IX86_BUILTIN_VFMADDSUBPS128_MASK3,
29704 IX86_BUILTIN_VFMADDSUBPS128_MASKZ,
29705 IX86_BUILTIN_VFMSUBADDPD256_MASK3,
29706 IX86_BUILTIN_VFMSUBADDPD128_MASK3,
29707 IX86_BUILTIN_VFMSUBADDPS256_MASK3,
29708 IX86_BUILTIN_VFMSUBADDPS128_MASK3,
29709 IX86_BUILTIN_INSERTF64X2_256,
29710 IX86_BUILTIN_INSERTI64X2_256,
29711 IX86_BUILTIN_PSRAVV16HI,
29712 IX86_BUILTIN_PSRAVV8HI,
29713 IX86_BUILTIN_PMADDUBSW256_MASK,
29714 IX86_BUILTIN_PMADDUBSW128_MASK,
29715 IX86_BUILTIN_PMADDWD256_MASK,
29716 IX86_BUILTIN_PMADDWD128_MASK,
29717 IX86_BUILTIN_PSRLVV16HI,
29718 IX86_BUILTIN_PSRLVV8HI,
29719 IX86_BUILTIN_CVTPS2DQ256_MASK,
29720 IX86_BUILTIN_CVTPS2DQ128_MASK,
29721 IX86_BUILTIN_CVTPS2UDQ256,
29722 IX86_BUILTIN_CVTPS2UDQ128,
29723 IX86_BUILTIN_CVTPS2QQ256,
29724 IX86_BUILTIN_CVTPS2QQ128,
29725 IX86_BUILTIN_CVTPS2UQQ256,
29726 IX86_BUILTIN_CVTPS2UQQ128,
29727 IX86_BUILTIN_GETMANTPS256,
29728 IX86_BUILTIN_GETMANTPS128,
29729 IX86_BUILTIN_GETMANTPD256,
29730 IX86_BUILTIN_GETMANTPD128,
29731 IX86_BUILTIN_MOVDDUP256_MASK,
29732 IX86_BUILTIN_MOVDDUP128_MASK,
29733 IX86_BUILTIN_MOVSHDUP256_MASK,
29734 IX86_BUILTIN_MOVSHDUP128_MASK,
29735 IX86_BUILTIN_MOVSLDUP256_MASK,
29736 IX86_BUILTIN_MOVSLDUP128_MASK,
29737 IX86_BUILTIN_CVTQQ2PS256,
29738 IX86_BUILTIN_CVTQQ2PS128,
29739 IX86_BUILTIN_CVTUQQ2PS256,
29740 IX86_BUILTIN_CVTUQQ2PS128,
29741 IX86_BUILTIN_CVTQQ2PD256,
29742 IX86_BUILTIN_CVTQQ2PD128,
29743 IX86_BUILTIN_CVTUQQ2PD256,
29744 IX86_BUILTIN_CVTUQQ2PD128,
29745 IX86_BUILTIN_VPERMT2VARQ256,
29746 IX86_BUILTIN_VPERMT2VARQ256_MASKZ,
29747 IX86_BUILTIN_VPERMT2VARD256,
29748 IX86_BUILTIN_VPERMT2VARD256_MASKZ,
29749 IX86_BUILTIN_VPERMI2VARQ256,
29750 IX86_BUILTIN_VPERMI2VARD256,
29751 IX86_BUILTIN_VPERMT2VARPD256,
29752 IX86_BUILTIN_VPERMT2VARPD256_MASKZ,
29753 IX86_BUILTIN_VPERMT2VARPS256,
29754 IX86_BUILTIN_VPERMT2VARPS256_MASKZ,
29755 IX86_BUILTIN_VPERMI2VARPD256,
29756 IX86_BUILTIN_VPERMI2VARPS256,
29757 IX86_BUILTIN_VPERMT2VARQ128,
29758 IX86_BUILTIN_VPERMT2VARQ128_MASKZ,
29759 IX86_BUILTIN_VPERMT2VARD128,
29760 IX86_BUILTIN_VPERMT2VARD128_MASKZ,
29761 IX86_BUILTIN_VPERMI2VARQ128,
29762 IX86_BUILTIN_VPERMI2VARD128,
29763 IX86_BUILTIN_VPERMT2VARPD128,
29764 IX86_BUILTIN_VPERMT2VARPD128_MASKZ,
29765 IX86_BUILTIN_VPERMT2VARPS128,
29766 IX86_BUILTIN_VPERMT2VARPS128_MASKZ,
29767 IX86_BUILTIN_VPERMI2VARPD128,
29768 IX86_BUILTIN_VPERMI2VARPS128,
29769 IX86_BUILTIN_PSHUFB256_MASK,
29770 IX86_BUILTIN_PSHUFB128_MASK,
29771 IX86_BUILTIN_PSHUFHW256_MASK,
29772 IX86_BUILTIN_PSHUFHW128_MASK,
29773 IX86_BUILTIN_PSHUFLW256_MASK,
29774 IX86_BUILTIN_PSHUFLW128_MASK,
29775 IX86_BUILTIN_PSHUFD256_MASK,
29776 IX86_BUILTIN_PSHUFD128_MASK,
29777 IX86_BUILTIN_SHUFPD256_MASK,
29778 IX86_BUILTIN_SHUFPD128_MASK,
29779 IX86_BUILTIN_SHUFPS256_MASK,
29780 IX86_BUILTIN_SHUFPS128_MASK,
29781 IX86_BUILTIN_PROLVQ256,
29782 IX86_BUILTIN_PROLVQ128,
29783 IX86_BUILTIN_PROLQ256,
29784 IX86_BUILTIN_PROLQ128,
29785 IX86_BUILTIN_PRORVQ256,
29786 IX86_BUILTIN_PRORVQ128,
29787 IX86_BUILTIN_PRORQ256,
29788 IX86_BUILTIN_PRORQ128,
29789 IX86_BUILTIN_PSRAVQ128,
29790 IX86_BUILTIN_PSRAVQ256,
29791 IX86_BUILTIN_PSLLVV4DI_MASK,
29792 IX86_BUILTIN_PSLLVV2DI_MASK,
29793 IX86_BUILTIN_PSLLVV8SI_MASK,
29794 IX86_BUILTIN_PSLLVV4SI_MASK,
29795 IX86_BUILTIN_PSRAVV8SI_MASK,
29796 IX86_BUILTIN_PSRAVV4SI_MASK,
29797 IX86_BUILTIN_PSRLVV4DI_MASK,
29798 IX86_BUILTIN_PSRLVV2DI_MASK,
29799 IX86_BUILTIN_PSRLVV8SI_MASK,
29800 IX86_BUILTIN_PSRLVV4SI_MASK,
29801 IX86_BUILTIN_PSRAWI256_MASK,
29802 IX86_BUILTIN_PSRAW256_MASK,
29803 IX86_BUILTIN_PSRAWI128_MASK,
29804 IX86_BUILTIN_PSRAW128_MASK,
29805 IX86_BUILTIN_PSRLWI256_MASK,
29806 IX86_BUILTIN_PSRLW256_MASK,
29807 IX86_BUILTIN_PSRLWI128_MASK,
29808 IX86_BUILTIN_PSRLW128_MASK,
29809 IX86_BUILTIN_PRORVD256,
29810 IX86_BUILTIN_PROLVD256,
29811 IX86_BUILTIN_PRORD256,
29812 IX86_BUILTIN_PROLD256,
29813 IX86_BUILTIN_PRORVD128,
29814 IX86_BUILTIN_PROLVD128,
29815 IX86_BUILTIN_PRORD128,
29816 IX86_BUILTIN_PROLD128,
29817 IX86_BUILTIN_FPCLASSPD256,
29818 IX86_BUILTIN_FPCLASSPD128,
29819 IX86_BUILTIN_FPCLASSSD,
29820 IX86_BUILTIN_FPCLASSPS256,
29821 IX86_BUILTIN_FPCLASSPS128,
29822 IX86_BUILTIN_FPCLASSSS,
29823 IX86_BUILTIN_CVTB2MASK128,
29824 IX86_BUILTIN_CVTB2MASK256,
29825 IX86_BUILTIN_CVTW2MASK128,
29826 IX86_BUILTIN_CVTW2MASK256,
29827 IX86_BUILTIN_CVTD2MASK128,
29828 IX86_BUILTIN_CVTD2MASK256,
29829 IX86_BUILTIN_CVTQ2MASK128,
29830 IX86_BUILTIN_CVTQ2MASK256,
29831 IX86_BUILTIN_CVTMASK2B128,
29832 IX86_BUILTIN_CVTMASK2B256,
29833 IX86_BUILTIN_CVTMASK2W128,
29834 IX86_BUILTIN_CVTMASK2W256,
29835 IX86_BUILTIN_CVTMASK2D128,
29836 IX86_BUILTIN_CVTMASK2D256,
29837 IX86_BUILTIN_CVTMASK2Q128,
29838 IX86_BUILTIN_CVTMASK2Q256,
29839 IX86_BUILTIN_PCMPEQB128_MASK,
29840 IX86_BUILTIN_PCMPEQB256_MASK,
29841 IX86_BUILTIN_PCMPEQW128_MASK,
29842 IX86_BUILTIN_PCMPEQW256_MASK,
29843 IX86_BUILTIN_PCMPEQD128_MASK,
29844 IX86_BUILTIN_PCMPEQD256_MASK,
29845 IX86_BUILTIN_PCMPEQQ128_MASK,
29846 IX86_BUILTIN_PCMPEQQ256_MASK,
29847 IX86_BUILTIN_PCMPGTB128_MASK,
29848 IX86_BUILTIN_PCMPGTB256_MASK,
29849 IX86_BUILTIN_PCMPGTW128_MASK,
29850 IX86_BUILTIN_PCMPGTW256_MASK,
29851 IX86_BUILTIN_PCMPGTD128_MASK,
29852 IX86_BUILTIN_PCMPGTD256_MASK,
29853 IX86_BUILTIN_PCMPGTQ128_MASK,
29854 IX86_BUILTIN_PCMPGTQ256_MASK,
29855 IX86_BUILTIN_PTESTMB128,
29856 IX86_BUILTIN_PTESTMB256,
29857 IX86_BUILTIN_PTESTMW128,
29858 IX86_BUILTIN_PTESTMW256,
29859 IX86_BUILTIN_PTESTMD128,
29860 IX86_BUILTIN_PTESTMD256,
29861 IX86_BUILTIN_PTESTMQ128,
29862 IX86_BUILTIN_PTESTMQ256,
29863 IX86_BUILTIN_PTESTNMB128,
29864 IX86_BUILTIN_PTESTNMB256,
29865 IX86_BUILTIN_PTESTNMW128,
29866 IX86_BUILTIN_PTESTNMW256,
29867 IX86_BUILTIN_PTESTNMD128,
29868 IX86_BUILTIN_PTESTNMD256,
29869 IX86_BUILTIN_PTESTNMQ128,
29870 IX86_BUILTIN_PTESTNMQ256,
29871 IX86_BUILTIN_PBROADCASTMB128,
29872 IX86_BUILTIN_PBROADCASTMB256,
29873 IX86_BUILTIN_PBROADCASTMW128,
29874 IX86_BUILTIN_PBROADCASTMW256,
29875 IX86_BUILTIN_COMPRESSPD256,
29876 IX86_BUILTIN_COMPRESSPD128,
29877 IX86_BUILTIN_COMPRESSPS256,
29878 IX86_BUILTIN_COMPRESSPS128,
29879 IX86_BUILTIN_PCOMPRESSQ256,
29880 IX86_BUILTIN_PCOMPRESSQ128,
29881 IX86_BUILTIN_PCOMPRESSD256,
29882 IX86_BUILTIN_PCOMPRESSD128,
29883 IX86_BUILTIN_EXPANDPD256,
29884 IX86_BUILTIN_EXPANDPD128,
29885 IX86_BUILTIN_EXPANDPS256,
29886 IX86_BUILTIN_EXPANDPS128,
29887 IX86_BUILTIN_PEXPANDQ256,
29888 IX86_BUILTIN_PEXPANDQ128,
29889 IX86_BUILTIN_PEXPANDD256,
29890 IX86_BUILTIN_PEXPANDD128,
29891 IX86_BUILTIN_EXPANDPD256Z,
29892 IX86_BUILTIN_EXPANDPD128Z,
29893 IX86_BUILTIN_EXPANDPS256Z,
29894 IX86_BUILTIN_EXPANDPS128Z,
29895 IX86_BUILTIN_PEXPANDQ256Z,
29896 IX86_BUILTIN_PEXPANDQ128Z,
29897 IX86_BUILTIN_PEXPANDD256Z,
29898 IX86_BUILTIN_PEXPANDD128Z,
29899 IX86_BUILTIN_PMAXSD256_MASK,
29900 IX86_BUILTIN_PMINSD256_MASK,
29901 IX86_BUILTIN_PMAXUD256_MASK,
29902 IX86_BUILTIN_PMINUD256_MASK,
29903 IX86_BUILTIN_PMAXSD128_MASK,
29904 IX86_BUILTIN_PMINSD128_MASK,
29905 IX86_BUILTIN_PMAXUD128_MASK,
29906 IX86_BUILTIN_PMINUD128_MASK,
29907 IX86_BUILTIN_PMAXSQ256_MASK,
29908 IX86_BUILTIN_PMINSQ256_MASK,
29909 IX86_BUILTIN_PMAXUQ256_MASK,
29910 IX86_BUILTIN_PMINUQ256_MASK,
29911 IX86_BUILTIN_PMAXSQ128_MASK,
29912 IX86_BUILTIN_PMINSQ128_MASK,
29913 IX86_BUILTIN_PMAXUQ128_MASK,
29914 IX86_BUILTIN_PMINUQ128_MASK,
29915 IX86_BUILTIN_PMINSB256_MASK,
29916 IX86_BUILTIN_PMINUB256_MASK,
29917 IX86_BUILTIN_PMAXSB256_MASK,
29918 IX86_BUILTIN_PMAXUB256_MASK,
29919 IX86_BUILTIN_PMINSB128_MASK,
29920 IX86_BUILTIN_PMINUB128_MASK,
29921 IX86_BUILTIN_PMAXSB128_MASK,
29922 IX86_BUILTIN_PMAXUB128_MASK,
29923 IX86_BUILTIN_PMINSW256_MASK,
29924 IX86_BUILTIN_PMINUW256_MASK,
29925 IX86_BUILTIN_PMAXSW256_MASK,
29926 IX86_BUILTIN_PMAXUW256_MASK,
29927 IX86_BUILTIN_PMINSW128_MASK,
29928 IX86_BUILTIN_PMINUW128_MASK,
29929 IX86_BUILTIN_PMAXSW128_MASK,
29930 IX86_BUILTIN_PMAXUW128_MASK,
29931 IX86_BUILTIN_VPCONFLICTQ256,
29932 IX86_BUILTIN_VPCONFLICTD256,
29933 IX86_BUILTIN_VPCLZCNTQ256,
29934 IX86_BUILTIN_VPCLZCNTD256,
29935 IX86_BUILTIN_UNPCKHPD256_MASK,
29936 IX86_BUILTIN_UNPCKHPD128_MASK,
29937 IX86_BUILTIN_UNPCKHPS256_MASK,
29938 IX86_BUILTIN_UNPCKHPS128_MASK,
29939 IX86_BUILTIN_UNPCKLPD256_MASK,
29940 IX86_BUILTIN_UNPCKLPD128_MASK,
29941 IX86_BUILTIN_UNPCKLPS256_MASK,
29942 IX86_BUILTIN_VPCONFLICTQ128,
29943 IX86_BUILTIN_VPCONFLICTD128,
29944 IX86_BUILTIN_VPCLZCNTQ128,
29945 IX86_BUILTIN_VPCLZCNTD128,
29946 IX86_BUILTIN_UNPCKLPS128_MASK,
29947 IX86_BUILTIN_ALIGND256,
29948 IX86_BUILTIN_ALIGNQ256,
29949 IX86_BUILTIN_ALIGND128,
29950 IX86_BUILTIN_ALIGNQ128,
29951 IX86_BUILTIN_CVTPS2PH256_MASK,
29952 IX86_BUILTIN_CVTPS2PH_MASK,
29953 IX86_BUILTIN_CVTPH2PS_MASK,
29954 IX86_BUILTIN_CVTPH2PS256_MASK,
29955 IX86_BUILTIN_PUNPCKHDQ128_MASK,
29956 IX86_BUILTIN_PUNPCKHDQ256_MASK,
29957 IX86_BUILTIN_PUNPCKHQDQ128_MASK,
29958 IX86_BUILTIN_PUNPCKHQDQ256_MASK,
29959 IX86_BUILTIN_PUNPCKLDQ128_MASK,
29960 IX86_BUILTIN_PUNPCKLDQ256_MASK,
29961 IX86_BUILTIN_PUNPCKLQDQ128_MASK,
29962 IX86_BUILTIN_PUNPCKLQDQ256_MASK,
29963 IX86_BUILTIN_PUNPCKHBW128_MASK,
29964 IX86_BUILTIN_PUNPCKHBW256_MASK,
29965 IX86_BUILTIN_PUNPCKHWD128_MASK,
29966 IX86_BUILTIN_PUNPCKHWD256_MASK,
29967 IX86_BUILTIN_PUNPCKLBW128_MASK,
29968 IX86_BUILTIN_PUNPCKLBW256_MASK,
29969 IX86_BUILTIN_PUNPCKLWD128_MASK,
29970 IX86_BUILTIN_PUNPCKLWD256_MASK,
29971 IX86_BUILTIN_PSLLVV16HI,
29972 IX86_BUILTIN_PSLLVV8HI,
29973 IX86_BUILTIN_PACKSSDW256_MASK,
29974 IX86_BUILTIN_PACKSSDW128_MASK,
29975 IX86_BUILTIN_PACKUSDW256_MASK,
29976 IX86_BUILTIN_PACKUSDW128_MASK,
29977 IX86_BUILTIN_PAVGB256_MASK,
29978 IX86_BUILTIN_PAVGW256_MASK,
29979 IX86_BUILTIN_PAVGB128_MASK,
29980 IX86_BUILTIN_PAVGW128_MASK,
29981 IX86_BUILTIN_VPERMVARSF256_MASK,
29982 IX86_BUILTIN_VPERMVARDF256_MASK,
29983 IX86_BUILTIN_VPERMDF256_MASK,
29984 IX86_BUILTIN_PABSB256_MASK,
29985 IX86_BUILTIN_PABSB128_MASK,
29986 IX86_BUILTIN_PABSW256_MASK,
29987 IX86_BUILTIN_PABSW128_MASK,
29988 IX86_BUILTIN_VPERMILVARPD_MASK,
29989 IX86_BUILTIN_VPERMILVARPS_MASK,
29990 IX86_BUILTIN_VPERMILVARPD256_MASK,
29991 IX86_BUILTIN_VPERMILVARPS256_MASK,
29992 IX86_BUILTIN_VPERMILPD_MASK,
29993 IX86_BUILTIN_VPERMILPS_MASK,
29994 IX86_BUILTIN_VPERMILPD256_MASK,
29995 IX86_BUILTIN_VPERMILPS256_MASK,
29996 IX86_BUILTIN_BLENDMQ256,
29997 IX86_BUILTIN_BLENDMD256,
29998 IX86_BUILTIN_BLENDMPD256,
29999 IX86_BUILTIN_BLENDMPS256,
30000 IX86_BUILTIN_BLENDMQ128,
30001 IX86_BUILTIN_BLENDMD128,
30002 IX86_BUILTIN_BLENDMPD128,
30003 IX86_BUILTIN_BLENDMPS128,
30004 IX86_BUILTIN_BLENDMW256,
30005 IX86_BUILTIN_BLENDMB256,
30006 IX86_BUILTIN_BLENDMW128,
30007 IX86_BUILTIN_BLENDMB128,
30008 IX86_BUILTIN_PMULLD256_MASK,
30009 IX86_BUILTIN_PMULLD128_MASK,
30010 IX86_BUILTIN_PMULUDQ256_MASK,
30011 IX86_BUILTIN_PMULDQ256_MASK,
30012 IX86_BUILTIN_PMULDQ128_MASK,
30013 IX86_BUILTIN_PMULUDQ128_MASK,
30014 IX86_BUILTIN_CVTPD2PS256_MASK,
30015 IX86_BUILTIN_CVTPD2PS_MASK,
30016 IX86_BUILTIN_VPERMVARSI256_MASK,
30017 IX86_BUILTIN_VPERMVARDI256_MASK,
30018 IX86_BUILTIN_VPERMDI256_MASK,
30019 IX86_BUILTIN_CMPQ256,
30020 IX86_BUILTIN_CMPD256,
30021 IX86_BUILTIN_UCMPQ256,
30022 IX86_BUILTIN_UCMPD256,
30023 IX86_BUILTIN_CMPB256,
30024 IX86_BUILTIN_CMPW256,
30025 IX86_BUILTIN_UCMPB256,
30026 IX86_BUILTIN_UCMPW256,
30027 IX86_BUILTIN_CMPPD256_MASK,
30028 IX86_BUILTIN_CMPPS256_MASK,
30029 IX86_BUILTIN_CMPQ128,
30030 IX86_BUILTIN_CMPD128,
30031 IX86_BUILTIN_UCMPQ128,
30032 IX86_BUILTIN_UCMPD128,
30033 IX86_BUILTIN_CMPB128,
30034 IX86_BUILTIN_CMPW128,
30035 IX86_BUILTIN_UCMPB128,
30036 IX86_BUILTIN_UCMPW128,
30037 IX86_BUILTIN_CMPPD128_MASK,
30038 IX86_BUILTIN_CMPPS128_MASK,
30040 IX86_BUILTIN_GATHER3SIV8SF,
30041 IX86_BUILTIN_GATHER3SIV4SF,
30042 IX86_BUILTIN_GATHER3SIV4DF,
30043 IX86_BUILTIN_GATHER3SIV2DF,
30044 IX86_BUILTIN_GATHER3DIV8SF,
30045 IX86_BUILTIN_GATHER3DIV4SF,
30046 IX86_BUILTIN_GATHER3DIV4DF,
30047 IX86_BUILTIN_GATHER3DIV2DF,
30048 IX86_BUILTIN_GATHER3SIV8SI,
30049 IX86_BUILTIN_GATHER3SIV4SI,
30050 IX86_BUILTIN_GATHER3SIV4DI,
30051 IX86_BUILTIN_GATHER3SIV2DI,
30052 IX86_BUILTIN_GATHER3DIV8SI,
30053 IX86_BUILTIN_GATHER3DIV4SI,
30054 IX86_BUILTIN_GATHER3DIV4DI,
30055 IX86_BUILTIN_GATHER3DIV2DI,
30056 IX86_BUILTIN_SCATTERSIV8SF,
30057 IX86_BUILTIN_SCATTERSIV4SF,
30058 IX86_BUILTIN_SCATTERSIV4DF,
30059 IX86_BUILTIN_SCATTERSIV2DF,
30060 IX86_BUILTIN_SCATTERDIV8SF,
30061 IX86_BUILTIN_SCATTERDIV4SF,
30062 IX86_BUILTIN_SCATTERDIV4DF,
30063 IX86_BUILTIN_SCATTERDIV2DF,
30064 IX86_BUILTIN_SCATTERSIV8SI,
30065 IX86_BUILTIN_SCATTERSIV4SI,
30066 IX86_BUILTIN_SCATTERSIV4DI,
30067 IX86_BUILTIN_SCATTERSIV2DI,
30068 IX86_BUILTIN_SCATTERDIV8SI,
30069 IX86_BUILTIN_SCATTERDIV4SI,
30070 IX86_BUILTIN_SCATTERDIV4DI,
30071 IX86_BUILTIN_SCATTERDIV2DI,
30074 IX86_BUILTIN_RANGESD128,
30075 IX86_BUILTIN_RANGESS128,
30076 IX86_BUILTIN_KUNPCKWD,
30077 IX86_BUILTIN_KUNPCKDQ,
30078 IX86_BUILTIN_BROADCASTF32x2_512,
30079 IX86_BUILTIN_BROADCASTI32x2_512,
30080 IX86_BUILTIN_BROADCASTF64X2_512,
30081 IX86_BUILTIN_BROADCASTI64X2_512,
30082 IX86_BUILTIN_BROADCASTF32X8_512,
30083 IX86_BUILTIN_BROADCASTI32X8_512,
30084 IX86_BUILTIN_EXTRACTF64X2_512,
30085 IX86_BUILTIN_EXTRACTF32X8,
30086 IX86_BUILTIN_EXTRACTI64X2_512,
30087 IX86_BUILTIN_EXTRACTI32X8,
30088 IX86_BUILTIN_REDUCEPD512_MASK,
30089 IX86_BUILTIN_REDUCEPS512_MASK,
30090 IX86_BUILTIN_PMULLQ512,
30091 IX86_BUILTIN_XORPD512,
30092 IX86_BUILTIN_XORPS512,
30093 IX86_BUILTIN_ORPD512,
30094 IX86_BUILTIN_ORPS512,
30095 IX86_BUILTIN_ANDPD512,
30096 IX86_BUILTIN_ANDPS512,
30097 IX86_BUILTIN_ANDNPD512,
30098 IX86_BUILTIN_ANDNPS512,
30099 IX86_BUILTIN_INSERTF32X8,
30100 IX86_BUILTIN_INSERTI32X8,
30101 IX86_BUILTIN_INSERTF64X2_512,
30102 IX86_BUILTIN_INSERTI64X2_512,
30103 IX86_BUILTIN_FPCLASSPD512,
30104 IX86_BUILTIN_FPCLASSPS512,
30105 IX86_BUILTIN_CVTD2MASK512,
30106 IX86_BUILTIN_CVTQ2MASK512,
30107 IX86_BUILTIN_CVTMASK2D512,
30108 IX86_BUILTIN_CVTMASK2Q512,
30109 IX86_BUILTIN_CVTPD2QQ512,
30110 IX86_BUILTIN_CVTPS2QQ512,
30111 IX86_BUILTIN_CVTPD2UQQ512,
30112 IX86_BUILTIN_CVTPS2UQQ512,
30113 IX86_BUILTIN_CVTQQ2PS512,
30114 IX86_BUILTIN_CVTUQQ2PS512,
30115 IX86_BUILTIN_CVTQQ2PD512,
30116 IX86_BUILTIN_CVTUQQ2PD512,
30117 IX86_BUILTIN_CVTTPS2QQ512,
30118 IX86_BUILTIN_CVTTPS2UQQ512,
30119 IX86_BUILTIN_CVTTPD2QQ512,
30120 IX86_BUILTIN_CVTTPD2UQQ512,
30121 IX86_BUILTIN_RANGEPS512,
30122 IX86_BUILTIN_RANGEPD512,
30125 IX86_BUILTIN_PACKUSDW512,
30126 IX86_BUILTIN_PACKSSDW512,
30127 IX86_BUILTIN_LOADDQUHI512_MASK,
30128 IX86_BUILTIN_LOADDQUQI512_MASK,
30129 IX86_BUILTIN_PSLLDQ512,
30130 IX86_BUILTIN_PSRLDQ512,
30131 IX86_BUILTIN_STOREDQUHI512_MASK,
30132 IX86_BUILTIN_STOREDQUQI512_MASK,
30133 IX86_BUILTIN_PALIGNR512,
30134 IX86_BUILTIN_PALIGNR512_MASK,
30135 IX86_BUILTIN_MOVDQUHI512_MASK,
30136 IX86_BUILTIN_MOVDQUQI512_MASK,
30137 IX86_BUILTIN_PSADBW512,
30138 IX86_BUILTIN_DBPSADBW512,
30139 IX86_BUILTIN_PBROADCASTB512,
30140 IX86_BUILTIN_PBROADCASTB512_GPR,
30141 IX86_BUILTIN_PBROADCASTW512,
30142 IX86_BUILTIN_PBROADCASTW512_GPR,
30143 IX86_BUILTIN_PMOVSXBW512_MASK,
30144 IX86_BUILTIN_PMOVZXBW512_MASK,
30145 IX86_BUILTIN_VPERMVARHI512_MASK,
30146 IX86_BUILTIN_VPERMT2VARHI512,
30147 IX86_BUILTIN_VPERMT2VARHI512_MASKZ,
30148 IX86_BUILTIN_VPERMI2VARHI512,
30149 IX86_BUILTIN_PAVGB512,
30150 IX86_BUILTIN_PAVGW512,
30151 IX86_BUILTIN_PADDB512,
30152 IX86_BUILTIN_PSUBB512,
30153 IX86_BUILTIN_PSUBSB512,
30154 IX86_BUILTIN_PADDSB512,
30155 IX86_BUILTIN_PSUBUSB512,
30156 IX86_BUILTIN_PADDUSB512,
30157 IX86_BUILTIN_PSUBW512,
30158 IX86_BUILTIN_PADDW512,
30159 IX86_BUILTIN_PSUBSW512,
30160 IX86_BUILTIN_PADDSW512,
30161 IX86_BUILTIN_PSUBUSW512,
30162 IX86_BUILTIN_PADDUSW512,
30163 IX86_BUILTIN_PMAXUW512,
30164 IX86_BUILTIN_PMAXSW512,
30165 IX86_BUILTIN_PMINUW512,
30166 IX86_BUILTIN_PMINSW512,
30167 IX86_BUILTIN_PMAXUB512,
30168 IX86_BUILTIN_PMAXSB512,
30169 IX86_BUILTIN_PMINUB512,
30170 IX86_BUILTIN_PMINSB512,
30171 IX86_BUILTIN_PMOVWB512,
30172 IX86_BUILTIN_PMOVSWB512,
30173 IX86_BUILTIN_PMOVUSWB512,
30174 IX86_BUILTIN_PMULHRSW512_MASK,
30175 IX86_BUILTIN_PMULHUW512_MASK,
30176 IX86_BUILTIN_PMULHW512_MASK,
30177 IX86_BUILTIN_PMULLW512_MASK,
30178 IX86_BUILTIN_PSLLWI512_MASK,
30179 IX86_BUILTIN_PSLLW512_MASK,
30180 IX86_BUILTIN_PACKSSWB512,
30181 IX86_BUILTIN_PACKUSWB512,
30182 IX86_BUILTIN_PSRAVV32HI,
30183 IX86_BUILTIN_PMADDUBSW512_MASK,
30184 IX86_BUILTIN_PMADDWD512_MASK,
30185 IX86_BUILTIN_PSRLVV32HI,
30186 IX86_BUILTIN_PUNPCKHBW512,
30187 IX86_BUILTIN_PUNPCKHWD512,
30188 IX86_BUILTIN_PUNPCKLBW512,
30189 IX86_BUILTIN_PUNPCKLWD512,
30190 IX86_BUILTIN_PSHUFB512,
30191 IX86_BUILTIN_PSHUFHW512,
30192 IX86_BUILTIN_PSHUFLW512,
30193 IX86_BUILTIN_PSRAWI512,
30194 IX86_BUILTIN_PSRAW512,
30195 IX86_BUILTIN_PSRLWI512,
30196 IX86_BUILTIN_PSRLW512,
30197 IX86_BUILTIN_CVTB2MASK512,
30198 IX86_BUILTIN_CVTW2MASK512,
30199 IX86_BUILTIN_CVTMASK2B512,
30200 IX86_BUILTIN_CVTMASK2W512,
30201 IX86_BUILTIN_PCMPEQB512_MASK,
30202 IX86_BUILTIN_PCMPEQW512_MASK,
30203 IX86_BUILTIN_PCMPGTB512_MASK,
30204 IX86_BUILTIN_PCMPGTW512_MASK,
30205 IX86_BUILTIN_PTESTMB512,
30206 IX86_BUILTIN_PTESTMW512,
30207 IX86_BUILTIN_PTESTNMB512,
30208 IX86_BUILTIN_PTESTNMW512,
30209 IX86_BUILTIN_PSLLVV32HI,
30210 IX86_BUILTIN_PABSB512,
30211 IX86_BUILTIN_PABSW512,
30212 IX86_BUILTIN_BLENDMW512,
30213 IX86_BUILTIN_BLENDMB512,
30214 IX86_BUILTIN_CMPB512,
30215 IX86_BUILTIN_CMPW512,
30216 IX86_BUILTIN_UCMPB512,
30217 IX86_BUILTIN_UCMPW512,
30219 /* Alternate 4 and 8 element gather/scatter for the vectorizer
30220 where all operands are 32-byte or 64-byte wide respectively. */
30221 IX86_BUILTIN_GATHERALTSIV4DF,
30222 IX86_BUILTIN_GATHERALTDIV8SF,
30223 IX86_BUILTIN_GATHERALTSIV4DI,
30224 IX86_BUILTIN_GATHERALTDIV8SI,
30225 IX86_BUILTIN_GATHER3ALTDIV16SF,
30226 IX86_BUILTIN_GATHER3ALTDIV16SI,
30227 IX86_BUILTIN_GATHER3ALTSIV4DF,
30228 IX86_BUILTIN_GATHER3ALTDIV8SF,
30229 IX86_BUILTIN_GATHER3ALTSIV4DI,
30230 IX86_BUILTIN_GATHER3ALTDIV8SI,
30231 IX86_BUILTIN_GATHER3ALTSIV8DF,
30232 IX86_BUILTIN_GATHER3ALTSIV8DI,
30233 IX86_BUILTIN_GATHER3DIV16SF,
30234 IX86_BUILTIN_GATHER3DIV16SI,
30235 IX86_BUILTIN_GATHER3DIV8DF,
30236 IX86_BUILTIN_GATHER3DIV8DI,
30237 IX86_BUILTIN_GATHER3SIV16SF,
30238 IX86_BUILTIN_GATHER3SIV16SI,
30239 IX86_BUILTIN_GATHER3SIV8DF,
30240 IX86_BUILTIN_GATHER3SIV8DI,
30241 IX86_BUILTIN_SCATTERDIV16SF,
30242 IX86_BUILTIN_SCATTERDIV16SI,
30243 IX86_BUILTIN_SCATTERDIV8DF,
30244 IX86_BUILTIN_SCATTERDIV8DI,
30245 IX86_BUILTIN_SCATTERSIV16SF,
30246 IX86_BUILTIN_SCATTERSIV16SI,
30247 IX86_BUILTIN_SCATTERSIV8DF,
30248 IX86_BUILTIN_SCATTERSIV8DI,
30251 IX86_BUILTIN_GATHERPFQPD,
30252 IX86_BUILTIN_GATHERPFDPS,
30253 IX86_BUILTIN_GATHERPFDPD,
30254 IX86_BUILTIN_GATHERPFQPS,
30255 IX86_BUILTIN_SCATTERPFDPD,
30256 IX86_BUILTIN_SCATTERPFDPS,
30257 IX86_BUILTIN_SCATTERPFQPD,
30258 IX86_BUILTIN_SCATTERPFQPS,
30261 IX86_BUILTIN_EXP2PD_MASK,
30262 IX86_BUILTIN_EXP2PS_MASK,
30263 IX86_BUILTIN_EXP2PS,
30264 IX86_BUILTIN_RCP28PD,
30265 IX86_BUILTIN_RCP28PS,
30266 IX86_BUILTIN_RCP28SD,
30267 IX86_BUILTIN_RCP28SS,
30268 IX86_BUILTIN_RSQRT28PD,
30269 IX86_BUILTIN_RSQRT28PS,
30270 IX86_BUILTIN_RSQRT28SD,
30271 IX86_BUILTIN_RSQRT28SS,
30274 IX86_BUILTIN_VPMADD52LUQ512,
30275 IX86_BUILTIN_VPMADD52HUQ512,
30276 IX86_BUILTIN_VPMADD52LUQ256,
30277 IX86_BUILTIN_VPMADD52HUQ256,
30278 IX86_BUILTIN_VPMADD52LUQ128,
30279 IX86_BUILTIN_VPMADD52HUQ128,
30280 IX86_BUILTIN_VPMADD52LUQ512_MASKZ,
30281 IX86_BUILTIN_VPMADD52HUQ512_MASKZ,
30282 IX86_BUILTIN_VPMADD52LUQ256_MASKZ,
30283 IX86_BUILTIN_VPMADD52HUQ256_MASKZ,
30284 IX86_BUILTIN_VPMADD52LUQ128_MASKZ,
30285 IX86_BUILTIN_VPMADD52HUQ128_MASKZ,
30288 IX86_BUILTIN_VPMULTISHIFTQB512,
30289 IX86_BUILTIN_VPMULTISHIFTQB256,
30290 IX86_BUILTIN_VPMULTISHIFTQB128,
30291 IX86_BUILTIN_VPERMVARQI512_MASK,
30292 IX86_BUILTIN_VPERMT2VARQI512,
30293 IX86_BUILTIN_VPERMT2VARQI512_MASKZ,
30294 IX86_BUILTIN_VPERMI2VARQI512,
30295 IX86_BUILTIN_VPERMVARQI256_MASK,
30296 IX86_BUILTIN_VPERMVARQI128_MASK,
30297 IX86_BUILTIN_VPERMT2VARQI256,
30298 IX86_BUILTIN_VPERMT2VARQI256_MASKZ,
30299 IX86_BUILTIN_VPERMT2VARQI128,
30300 IX86_BUILTIN_VPERMT2VARQI128_MASKZ,
30301 IX86_BUILTIN_VPERMI2VARQI256,
30302 IX86_BUILTIN_VPERMI2VARQI128,
30304 /* SHA builtins. */
30305 IX86_BUILTIN_SHA1MSG1,
30306 IX86_BUILTIN_SHA1MSG2,
30307 IX86_BUILTIN_SHA1NEXTE,
30308 IX86_BUILTIN_SHA1RNDS4,
30309 IX86_BUILTIN_SHA256MSG1,
30310 IX86_BUILTIN_SHA256MSG2,
30311 IX86_BUILTIN_SHA256RNDS2,
30313 /* CLWB instructions. */
30316 /* PCOMMIT instructions. */
30317 IX86_BUILTIN_PCOMMIT,
30319 /* CLFLUSHOPT instructions. */
30320 IX86_BUILTIN_CLFLUSHOPT,
30322 /* TFmode support builtins. */
30324 IX86_BUILTIN_HUGE_VALQ,
30325 IX86_BUILTIN_FABSQ,
30326 IX86_BUILTIN_COPYSIGNQ,
30328 /* Vectorizer support builtins. */
30329 IX86_BUILTIN_CEILPD_VEC_PACK_SFIX512,
30330 IX86_BUILTIN_CPYSGNPS,
30331 IX86_BUILTIN_CPYSGNPD,
30332 IX86_BUILTIN_CPYSGNPS256,
30333 IX86_BUILTIN_CPYSGNPS512,
30334 IX86_BUILTIN_CPYSGNPD256,
30335 IX86_BUILTIN_CPYSGNPD512,
30336 IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX512,
30337 IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX512,
30340 /* FMA4 instructions. */
30341 IX86_BUILTIN_VFMADDSS,
30342 IX86_BUILTIN_VFMADDSD,
30343 IX86_BUILTIN_VFMADDPS,
30344 IX86_BUILTIN_VFMADDPD,
30345 IX86_BUILTIN_VFMADDPS256,
30346 IX86_BUILTIN_VFMADDPD256,
30347 IX86_BUILTIN_VFMADDSUBPS,
30348 IX86_BUILTIN_VFMADDSUBPD,
30349 IX86_BUILTIN_VFMADDSUBPS256,
30350 IX86_BUILTIN_VFMADDSUBPD256,
30352 /* FMA3 instructions. */
30353 IX86_BUILTIN_VFMADDSS3,
30354 IX86_BUILTIN_VFMADDSD3,
30356 /* XOP instructions. */
30357 IX86_BUILTIN_VPCMOV,
30358 IX86_BUILTIN_VPCMOV_V2DI,
30359 IX86_BUILTIN_VPCMOV_V4SI,
30360 IX86_BUILTIN_VPCMOV_V8HI,
30361 IX86_BUILTIN_VPCMOV_V16QI,
30362 IX86_BUILTIN_VPCMOV_V4SF,
30363 IX86_BUILTIN_VPCMOV_V2DF,
30364 IX86_BUILTIN_VPCMOV256,
30365 IX86_BUILTIN_VPCMOV_V4DI256,
30366 IX86_BUILTIN_VPCMOV_V8SI256,
30367 IX86_BUILTIN_VPCMOV_V16HI256,
30368 IX86_BUILTIN_VPCMOV_V32QI256,
30369 IX86_BUILTIN_VPCMOV_V8SF256,
30370 IX86_BUILTIN_VPCMOV_V4DF256,
30372 IX86_BUILTIN_VPPERM,
30374 IX86_BUILTIN_VPMACSSWW,
30375 IX86_BUILTIN_VPMACSWW,
30376 IX86_BUILTIN_VPMACSSWD,
30377 IX86_BUILTIN_VPMACSWD,
30378 IX86_BUILTIN_VPMACSSDD,
30379 IX86_BUILTIN_VPMACSDD,
30380 IX86_BUILTIN_VPMACSSDQL,
30381 IX86_BUILTIN_VPMACSSDQH,
30382 IX86_BUILTIN_VPMACSDQL,
30383 IX86_BUILTIN_VPMACSDQH,
30384 IX86_BUILTIN_VPMADCSSWD,
30385 IX86_BUILTIN_VPMADCSWD,
30387 IX86_BUILTIN_VPHADDBW,
30388 IX86_BUILTIN_VPHADDBD,
30389 IX86_BUILTIN_VPHADDBQ,
30390 IX86_BUILTIN_VPHADDWD,
30391 IX86_BUILTIN_VPHADDWQ,
30392 IX86_BUILTIN_VPHADDDQ,
30393 IX86_BUILTIN_VPHADDUBW,
30394 IX86_BUILTIN_VPHADDUBD,
30395 IX86_BUILTIN_VPHADDUBQ,
30396 IX86_BUILTIN_VPHADDUWD,
30397 IX86_BUILTIN_VPHADDUWQ,
30398 IX86_BUILTIN_VPHADDUDQ,
30399 IX86_BUILTIN_VPHSUBBW,
30400 IX86_BUILTIN_VPHSUBWD,
30401 IX86_BUILTIN_VPHSUBDQ,
30403 IX86_BUILTIN_VPROTB,
30404 IX86_BUILTIN_VPROTW,
30405 IX86_BUILTIN_VPROTD,
30406 IX86_BUILTIN_VPROTQ,
30407 IX86_BUILTIN_VPROTB_IMM,
30408 IX86_BUILTIN_VPROTW_IMM,
30409 IX86_BUILTIN_VPROTD_IMM,
30410 IX86_BUILTIN_VPROTQ_IMM,
30412 IX86_BUILTIN_VPSHLB,
30413 IX86_BUILTIN_VPSHLW,
30414 IX86_BUILTIN_VPSHLD,
30415 IX86_BUILTIN_VPSHLQ,
30416 IX86_BUILTIN_VPSHAB,
30417 IX86_BUILTIN_VPSHAW,
30418 IX86_BUILTIN_VPSHAD,
30419 IX86_BUILTIN_VPSHAQ,
30421 IX86_BUILTIN_VFRCZSS,
30422 IX86_BUILTIN_VFRCZSD,
30423 IX86_BUILTIN_VFRCZPS,
30424 IX86_BUILTIN_VFRCZPD,
30425 IX86_BUILTIN_VFRCZPS256,
30426 IX86_BUILTIN_VFRCZPD256,
30428 IX86_BUILTIN_VPCOMEQUB,
30429 IX86_BUILTIN_VPCOMNEUB,
30430 IX86_BUILTIN_VPCOMLTUB,
30431 IX86_BUILTIN_VPCOMLEUB,
30432 IX86_BUILTIN_VPCOMGTUB,
30433 IX86_BUILTIN_VPCOMGEUB,
30434 IX86_BUILTIN_VPCOMFALSEUB,
30435 IX86_BUILTIN_VPCOMTRUEUB,
30437 IX86_BUILTIN_VPCOMEQUW,
30438 IX86_BUILTIN_VPCOMNEUW,
30439 IX86_BUILTIN_VPCOMLTUW,
30440 IX86_BUILTIN_VPCOMLEUW,
30441 IX86_BUILTIN_VPCOMGTUW,
30442 IX86_BUILTIN_VPCOMGEUW,
30443 IX86_BUILTIN_VPCOMFALSEUW,
30444 IX86_BUILTIN_VPCOMTRUEUW,
30446 IX86_BUILTIN_VPCOMEQUD,
30447 IX86_BUILTIN_VPCOMNEUD,
30448 IX86_BUILTIN_VPCOMLTUD,
30449 IX86_BUILTIN_VPCOMLEUD,
30450 IX86_BUILTIN_VPCOMGTUD,
30451 IX86_BUILTIN_VPCOMGEUD,
30452 IX86_BUILTIN_VPCOMFALSEUD,
30453 IX86_BUILTIN_VPCOMTRUEUD,
30455 IX86_BUILTIN_VPCOMEQUQ,
30456 IX86_BUILTIN_VPCOMNEUQ,
30457 IX86_BUILTIN_VPCOMLTUQ,
30458 IX86_BUILTIN_VPCOMLEUQ,
30459 IX86_BUILTIN_VPCOMGTUQ,
30460 IX86_BUILTIN_VPCOMGEUQ,
30461 IX86_BUILTIN_VPCOMFALSEUQ,
30462 IX86_BUILTIN_VPCOMTRUEUQ,
30464 IX86_BUILTIN_VPCOMEQB,
30465 IX86_BUILTIN_VPCOMNEB,
30466 IX86_BUILTIN_VPCOMLTB,
30467 IX86_BUILTIN_VPCOMLEB,
30468 IX86_BUILTIN_VPCOMGTB,
30469 IX86_BUILTIN_VPCOMGEB,
30470 IX86_BUILTIN_VPCOMFALSEB,
30471 IX86_BUILTIN_VPCOMTRUEB,
30473 IX86_BUILTIN_VPCOMEQW,
30474 IX86_BUILTIN_VPCOMNEW,
30475 IX86_BUILTIN_VPCOMLTW,
30476 IX86_BUILTIN_VPCOMLEW,
30477 IX86_BUILTIN_VPCOMGTW,
30478 IX86_BUILTIN_VPCOMGEW,
30479 IX86_BUILTIN_VPCOMFALSEW,
30480 IX86_BUILTIN_VPCOMTRUEW,
30482 IX86_BUILTIN_VPCOMEQD,
30483 IX86_BUILTIN_VPCOMNED,
30484 IX86_BUILTIN_VPCOMLTD,
30485 IX86_BUILTIN_VPCOMLED,
30486 IX86_BUILTIN_VPCOMGTD,
30487 IX86_BUILTIN_VPCOMGED,
30488 IX86_BUILTIN_VPCOMFALSED,
30489 IX86_BUILTIN_VPCOMTRUED,
30491 IX86_BUILTIN_VPCOMEQQ,
30492 IX86_BUILTIN_VPCOMNEQ,
30493 IX86_BUILTIN_VPCOMLTQ,
30494 IX86_BUILTIN_VPCOMLEQ,
30495 IX86_BUILTIN_VPCOMGTQ,
30496 IX86_BUILTIN_VPCOMGEQ,
30497 IX86_BUILTIN_VPCOMFALSEQ,
30498 IX86_BUILTIN_VPCOMTRUEQ,
30500 /* LWP instructions. */
30501 IX86_BUILTIN_LLWPCB,
30502 IX86_BUILTIN_SLWPCB,
30503 IX86_BUILTIN_LWPVAL32,
30504 IX86_BUILTIN_LWPVAL64,
30505 IX86_BUILTIN_LWPINS32,
30506 IX86_BUILTIN_LWPINS64,
30511 IX86_BUILTIN_XBEGIN,
30513 IX86_BUILTIN_XABORT,
30514 IX86_BUILTIN_XTEST,
30517 IX86_BUILTIN_BNDMK,
30518 IX86_BUILTIN_BNDSTX,
30519 IX86_BUILTIN_BNDLDX,
30520 IX86_BUILTIN_BNDCL,
30521 IX86_BUILTIN_BNDCU,
30522 IX86_BUILTIN_BNDRET,
30523 IX86_BUILTIN_BNDNARROW,
30524 IX86_BUILTIN_BNDINT,
30525 IX86_BUILTIN_SIZEOF,
30526 IX86_BUILTIN_BNDLOWER,
30527 IX86_BUILTIN_BNDUPPER,
30529 /* BMI instructions. */
30530 IX86_BUILTIN_BEXTR32,
30531 IX86_BUILTIN_BEXTR64,
30534 /* TBM instructions. */
30535 IX86_BUILTIN_BEXTRI32,
30536 IX86_BUILTIN_BEXTRI64,
30538 /* BMI2 instructions. */
30539 IX86_BUILTIN_BZHI32,
30540 IX86_BUILTIN_BZHI64,
30541 IX86_BUILTIN_PDEP32,
30542 IX86_BUILTIN_PDEP64,
30543 IX86_BUILTIN_PEXT32,
30544 IX86_BUILTIN_PEXT64,
30546 /* ADX instructions. */
30547 IX86_BUILTIN_ADDCARRYX32,
30548 IX86_BUILTIN_ADDCARRYX64,
30550 /* SBB instructions. */
30551 IX86_BUILTIN_SBB32,
30552 IX86_BUILTIN_SBB64,
30554 /* FSGSBASE instructions. */
30555 IX86_BUILTIN_RDFSBASE32,
30556 IX86_BUILTIN_RDFSBASE64,
30557 IX86_BUILTIN_RDGSBASE32,
30558 IX86_BUILTIN_RDGSBASE64,
30559 IX86_BUILTIN_WRFSBASE32,
30560 IX86_BUILTIN_WRFSBASE64,
30561 IX86_BUILTIN_WRGSBASE32,
30562 IX86_BUILTIN_WRGSBASE64,
30564 /* RDRND instructions. */
30565 IX86_BUILTIN_RDRAND16_STEP,
30566 IX86_BUILTIN_RDRAND32_STEP,
30567 IX86_BUILTIN_RDRAND64_STEP,
30569 /* RDSEED instructions. */
30570 IX86_BUILTIN_RDSEED16_STEP,
30571 IX86_BUILTIN_RDSEED32_STEP,
30572 IX86_BUILTIN_RDSEED64_STEP,
30574 /* F16C instructions. */
30575 IX86_BUILTIN_CVTPH2PS,
30576 IX86_BUILTIN_CVTPH2PS256,
30577 IX86_BUILTIN_CVTPS2PH,
30578 IX86_BUILTIN_CVTPS2PH256,
30580 /* MONITORX and MWAITX instrucions. */
30581 IX86_BUILTIN_MONITORX,
30582 IX86_BUILTIN_MWAITX,
30584 /* CFString built-in for darwin */
30585 IX86_BUILTIN_CFSTRING,
30587 /* Builtins to get CPU type and supported features. */
30588 IX86_BUILTIN_CPU_INIT,
30589 IX86_BUILTIN_CPU_IS,
30590 IX86_BUILTIN_CPU_SUPPORTS,
30592 /* Read/write FLAGS register built-ins. */
30593 IX86_BUILTIN_READ_FLAGS,
30594 IX86_BUILTIN_WRITE_FLAGS,
30599 /* Table for the ix86 builtin decls. */
30600 static GTY(()) tree ix86_builtins[(int) IX86_BUILTIN_MAX];
30602 /* Table of all of the builtin functions that are possible with different ISA's
30603 but are waiting to be built until a function is declared to use that
30605 struct builtin_isa {
30606 const char *name; /* function name */
30607 enum ix86_builtin_func_type tcode; /* type to use in the declaration */
30608 HOST_WIDE_INT isa; /* isa_flags this builtin is defined for */
30609 bool const_p; /* true if the declaration is constant */
30610 bool leaf_p; /* true if the declaration has leaf attribute */
30611 bool nothrow_p; /* true if the declaration has nothrow attribute */
30612 bool set_and_not_built_p;
30615 static struct builtin_isa ix86_builtins_isa[(int) IX86_BUILTIN_MAX];
30617 /* Bits that can still enable any inclusion of a builtin. */
30618 static HOST_WIDE_INT deferred_isa_values = 0;
30620 /* Add an ix86 target builtin function with CODE, NAME and TYPE. Save the MASK
30621 of which isa_flags to use in the ix86_builtins_isa array. Stores the
30622 function decl in the ix86_builtins array. Returns the function decl or
30623 NULL_TREE, if the builtin was not added.
30625 If the front end has a special hook for builtin functions, delay adding
30626 builtin functions that aren't in the current ISA until the ISA is changed
30627 with function specific optimization. Doing so, can save about 300K for the
30628 default compiler. When the builtin is expanded, check at that time whether
30631 If the front end doesn't have a special hook, record all builtins, even if
30632 it isn't an instruction set in the current ISA in case the user uses
30633 function specific options for a different ISA, so that we don't get scope
30634 errors if a builtin is added in the middle of a function scope. */
30637 def_builtin (HOST_WIDE_INT mask, const char *name,
30638 enum ix86_builtin_func_type tcode,
30639 enum ix86_builtins code)
30641 tree decl = NULL_TREE;
30643 if (!(mask & OPTION_MASK_ISA_64BIT) || TARGET_64BIT)
30645 ix86_builtins_isa[(int) code].isa = mask;
30647 mask &= ~OPTION_MASK_ISA_64BIT;
30649 || (mask & ix86_isa_flags) != 0
30650 || (lang_hooks.builtin_function
30651 == lang_hooks.builtin_function_ext_scope))
30654 tree type = ix86_get_builtin_func_type (tcode);
30655 decl = add_builtin_function (name, type, code, BUILT_IN_MD,
30657 ix86_builtins[(int) code] = decl;
30658 ix86_builtins_isa[(int) code].set_and_not_built_p = false;
30662 /* Just a MASK where set_and_not_built_p == true can potentially
30663 include a builtin. */
30664 deferred_isa_values |= mask;
30665 ix86_builtins[(int) code] = NULL_TREE;
30666 ix86_builtins_isa[(int) code].tcode = tcode;
30667 ix86_builtins_isa[(int) code].name = name;
30668 ix86_builtins_isa[(int) code].leaf_p = false;
30669 ix86_builtins_isa[(int) code].nothrow_p = false;
30670 ix86_builtins_isa[(int) code].const_p = false;
30671 ix86_builtins_isa[(int) code].set_and_not_built_p = true;
30678 /* Like def_builtin, but also marks the function decl "const". */
30681 def_builtin_const (HOST_WIDE_INT mask, const char *name,
30682 enum ix86_builtin_func_type tcode, enum ix86_builtins code)
30684 tree decl = def_builtin (mask, name, tcode, code);
30686 TREE_READONLY (decl) = 1;
30688 ix86_builtins_isa[(int) code].const_p = true;
30693 /* Add any new builtin functions for a given ISA that may not have been
30694 declared. This saves a bit of space compared to adding all of the
30695 declarations to the tree, even if we didn't use them. */
30698 ix86_add_new_builtins (HOST_WIDE_INT isa)
30700 if ((isa & deferred_isa_values) == 0)
30703 /* Bits in ISA value can be removed from potential isa values. */
30704 deferred_isa_values &= ~isa;
30707 tree saved_current_target_pragma = current_target_pragma;
30708 current_target_pragma = NULL_TREE;
30710 for (i = 0; i < (int)IX86_BUILTIN_MAX; i++)
30712 if ((ix86_builtins_isa[i].isa & isa) != 0
30713 && ix86_builtins_isa[i].set_and_not_built_p)
30717 /* Don't define the builtin again. */
30718 ix86_builtins_isa[i].set_and_not_built_p = false;
30720 type = ix86_get_builtin_func_type (ix86_builtins_isa[i].tcode);
30721 decl = add_builtin_function_ext_scope (ix86_builtins_isa[i].name,
30722 type, i, BUILT_IN_MD, NULL,
30725 ix86_builtins[i] = decl;
30726 if (ix86_builtins_isa[i].const_p)
30727 TREE_READONLY (decl) = 1;
30728 if (ix86_builtins_isa[i].leaf_p)
30729 DECL_ATTRIBUTES (decl) = build_tree_list (get_identifier ("leaf"),
30731 if (ix86_builtins_isa[i].nothrow_p)
30732 TREE_NOTHROW (decl) = 1;
30736 current_target_pragma = saved_current_target_pragma;
30739 /* Bits for builtin_description.flag. */
30741 /* Set when we don't support the comparison natively, and should
30742 swap_comparison in order to support it. */
30743 #define BUILTIN_DESC_SWAP_OPERANDS 1
30745 struct builtin_description
30747 const HOST_WIDE_INT mask;
30748 const enum insn_code icode;
30749 const char *const name;
30750 const enum ix86_builtins code;
30751 const enum rtx_code comparison;
30755 static const struct builtin_description bdesc_comi[] =
30757 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comieq", IX86_BUILTIN_COMIEQSS, UNEQ, 0 },
30758 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comilt", IX86_BUILTIN_COMILTSS, UNLT, 0 },
30759 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comile", IX86_BUILTIN_COMILESS, UNLE, 0 },
30760 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comigt", IX86_BUILTIN_COMIGTSS, GT, 0 },
30761 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comige", IX86_BUILTIN_COMIGESS, GE, 0 },
30762 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comineq", IX86_BUILTIN_COMINEQSS, LTGT, 0 },
30763 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomieq", IX86_BUILTIN_UCOMIEQSS, UNEQ, 0 },
30764 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomilt", IX86_BUILTIN_UCOMILTSS, UNLT, 0 },
30765 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomile", IX86_BUILTIN_UCOMILESS, UNLE, 0 },
30766 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomigt", IX86_BUILTIN_UCOMIGTSS, GT, 0 },
30767 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomige", IX86_BUILTIN_UCOMIGESS, GE, 0 },
30768 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomineq", IX86_BUILTIN_UCOMINEQSS, LTGT, 0 },
30769 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdeq", IX86_BUILTIN_COMIEQSD, UNEQ, 0 },
30770 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdlt", IX86_BUILTIN_COMILTSD, UNLT, 0 },
30771 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdle", IX86_BUILTIN_COMILESD, UNLE, 0 },
30772 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdgt", IX86_BUILTIN_COMIGTSD, GT, 0 },
30773 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdge", IX86_BUILTIN_COMIGESD, GE, 0 },
30774 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdneq", IX86_BUILTIN_COMINEQSD, LTGT, 0 },
30775 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdeq", IX86_BUILTIN_UCOMIEQSD, UNEQ, 0 },
30776 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdlt", IX86_BUILTIN_UCOMILTSD, UNLT, 0 },
30777 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdle", IX86_BUILTIN_UCOMILESD, UNLE, 0 },
30778 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdgt", IX86_BUILTIN_UCOMIGTSD, GT, 0 },
30779 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdge", IX86_BUILTIN_UCOMIGESD, GE, 0 },
30780 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdneq", IX86_BUILTIN_UCOMINEQSD, LTGT, 0 },
30783 static const struct builtin_description bdesc_pcmpestr[] =
30786 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestri128", IX86_BUILTIN_PCMPESTRI128, UNKNOWN, 0 },
30787 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestrm128", IX86_BUILTIN_PCMPESTRM128, UNKNOWN, 0 },
30788 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestria128", IX86_BUILTIN_PCMPESTRA128, UNKNOWN, (int) CCAmode },
30789 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestric128", IX86_BUILTIN_PCMPESTRC128, UNKNOWN, (int) CCCmode },
30790 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestrio128", IX86_BUILTIN_PCMPESTRO128, UNKNOWN, (int) CCOmode },
30791 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestris128", IX86_BUILTIN_PCMPESTRS128, UNKNOWN, (int) CCSmode },
30792 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestriz128", IX86_BUILTIN_PCMPESTRZ128, UNKNOWN, (int) CCZmode },
30795 static const struct builtin_description bdesc_pcmpistr[] =
30798 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistri128", IX86_BUILTIN_PCMPISTRI128, UNKNOWN, 0 },
30799 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistrm128", IX86_BUILTIN_PCMPISTRM128, UNKNOWN, 0 },
30800 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistria128", IX86_BUILTIN_PCMPISTRA128, UNKNOWN, (int) CCAmode },
30801 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistric128", IX86_BUILTIN_PCMPISTRC128, UNKNOWN, (int) CCCmode },
30802 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistrio128", IX86_BUILTIN_PCMPISTRO128, UNKNOWN, (int) CCOmode },
30803 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistris128", IX86_BUILTIN_PCMPISTRS128, UNKNOWN, (int) CCSmode },
30804 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistriz128", IX86_BUILTIN_PCMPISTRZ128, UNKNOWN, (int) CCZmode },
30807 /* Special builtins with variable number of arguments. */
30808 static const struct builtin_description bdesc_special_args[] =
30810 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_rdtsc", IX86_BUILTIN_RDTSC, UNKNOWN, (int) UINT64_FTYPE_VOID },
30811 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_rdtscp", IX86_BUILTIN_RDTSCP, UNKNOWN, (int) UINT64_FTYPE_PUNSIGNED },
30812 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_pause, "__builtin_ia32_pause", IX86_BUILTIN_PAUSE, UNKNOWN, (int) VOID_FTYPE_VOID },
30814 /* 80387 (for use internally for atomic compound assignment). */
30815 { 0, CODE_FOR_fnstenv, "__builtin_ia32_fnstenv", IX86_BUILTIN_FNSTENV, UNKNOWN, (int) VOID_FTYPE_PVOID },
30816 { 0, CODE_FOR_fldenv, "__builtin_ia32_fldenv", IX86_BUILTIN_FLDENV, UNKNOWN, (int) VOID_FTYPE_PCVOID },
30817 { 0, CODE_FOR_fnstsw, "__builtin_ia32_fnstsw", IX86_BUILTIN_FNSTSW, UNKNOWN, (int) USHORT_FTYPE_VOID },
30818 { 0, CODE_FOR_fnclex, "__builtin_ia32_fnclex", IX86_BUILTIN_FNCLEX, UNKNOWN, (int) VOID_FTYPE_VOID },
30821 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_emms, "__builtin_ia32_emms", IX86_BUILTIN_EMMS, UNKNOWN, (int) VOID_FTYPE_VOID },
30824 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_femms, "__builtin_ia32_femms", IX86_BUILTIN_FEMMS, UNKNOWN, (int) VOID_FTYPE_VOID },
30826 /* FXSR, XSAVE, XSAVEOPT, XSAVEC and XSAVES. */
30827 { OPTION_MASK_ISA_FXSR, CODE_FOR_nothing, "__builtin_ia32_fxsave", IX86_BUILTIN_FXSAVE, UNKNOWN, (int) VOID_FTYPE_PVOID },
30828 { OPTION_MASK_ISA_FXSR, CODE_FOR_nothing, "__builtin_ia32_fxrstor", IX86_BUILTIN_FXRSTOR, UNKNOWN, (int) VOID_FTYPE_PVOID },
30829 { OPTION_MASK_ISA_XSAVE, CODE_FOR_nothing, "__builtin_ia32_xsave", IX86_BUILTIN_XSAVE, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
30830 { OPTION_MASK_ISA_XSAVE, CODE_FOR_nothing, "__builtin_ia32_xrstor", IX86_BUILTIN_XRSTOR, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
30831 { OPTION_MASK_ISA_XSAVEOPT, CODE_FOR_nothing, "__builtin_ia32_xsaveopt", IX86_BUILTIN_XSAVEOPT, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
30832 { OPTION_MASK_ISA_XSAVES, CODE_FOR_nothing, "__builtin_ia32_xsaves", IX86_BUILTIN_XSAVES, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
30833 { OPTION_MASK_ISA_XSAVES, CODE_FOR_nothing, "__builtin_ia32_xrstors", IX86_BUILTIN_XRSTORS, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
30834 { OPTION_MASK_ISA_XSAVEC, CODE_FOR_nothing, "__builtin_ia32_xsavec", IX86_BUILTIN_XSAVEC, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
30836 { OPTION_MASK_ISA_FXSR | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_fxsave64", IX86_BUILTIN_FXSAVE64, UNKNOWN, (int) VOID_FTYPE_PVOID },
30837 { OPTION_MASK_ISA_FXSR | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_fxrstor64", IX86_BUILTIN_FXRSTOR64, UNKNOWN, (int) VOID_FTYPE_PVOID },
30838 { OPTION_MASK_ISA_XSAVE | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_xsave64", IX86_BUILTIN_XSAVE64, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
30839 { OPTION_MASK_ISA_XSAVE | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_xrstor64", IX86_BUILTIN_XRSTOR64, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
30840 { OPTION_MASK_ISA_XSAVEOPT | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_xsaveopt64", IX86_BUILTIN_XSAVEOPT64, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
30841 { OPTION_MASK_ISA_XSAVES | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_xsaves64", IX86_BUILTIN_XSAVES64, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
30842 { OPTION_MASK_ISA_XSAVES | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_xrstors64", IX86_BUILTIN_XRSTORS64, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
30843 { OPTION_MASK_ISA_XSAVEC | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_xsavec64", IX86_BUILTIN_XSAVEC64, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
30846 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_storeups, "__builtin_ia32_storeups", IX86_BUILTIN_STOREUPS, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V4SF },
30847 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movntv4sf, "__builtin_ia32_movntps", IX86_BUILTIN_MOVNTPS, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V4SF },
30848 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_loadups, "__builtin_ia32_loadups", IX86_BUILTIN_LOADUPS, UNKNOWN, (int) V4SF_FTYPE_PCFLOAT },
30850 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_loadhps_exp, "__builtin_ia32_loadhps", IX86_BUILTIN_LOADHPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_PCV2SF },
30851 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_loadlps_exp, "__builtin_ia32_loadlps", IX86_BUILTIN_LOADLPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_PCV2SF },
30852 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_storehps, "__builtin_ia32_storehps", IX86_BUILTIN_STOREHPS, UNKNOWN, (int) VOID_FTYPE_PV2SF_V4SF },
30853 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_storelps, "__builtin_ia32_storelps", IX86_BUILTIN_STORELPS, UNKNOWN, (int) VOID_FTYPE_PV2SF_V4SF },
30855 /* SSE or 3DNow!A */
30856 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_sse_sfence, "__builtin_ia32_sfence", IX86_BUILTIN_SFENCE, UNKNOWN, (int) VOID_FTYPE_VOID },
30857 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_sse_movntq, "__builtin_ia32_movntq", IX86_BUILTIN_MOVNTQ, UNKNOWN, (int) VOID_FTYPE_PULONGLONG_ULONGLONG },
30860 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_lfence, "__builtin_ia32_lfence", IX86_BUILTIN_LFENCE, UNKNOWN, (int) VOID_FTYPE_VOID },
30861 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_mfence, 0, IX86_BUILTIN_MFENCE, UNKNOWN, (int) VOID_FTYPE_VOID },
30862 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_storeupd, "__builtin_ia32_storeupd", IX86_BUILTIN_STOREUPD, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V2DF },
30863 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_storedquv16qi, "__builtin_ia32_storedqu", IX86_BUILTIN_STOREDQU, UNKNOWN, (int) VOID_FTYPE_PCHAR_V16QI },
30864 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movntv2df, "__builtin_ia32_movntpd", IX86_BUILTIN_MOVNTPD, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V2DF },
30865 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movntv2di, "__builtin_ia32_movntdq", IX86_BUILTIN_MOVNTDQ, UNKNOWN, (int) VOID_FTYPE_PV2DI_V2DI },
30866 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movntisi, "__builtin_ia32_movnti", IX86_BUILTIN_MOVNTI, UNKNOWN, (int) VOID_FTYPE_PINT_INT },
30867 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_movntidi, "__builtin_ia32_movnti64", IX86_BUILTIN_MOVNTI64, UNKNOWN, (int) VOID_FTYPE_PLONGLONG_LONGLONG },
30868 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loadupd, "__builtin_ia32_loadupd", IX86_BUILTIN_LOADUPD, UNKNOWN, (int) V2DF_FTYPE_PCDOUBLE },
30869 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loaddquv16qi, "__builtin_ia32_loaddqu", IX86_BUILTIN_LOADDQU, UNKNOWN, (int) V16QI_FTYPE_PCCHAR },
30871 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loadhpd_exp, "__builtin_ia32_loadhpd", IX86_BUILTIN_LOADHPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_PCDOUBLE },
30872 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loadlpd_exp, "__builtin_ia32_loadlpd", IX86_BUILTIN_LOADLPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_PCDOUBLE },
30875 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_lddqu, "__builtin_ia32_lddqu", IX86_BUILTIN_LDDQU, UNKNOWN, (int) V16QI_FTYPE_PCCHAR },
30878 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_movntdqa, "__builtin_ia32_movntdqa", IX86_BUILTIN_MOVNTDQA, UNKNOWN, (int) V2DI_FTYPE_PV2DI },
30881 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_vmmovntv2df, "__builtin_ia32_movntsd", IX86_BUILTIN_MOVNTSD, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V2DF },
30882 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_vmmovntv4sf, "__builtin_ia32_movntss", IX86_BUILTIN_MOVNTSS, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V4SF },
30885 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vzeroall, "__builtin_ia32_vzeroall", IX86_BUILTIN_VZEROALL, UNKNOWN, (int) VOID_FTYPE_VOID },
30886 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vzeroupper, "__builtin_ia32_vzeroupper", IX86_BUILTIN_VZEROUPPER, UNKNOWN, (int) VOID_FTYPE_VOID },
30888 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_dupv4sf, "__builtin_ia32_vbroadcastss", IX86_BUILTIN_VBROADCASTSS, UNKNOWN, (int) V4SF_FTYPE_PCFLOAT },
30889 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_dupv4df, "__builtin_ia32_vbroadcastsd256", IX86_BUILTIN_VBROADCASTSD256, UNKNOWN, (int) V4DF_FTYPE_PCDOUBLE },
30890 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_dupv8sf, "__builtin_ia32_vbroadcastss256", IX86_BUILTIN_VBROADCASTSS256, UNKNOWN, (int) V8SF_FTYPE_PCFLOAT },
30891 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vbroadcastf128_v4df, "__builtin_ia32_vbroadcastf128_pd256", IX86_BUILTIN_VBROADCASTPD256, UNKNOWN, (int) V4DF_FTYPE_PCV2DF },
30892 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vbroadcastf128_v8sf, "__builtin_ia32_vbroadcastf128_ps256", IX86_BUILTIN_VBROADCASTPS256, UNKNOWN, (int) V8SF_FTYPE_PCV4SF },
30894 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_loadupd256, "__builtin_ia32_loadupd256", IX86_BUILTIN_LOADUPD256, UNKNOWN, (int) V4DF_FTYPE_PCDOUBLE },
30895 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_loadups256, "__builtin_ia32_loadups256", IX86_BUILTIN_LOADUPS256, UNKNOWN, (int) V8SF_FTYPE_PCFLOAT },
30896 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_storeupd256, "__builtin_ia32_storeupd256", IX86_BUILTIN_STOREUPD256, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V4DF },
30897 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_storeups256, "__builtin_ia32_storeups256", IX86_BUILTIN_STOREUPS256, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V8SF },
30898 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_loaddquv32qi, "__builtin_ia32_loaddqu256", IX86_BUILTIN_LOADDQU256, UNKNOWN, (int) V32QI_FTYPE_PCCHAR },
30899 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_storedquv32qi, "__builtin_ia32_storedqu256", IX86_BUILTIN_STOREDQU256, UNKNOWN, (int) VOID_FTYPE_PCHAR_V32QI },
30900 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_lddqu256, "__builtin_ia32_lddqu256", IX86_BUILTIN_LDDQU256, UNKNOWN, (int) V32QI_FTYPE_PCCHAR },
30902 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movntv4di, "__builtin_ia32_movntdq256", IX86_BUILTIN_MOVNTDQ256, UNKNOWN, (int) VOID_FTYPE_PV4DI_V4DI },
30903 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movntv4df, "__builtin_ia32_movntpd256", IX86_BUILTIN_MOVNTPD256, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V4DF },
30904 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movntv8sf, "__builtin_ia32_movntps256", IX86_BUILTIN_MOVNTPS256, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V8SF },
30906 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadpd, "__builtin_ia32_maskloadpd", IX86_BUILTIN_MASKLOADPD, UNKNOWN, (int) V2DF_FTYPE_PCV2DF_V2DI },
30907 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadps, "__builtin_ia32_maskloadps", IX86_BUILTIN_MASKLOADPS, UNKNOWN, (int) V4SF_FTYPE_PCV4SF_V4SI },
30908 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadpd256, "__builtin_ia32_maskloadpd256", IX86_BUILTIN_MASKLOADPD256, UNKNOWN, (int) V4DF_FTYPE_PCV4DF_V4DI },
30909 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadps256, "__builtin_ia32_maskloadps256", IX86_BUILTIN_MASKLOADPS256, UNKNOWN, (int) V8SF_FTYPE_PCV8SF_V8SI },
30910 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstorepd, "__builtin_ia32_maskstorepd", IX86_BUILTIN_MASKSTOREPD, UNKNOWN, (int) VOID_FTYPE_PV2DF_V2DI_V2DF },
30911 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstoreps, "__builtin_ia32_maskstoreps", IX86_BUILTIN_MASKSTOREPS, UNKNOWN, (int) VOID_FTYPE_PV4SF_V4SI_V4SF },
30912 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstorepd256, "__builtin_ia32_maskstorepd256", IX86_BUILTIN_MASKSTOREPD256, UNKNOWN, (int) VOID_FTYPE_PV4DF_V4DI_V4DF },
30913 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstoreps256, "__builtin_ia32_maskstoreps256", IX86_BUILTIN_MASKSTOREPS256, UNKNOWN, (int) VOID_FTYPE_PV8SF_V8SI_V8SF },
30916 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_movntdqa, "__builtin_ia32_movntdqa256", IX86_BUILTIN_MOVNTDQA256, UNKNOWN, (int) V4DI_FTYPE_PV4DI },
30917 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskloadd, "__builtin_ia32_maskloadd", IX86_BUILTIN_MASKLOADD, UNKNOWN, (int) V4SI_FTYPE_PCV4SI_V4SI },
30918 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskloadq, "__builtin_ia32_maskloadq", IX86_BUILTIN_MASKLOADQ, UNKNOWN, (int) V2DI_FTYPE_PCV2DI_V2DI },
30919 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskloadd256, "__builtin_ia32_maskloadd256", IX86_BUILTIN_MASKLOADD256, UNKNOWN, (int) V8SI_FTYPE_PCV8SI_V8SI },
30920 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskloadq256, "__builtin_ia32_maskloadq256", IX86_BUILTIN_MASKLOADQ256, UNKNOWN, (int) V4DI_FTYPE_PCV4DI_V4DI },
30921 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskstored, "__builtin_ia32_maskstored", IX86_BUILTIN_MASKSTORED, UNKNOWN, (int) VOID_FTYPE_PV4SI_V4SI_V4SI },
30922 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskstoreq, "__builtin_ia32_maskstoreq", IX86_BUILTIN_MASKSTOREQ, UNKNOWN, (int) VOID_FTYPE_PV2DI_V2DI_V2DI },
30923 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskstored256, "__builtin_ia32_maskstored256", IX86_BUILTIN_MASKSTORED256, UNKNOWN, (int) VOID_FTYPE_PV8SI_V8SI_V8SI },
30924 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskstoreq256, "__builtin_ia32_maskstoreq256", IX86_BUILTIN_MASKSTOREQ256, UNKNOWN, (int) VOID_FTYPE_PV4DI_V4DI_V4DI },
30927 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_compressstorev16sf_mask, "__builtin_ia32_compressstoresf512_mask", IX86_BUILTIN_COMPRESSPSSTORE512, UNKNOWN, (int) VOID_FTYPE_PV16SF_V16SF_HI },
30928 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_compressstorev16si_mask, "__builtin_ia32_compressstoresi512_mask", IX86_BUILTIN_PCOMPRESSDSTORE512, UNKNOWN, (int) VOID_FTYPE_PV16SI_V16SI_HI },
30929 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_compressstorev8df_mask, "__builtin_ia32_compressstoredf512_mask", IX86_BUILTIN_COMPRESSPDSTORE512, UNKNOWN, (int) VOID_FTYPE_PV8DF_V8DF_QI },
30930 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_compressstorev8di_mask, "__builtin_ia32_compressstoredi512_mask", IX86_BUILTIN_PCOMPRESSQSTORE512, UNKNOWN, (int) VOID_FTYPE_PV8DI_V8DI_QI },
30931 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_expandv16sf_mask, "__builtin_ia32_expandloadsf512_mask", IX86_BUILTIN_EXPANDPSLOAD512, UNKNOWN, (int) V16SF_FTYPE_PCV16SF_V16SF_HI },
30932 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_expandv16sf_maskz, "__builtin_ia32_expandloadsf512_maskz", IX86_BUILTIN_EXPANDPSLOAD512Z, UNKNOWN, (int) V16SF_FTYPE_PCV16SF_V16SF_HI },
30933 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_expandv16si_mask, "__builtin_ia32_expandloadsi512_mask", IX86_BUILTIN_PEXPANDDLOAD512, UNKNOWN, (int) V16SI_FTYPE_PCV16SI_V16SI_HI },
30934 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_expandv16si_maskz, "__builtin_ia32_expandloadsi512_maskz", IX86_BUILTIN_PEXPANDDLOAD512Z, UNKNOWN, (int) V16SI_FTYPE_PCV16SI_V16SI_HI },
30935 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_expandv8df_mask, "__builtin_ia32_expandloaddf512_mask", IX86_BUILTIN_EXPANDPDLOAD512, UNKNOWN, (int) V8DF_FTYPE_PCV8DF_V8DF_QI },
30936 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_expandv8df_maskz, "__builtin_ia32_expandloaddf512_maskz", IX86_BUILTIN_EXPANDPDLOAD512Z, UNKNOWN, (int) V8DF_FTYPE_PCV8DF_V8DF_QI },
30937 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_expandv8di_mask, "__builtin_ia32_expandloaddi512_mask", IX86_BUILTIN_PEXPANDQLOAD512, UNKNOWN, (int) V8DI_FTYPE_PCV8DI_V8DI_QI },
30938 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_expandv8di_maskz, "__builtin_ia32_expandloaddi512_maskz", IX86_BUILTIN_PEXPANDQLOAD512Z, UNKNOWN, (int) V8DI_FTYPE_PCV8DI_V8DI_QI },
30939 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_loaddquv16si_mask, "__builtin_ia32_loaddqusi512_mask", IX86_BUILTIN_LOADDQUSI512, UNKNOWN, (int) V16SI_FTYPE_PCV16SI_V16SI_HI },
30940 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_loaddquv8di_mask, "__builtin_ia32_loaddqudi512_mask", IX86_BUILTIN_LOADDQUDI512, UNKNOWN, (int) V8DI_FTYPE_PCV8DI_V8DI_QI },
30941 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_loadupd512_mask, "__builtin_ia32_loadupd512_mask", IX86_BUILTIN_LOADUPD512, UNKNOWN, (int) V8DF_FTYPE_PCV8DF_V8DF_QI },
30942 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_loadups512_mask, "__builtin_ia32_loadups512_mask", IX86_BUILTIN_LOADUPS512, UNKNOWN, (int) V16SF_FTYPE_PCV16SF_V16SF_HI },
30943 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_loadv16sf_mask, "__builtin_ia32_loadaps512_mask", IX86_BUILTIN_LOADAPS512, UNKNOWN, (int) V16SF_FTYPE_PCV16SF_V16SF_HI },
30944 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_loadv16si_mask, "__builtin_ia32_movdqa32load512_mask", IX86_BUILTIN_MOVDQA32LOAD512, UNKNOWN, (int) V16SI_FTYPE_PCV16SI_V16SI_HI },
30945 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_loadv8df_mask, "__builtin_ia32_loadapd512_mask", IX86_BUILTIN_LOADAPD512, UNKNOWN, (int) V8DF_FTYPE_PCV8DF_V8DF_QI },
30946 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_loadv8di_mask, "__builtin_ia32_movdqa64load512_mask", IX86_BUILTIN_MOVDQA64LOAD512, UNKNOWN, (int) V8DI_FTYPE_PCV8DI_V8DI_QI },
30947 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_movntv16sf, "__builtin_ia32_movntps512", IX86_BUILTIN_MOVNTPS512, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V16SF },
30948 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_movntv8df, "__builtin_ia32_movntpd512", IX86_BUILTIN_MOVNTPD512, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V8DF },
30949 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_movntv8di, "__builtin_ia32_movntdq512", IX86_BUILTIN_MOVNTDQ512, UNKNOWN, (int) VOID_FTYPE_PV8DI_V8DI },
30950 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_movntdqa, "__builtin_ia32_movntdqa512", IX86_BUILTIN_MOVNTDQA512, UNKNOWN, (int) V8DI_FTYPE_PV8DI },
30951 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_storedquv16si_mask, "__builtin_ia32_storedqusi512_mask", IX86_BUILTIN_STOREDQUSI512, UNKNOWN, (int) VOID_FTYPE_PV16SI_V16SI_HI },
30952 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_storedquv8di_mask, "__builtin_ia32_storedqudi512_mask", IX86_BUILTIN_STOREDQUDI512, UNKNOWN, (int) VOID_FTYPE_PV8DI_V8DI_QI },
30953 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_storeupd512_mask, "__builtin_ia32_storeupd512_mask", IX86_BUILTIN_STOREUPD512, UNKNOWN, (int) VOID_FTYPE_PV8DF_V8DF_QI },
30954 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_us_truncatev8div8si2_mask_store, "__builtin_ia32_pmovusqd512mem_mask", IX86_BUILTIN_PMOVUSQD512_MEM, UNKNOWN, (int) VOID_FTYPE_PV8SI_V8DI_QI },
30955 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ss_truncatev8div8si2_mask_store, "__builtin_ia32_pmovsqd512mem_mask", IX86_BUILTIN_PMOVSQD512_MEM, UNKNOWN, (int) VOID_FTYPE_PV8SI_V8DI_QI },
30956 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_truncatev8div8si2_mask_store, "__builtin_ia32_pmovqd512mem_mask", IX86_BUILTIN_PMOVQD512_MEM, UNKNOWN, (int) VOID_FTYPE_PV8SI_V8DI_QI },
30957 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_us_truncatev8div8hi2_mask_store, "__builtin_ia32_pmovusqw512mem_mask", IX86_BUILTIN_PMOVUSQW512_MEM, UNKNOWN, (int) VOID_FTYPE_PV8HI_V8DI_QI },
30958 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ss_truncatev8div8hi2_mask_store, "__builtin_ia32_pmovsqw512mem_mask", IX86_BUILTIN_PMOVSQW512_MEM, UNKNOWN, (int) VOID_FTYPE_PV8HI_V8DI_QI },
30959 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_truncatev8div8hi2_mask_store, "__builtin_ia32_pmovqw512mem_mask", IX86_BUILTIN_PMOVQW512_MEM, UNKNOWN, (int) VOID_FTYPE_PV8HI_V8DI_QI },
30960 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_us_truncatev16siv16hi2_mask_store, "__builtin_ia32_pmovusdw512mem_mask", IX86_BUILTIN_PMOVUSDW512_MEM, UNKNOWN, (int) VOID_FTYPE_PV16HI_V16SI_HI },
30961 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ss_truncatev16siv16hi2_mask_store, "__builtin_ia32_pmovsdw512mem_mask", IX86_BUILTIN_PMOVSDW512_MEM, UNKNOWN, (int) VOID_FTYPE_PV16HI_V16SI_HI },
30962 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_truncatev16siv16hi2_mask_store, "__builtin_ia32_pmovdw512mem_mask", IX86_BUILTIN_PMOVDW512_MEM, UNKNOWN, (int) VOID_FTYPE_PV16HI_V16SI_HI },
30963 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_truncatev8div16qi2_mask_store, "__builtin_ia32_pmovqb512mem_mask", IX86_BUILTIN_PMOVQB512_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V8DI_QI },
30964 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_us_truncatev8div16qi2_mask_store, "__builtin_ia32_pmovusqb512mem_mask", IX86_BUILTIN_PMOVUSQB512_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V8DI_QI },
30965 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ss_truncatev8div16qi2_mask_store, "__builtin_ia32_pmovsqb512mem_mask", IX86_BUILTIN_PMOVSQB512_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V8DI_QI },
30966 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_us_truncatev16siv16qi2_mask_store, "__builtin_ia32_pmovusdb512mem_mask", IX86_BUILTIN_PMOVUSDB512_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V16SI_HI },
30967 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ss_truncatev16siv16qi2_mask_store, "__builtin_ia32_pmovsdb512mem_mask", IX86_BUILTIN_PMOVSDB512_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V16SI_HI },
30968 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_truncatev16siv16qi2_mask_store, "__builtin_ia32_pmovdb512mem_mask", IX86_BUILTIN_PMOVDB512_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V16SI_HI },
30969 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_storeups512_mask, "__builtin_ia32_storeups512_mask", IX86_BUILTIN_STOREUPS512, UNKNOWN, (int) VOID_FTYPE_PV16SF_V16SF_HI },
30970 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_storev16sf_mask, "__builtin_ia32_storeaps512_mask", IX86_BUILTIN_STOREAPS512, UNKNOWN, (int) VOID_FTYPE_PV16SF_V16SF_HI },
30971 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_storev16si_mask, "__builtin_ia32_movdqa32store512_mask", IX86_BUILTIN_MOVDQA32STORE512, UNKNOWN, (int) VOID_FTYPE_PV16SI_V16SI_HI },
30972 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_storev8df_mask, "__builtin_ia32_storeapd512_mask", IX86_BUILTIN_STOREAPD512, UNKNOWN, (int) VOID_FTYPE_PV8DF_V8DF_QI },
30973 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_storev8di_mask, "__builtin_ia32_movdqa64store512_mask", IX86_BUILTIN_MOVDQA64STORE512, UNKNOWN, (int) VOID_FTYPE_PV8DI_V8DI_QI },
30975 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_llwpcb, "__builtin_ia32_llwpcb", IX86_BUILTIN_LLWPCB, UNKNOWN, (int) VOID_FTYPE_PVOID },
30976 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_slwpcb, "__builtin_ia32_slwpcb", IX86_BUILTIN_SLWPCB, UNKNOWN, (int) PVOID_FTYPE_VOID },
30977 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpvalsi3, "__builtin_ia32_lwpval32", IX86_BUILTIN_LWPVAL32, UNKNOWN, (int) VOID_FTYPE_UINT_UINT_UINT },
30978 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpvaldi3, "__builtin_ia32_lwpval64", IX86_BUILTIN_LWPVAL64, UNKNOWN, (int) VOID_FTYPE_UINT64_UINT_UINT },
30979 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpinssi3, "__builtin_ia32_lwpins32", IX86_BUILTIN_LWPINS32, UNKNOWN, (int) UCHAR_FTYPE_UINT_UINT_UINT },
30980 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpinsdi3, "__builtin_ia32_lwpins64", IX86_BUILTIN_LWPINS64, UNKNOWN, (int) UCHAR_FTYPE_UINT64_UINT_UINT },
30983 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdfsbasesi, "__builtin_ia32_rdfsbase32", IX86_BUILTIN_RDFSBASE32, UNKNOWN, (int) UNSIGNED_FTYPE_VOID },
30984 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdfsbasedi, "__builtin_ia32_rdfsbase64", IX86_BUILTIN_RDFSBASE64, UNKNOWN, (int) UINT64_FTYPE_VOID },
30985 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdgsbasesi, "__builtin_ia32_rdgsbase32", IX86_BUILTIN_RDGSBASE32, UNKNOWN, (int) UNSIGNED_FTYPE_VOID },
30986 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdgsbasedi, "__builtin_ia32_rdgsbase64", IX86_BUILTIN_RDGSBASE64, UNKNOWN, (int) UINT64_FTYPE_VOID },
30987 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrfsbasesi, "__builtin_ia32_wrfsbase32", IX86_BUILTIN_WRFSBASE32, UNKNOWN, (int) VOID_FTYPE_UNSIGNED },
30988 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrfsbasedi, "__builtin_ia32_wrfsbase64", IX86_BUILTIN_WRFSBASE64, UNKNOWN, (int) VOID_FTYPE_UINT64 },
30989 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrgsbasesi, "__builtin_ia32_wrgsbase32", IX86_BUILTIN_WRGSBASE32, UNKNOWN, (int) VOID_FTYPE_UNSIGNED },
30990 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrgsbasedi, "__builtin_ia32_wrgsbase64", IX86_BUILTIN_WRGSBASE64, UNKNOWN, (int) VOID_FTYPE_UINT64 },
30993 { OPTION_MASK_ISA_RTM, CODE_FOR_xbegin, "__builtin_ia32_xbegin", IX86_BUILTIN_XBEGIN, UNKNOWN, (int) UNSIGNED_FTYPE_VOID },
30994 { OPTION_MASK_ISA_RTM, CODE_FOR_xend, "__builtin_ia32_xend", IX86_BUILTIN_XEND, UNKNOWN, (int) VOID_FTYPE_VOID },
30995 { OPTION_MASK_ISA_RTM, CODE_FOR_xtest, "__builtin_ia32_xtest", IX86_BUILTIN_XTEST, UNKNOWN, (int) INT_FTYPE_VOID },
30998 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_loaddquv32hi_mask, "__builtin_ia32_loaddquhi512_mask", IX86_BUILTIN_LOADDQUHI512_MASK, UNKNOWN, (int) V32HI_FTYPE_PCV32HI_V32HI_SI },
30999 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512f_loaddquv64qi_mask, "__builtin_ia32_loaddquqi512_mask", IX86_BUILTIN_LOADDQUQI512_MASK, UNKNOWN, (int) V64QI_FTYPE_PCV64QI_V64QI_DI },
31000 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_storedquv32hi_mask, "__builtin_ia32_storedquhi512_mask", IX86_BUILTIN_STOREDQUHI512_MASK, UNKNOWN, (int) VOID_FTYPE_PV32HI_V32HI_SI },
31001 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_storedquv64qi_mask, "__builtin_ia32_storedquqi512_mask", IX86_BUILTIN_STOREDQUQI512_MASK, UNKNOWN, (int) VOID_FTYPE_PV64QI_V64QI_DI },
31004 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loaddquv16hi_mask, "__builtin_ia32_loaddquhi256_mask", IX86_BUILTIN_LOADDQUHI256_MASK, UNKNOWN, (int) V16HI_FTYPE_PCV16HI_V16HI_HI },
31005 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loaddquv8hi_mask, "__builtin_ia32_loaddquhi128_mask", IX86_BUILTIN_LOADDQUHI128_MASK, UNKNOWN, (int) V8HI_FTYPE_PCV8HI_V8HI_QI },
31006 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_loaddquv32qi_mask, "__builtin_ia32_loaddquqi256_mask", IX86_BUILTIN_LOADDQUQI256_MASK, UNKNOWN, (int) V32QI_FTYPE_PCV32QI_V32QI_SI },
31007 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_loaddquv16qi_mask, "__builtin_ia32_loaddquqi128_mask", IX86_BUILTIN_LOADDQUQI128_MASK, UNKNOWN, (int) V16QI_FTYPE_PCV16QI_V16QI_HI },
31008 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loadv4di_mask, "__builtin_ia32_movdqa64load256_mask", IX86_BUILTIN_MOVDQA64LOAD256_MASK, UNKNOWN, (int) V4DI_FTYPE_PCV4DI_V4DI_QI },
31009 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loadv2di_mask, "__builtin_ia32_movdqa64load128_mask", IX86_BUILTIN_MOVDQA64LOAD128_MASK, UNKNOWN, (int) V2DI_FTYPE_PCV2DI_V2DI_QI },
31010 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loadv8si_mask, "__builtin_ia32_movdqa32load256_mask", IX86_BUILTIN_MOVDQA32LOAD256_MASK, UNKNOWN, (int) V8SI_FTYPE_PCV8SI_V8SI_QI },
31011 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loadv4si_mask, "__builtin_ia32_movdqa32load128_mask", IX86_BUILTIN_MOVDQA32LOAD128_MASK, UNKNOWN, (int) V4SI_FTYPE_PCV4SI_V4SI_QI },
31012 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storev4di_mask, "__builtin_ia32_movdqa64store256_mask", IX86_BUILTIN_MOVDQA64STORE256_MASK, UNKNOWN, (int) VOID_FTYPE_PV4DI_V4DI_QI },
31013 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storev2di_mask, "__builtin_ia32_movdqa64store128_mask", IX86_BUILTIN_MOVDQA64STORE128_MASK, UNKNOWN, (int) VOID_FTYPE_PV2DI_V2DI_QI },
31014 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storev8si_mask, "__builtin_ia32_movdqa32store256_mask", IX86_BUILTIN_MOVDQA32STORE256_MASK, UNKNOWN, (int) VOID_FTYPE_PV8SI_V8SI_QI },
31015 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storev4si_mask, "__builtin_ia32_movdqa32store128_mask", IX86_BUILTIN_MOVDQA32STORE128_MASK, UNKNOWN, (int) VOID_FTYPE_PV4SI_V4SI_QI },
31016 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loadv4df_mask, "__builtin_ia32_loadapd256_mask", IX86_BUILTIN_LOADAPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_PCV4DF_V4DF_QI },
31017 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loadv2df_mask, "__builtin_ia32_loadapd128_mask", IX86_BUILTIN_LOADAPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_PCV2DF_V2DF_QI },
31018 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loadv8sf_mask, "__builtin_ia32_loadaps256_mask", IX86_BUILTIN_LOADAPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_PCV8SF_V8SF_QI },
31019 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loadv4sf_mask, "__builtin_ia32_loadaps128_mask", IX86_BUILTIN_LOADAPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_PCV4SF_V4SF_QI },
31020 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storev4df_mask, "__builtin_ia32_storeapd256_mask", IX86_BUILTIN_STOREAPD256_MASK, UNKNOWN, (int) VOID_FTYPE_PV4DF_V4DF_QI },
31021 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storev2df_mask, "__builtin_ia32_storeapd128_mask", IX86_BUILTIN_STOREAPD128_MASK, UNKNOWN, (int) VOID_FTYPE_PV2DF_V2DF_QI },
31022 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storev8sf_mask, "__builtin_ia32_storeaps256_mask", IX86_BUILTIN_STOREAPS256_MASK, UNKNOWN, (int) VOID_FTYPE_PV8SF_V8SF_QI },
31023 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storev4sf_mask, "__builtin_ia32_storeaps128_mask", IX86_BUILTIN_STOREAPS128_MASK, UNKNOWN, (int) VOID_FTYPE_PV4SF_V4SF_QI },
31024 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_loadupd256_mask, "__builtin_ia32_loadupd256_mask", IX86_BUILTIN_LOADUPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_PCV4DF_V4DF_QI },
31025 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_loadupd_mask, "__builtin_ia32_loadupd128_mask", IX86_BUILTIN_LOADUPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_PCV2DF_V2DF_QI },
31026 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_loadups256_mask, "__builtin_ia32_loadups256_mask", IX86_BUILTIN_LOADUPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_PCV8SF_V8SF_QI },
31027 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse_loadups_mask, "__builtin_ia32_loadups128_mask", IX86_BUILTIN_LOADUPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_PCV4SF_V4SF_QI },
31028 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storeupd256_mask, "__builtin_ia32_storeupd256_mask", IX86_BUILTIN_STOREUPD256_MASK, UNKNOWN, (int) VOID_FTYPE_PV4DF_V4DF_QI },
31029 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storeupd_mask, "__builtin_ia32_storeupd128_mask", IX86_BUILTIN_STOREUPD128_MASK, UNKNOWN, (int) VOID_FTYPE_PV2DF_V2DF_QI },
31030 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storeups256_mask, "__builtin_ia32_storeups256_mask", IX86_BUILTIN_STOREUPS256_MASK, UNKNOWN, (int) VOID_FTYPE_PV8SF_V8SF_QI },
31031 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storeups_mask, "__builtin_ia32_storeups128_mask", IX86_BUILTIN_STOREUPS128_MASK, UNKNOWN, (int) VOID_FTYPE_PV4SF_V4SF_QI },
31032 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loaddquv4di_mask, "__builtin_ia32_loaddqudi256_mask", IX86_BUILTIN_LOADDQUDI256_MASK, UNKNOWN, (int) V4DI_FTYPE_PCV4DI_V4DI_QI },
31033 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loaddquv2di_mask, "__builtin_ia32_loaddqudi128_mask", IX86_BUILTIN_LOADDQUDI128_MASK, UNKNOWN, (int) V2DI_FTYPE_PCV2DI_V2DI_QI },
31034 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_loaddquv8si_mask, "__builtin_ia32_loaddqusi256_mask", IX86_BUILTIN_LOADDQUSI256_MASK, UNKNOWN, (int) V8SI_FTYPE_PCV8SI_V8SI_QI },
31035 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_loaddquv4si_mask, "__builtin_ia32_loaddqusi128_mask", IX86_BUILTIN_LOADDQUSI128_MASK, UNKNOWN, (int) V4SI_FTYPE_PCV4SI_V4SI_QI },
31036 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storedquv4di_mask, "__builtin_ia32_storedqudi256_mask", IX86_BUILTIN_STOREDQUDI256_MASK, UNKNOWN, (int) VOID_FTYPE_PV4DI_V4DI_QI },
31037 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storedquv2di_mask, "__builtin_ia32_storedqudi128_mask", IX86_BUILTIN_STOREDQUDI128_MASK, UNKNOWN, (int) VOID_FTYPE_PV2DI_V2DI_QI },
31038 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storedquv8si_mask, "__builtin_ia32_storedqusi256_mask", IX86_BUILTIN_STOREDQUSI256_MASK, UNKNOWN, (int) VOID_FTYPE_PV8SI_V8SI_QI },
31039 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storedquv4si_mask, "__builtin_ia32_storedqusi128_mask", IX86_BUILTIN_STOREDQUSI128_MASK, UNKNOWN, (int) VOID_FTYPE_PV4SI_V4SI_QI },
31040 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storedquv16hi_mask, "__builtin_ia32_storedquhi256_mask", IX86_BUILTIN_STOREDQUHI256_MASK, UNKNOWN, (int) VOID_FTYPE_PV16HI_V16HI_HI },
31041 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storedquv8hi_mask, "__builtin_ia32_storedquhi128_mask", IX86_BUILTIN_STOREDQUHI128_MASK, UNKNOWN, (int) VOID_FTYPE_PV8HI_V8HI_QI },
31042 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storedquv32qi_mask, "__builtin_ia32_storedquqi256_mask", IX86_BUILTIN_STOREDQUQI256_MASK, UNKNOWN, (int) VOID_FTYPE_PV32QI_V32QI_SI },
31043 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_storedquv16qi_mask, "__builtin_ia32_storedquqi128_mask", IX86_BUILTIN_STOREDQUQI128_MASK, UNKNOWN, (int) VOID_FTYPE_PV16QI_V16QI_HI },
31044 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_compressstorev4df_mask, "__builtin_ia32_compressstoredf256_mask", IX86_BUILTIN_COMPRESSPDSTORE256, UNKNOWN, (int) VOID_FTYPE_PV4DF_V4DF_QI },
31045 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_compressstorev2df_mask, "__builtin_ia32_compressstoredf128_mask", IX86_BUILTIN_COMPRESSPDSTORE128, UNKNOWN, (int) VOID_FTYPE_PV2DF_V2DF_QI },
31046 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_compressstorev8sf_mask, "__builtin_ia32_compressstoresf256_mask", IX86_BUILTIN_COMPRESSPSSTORE256, UNKNOWN, (int) VOID_FTYPE_PV8SF_V8SF_QI },
31047 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_compressstorev4sf_mask, "__builtin_ia32_compressstoresf128_mask", IX86_BUILTIN_COMPRESSPSSTORE128, UNKNOWN, (int) VOID_FTYPE_PV4SF_V4SF_QI },
31048 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_compressstorev4di_mask, "__builtin_ia32_compressstoredi256_mask", IX86_BUILTIN_PCOMPRESSQSTORE256, UNKNOWN, (int) VOID_FTYPE_PV4DI_V4DI_QI },
31049 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_compressstorev2di_mask, "__builtin_ia32_compressstoredi128_mask", IX86_BUILTIN_PCOMPRESSQSTORE128, UNKNOWN, (int) VOID_FTYPE_PV2DI_V2DI_QI },
31050 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_compressstorev8si_mask, "__builtin_ia32_compressstoresi256_mask", IX86_BUILTIN_PCOMPRESSDSTORE256, UNKNOWN, (int) VOID_FTYPE_PV8SI_V8SI_QI },
31051 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_compressstorev4si_mask, "__builtin_ia32_compressstoresi128_mask", IX86_BUILTIN_PCOMPRESSDSTORE128, UNKNOWN, (int) VOID_FTYPE_PV4SI_V4SI_QI },
31052 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv4df_mask, "__builtin_ia32_expandloaddf256_mask", IX86_BUILTIN_EXPANDPDLOAD256, UNKNOWN, (int) V4DF_FTYPE_PCV4DF_V4DF_QI },
31053 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv2df_mask, "__builtin_ia32_expandloaddf128_mask", IX86_BUILTIN_EXPANDPDLOAD128, UNKNOWN, (int) V2DF_FTYPE_PCV2DF_V2DF_QI },
31054 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv8sf_mask, "__builtin_ia32_expandloadsf256_mask", IX86_BUILTIN_EXPANDPSLOAD256, UNKNOWN, (int) V8SF_FTYPE_PCV8SF_V8SF_QI },
31055 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv4sf_mask, "__builtin_ia32_expandloadsf128_mask", IX86_BUILTIN_EXPANDPSLOAD128, UNKNOWN, (int) V4SF_FTYPE_PCV4SF_V4SF_QI },
31056 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv4di_mask, "__builtin_ia32_expandloaddi256_mask", IX86_BUILTIN_PEXPANDQLOAD256, UNKNOWN, (int) V4DI_FTYPE_PCV4DI_V4DI_QI },
31057 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv2di_mask, "__builtin_ia32_expandloaddi128_mask", IX86_BUILTIN_PEXPANDQLOAD128, UNKNOWN, (int) V2DI_FTYPE_PCV2DI_V2DI_QI },
31058 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv8si_mask, "__builtin_ia32_expandloadsi256_mask", IX86_BUILTIN_PEXPANDDLOAD256, UNKNOWN, (int) V8SI_FTYPE_PCV8SI_V8SI_QI },
31059 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv4si_mask, "__builtin_ia32_expandloadsi128_mask", IX86_BUILTIN_PEXPANDDLOAD128, UNKNOWN, (int) V4SI_FTYPE_PCV4SI_V4SI_QI },
31060 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv4df_maskz, "__builtin_ia32_expandloaddf256_maskz", IX86_BUILTIN_EXPANDPDLOAD256Z, UNKNOWN, (int) V4DF_FTYPE_PCV4DF_V4DF_QI },
31061 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv2df_maskz, "__builtin_ia32_expandloaddf128_maskz", IX86_BUILTIN_EXPANDPDLOAD128Z, UNKNOWN, (int) V2DF_FTYPE_PCV2DF_V2DF_QI },
31062 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv8sf_maskz, "__builtin_ia32_expandloadsf256_maskz", IX86_BUILTIN_EXPANDPSLOAD256Z, UNKNOWN, (int) V8SF_FTYPE_PCV8SF_V8SF_QI },
31063 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv4sf_maskz, "__builtin_ia32_expandloadsf128_maskz", IX86_BUILTIN_EXPANDPSLOAD128Z, UNKNOWN, (int) V4SF_FTYPE_PCV4SF_V4SF_QI },
31064 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv4di_maskz, "__builtin_ia32_expandloaddi256_maskz", IX86_BUILTIN_PEXPANDQLOAD256Z, UNKNOWN, (int) V4DI_FTYPE_PCV4DI_V4DI_QI },
31065 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv2di_maskz, "__builtin_ia32_expandloaddi128_maskz", IX86_BUILTIN_PEXPANDQLOAD128Z, UNKNOWN, (int) V2DI_FTYPE_PCV2DI_V2DI_QI },
31066 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv8si_maskz, "__builtin_ia32_expandloadsi256_maskz", IX86_BUILTIN_PEXPANDDLOAD256Z, UNKNOWN, (int) V8SI_FTYPE_PCV8SI_V8SI_QI },
31067 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv4si_maskz, "__builtin_ia32_expandloadsi128_maskz", IX86_BUILTIN_PEXPANDDLOAD128Z, UNKNOWN, (int) V4SI_FTYPE_PCV4SI_V4SI_QI },
31068 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev4div4si2_mask_store, "__builtin_ia32_pmovqd256mem_mask", IX86_BUILTIN_PMOVQD256_MEM, UNKNOWN, (int) VOID_FTYPE_PV4SI_V4DI_QI },
31069 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev2div2si2_mask_store, "__builtin_ia32_pmovqd128mem_mask", IX86_BUILTIN_PMOVQD128_MEM, UNKNOWN, (int) VOID_FTYPE_PV4SI_V2DI_QI },
31070 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev4div4si2_mask_store, "__builtin_ia32_pmovsqd256mem_mask", IX86_BUILTIN_PMOVSQD256_MEM, UNKNOWN, (int) VOID_FTYPE_PV4SI_V4DI_QI },
31071 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev2div2si2_mask_store, "__builtin_ia32_pmovsqd128mem_mask", IX86_BUILTIN_PMOVSQD128_MEM, UNKNOWN, (int) VOID_FTYPE_PV4SI_V2DI_QI },
31072 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev4div4si2_mask_store, "__builtin_ia32_pmovusqd256mem_mask", IX86_BUILTIN_PMOVUSQD256_MEM, UNKNOWN, (int) VOID_FTYPE_PV4SI_V4DI_QI },
31073 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev2div2si2_mask_store, "__builtin_ia32_pmovusqd128mem_mask", IX86_BUILTIN_PMOVUSQD128_MEM, UNKNOWN, (int) VOID_FTYPE_PV4SI_V2DI_QI },
31074 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev4div4hi2_mask_store, "__builtin_ia32_pmovqw256mem_mask", IX86_BUILTIN_PMOVQW256_MEM, UNKNOWN, (int) VOID_FTYPE_PV8HI_V4DI_QI },
31075 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev2div2hi2_mask_store, "__builtin_ia32_pmovqw128mem_mask", IX86_BUILTIN_PMOVQW128_MEM, UNKNOWN, (int) VOID_FTYPE_PV8HI_V2DI_QI },
31076 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev4div4hi2_mask_store, "__builtin_ia32_pmovsqw256mem_mask", IX86_BUILTIN_PMOVSQW256_MEM, UNKNOWN, (int) VOID_FTYPE_PV8HI_V4DI_QI },
31077 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev2div2hi2_mask_store, "__builtin_ia32_pmovsqw128mem_mask", IX86_BUILTIN_PMOVSQW128_MEM, UNKNOWN, (int) VOID_FTYPE_PV8HI_V2DI_QI },
31078 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev4div4hi2_mask_store, "__builtin_ia32_pmovusqw256mem_mask", IX86_BUILTIN_PMOVUSQW256_MEM, UNKNOWN, (int) VOID_FTYPE_PV8HI_V4DI_QI },
31079 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev2div2hi2_mask_store, "__builtin_ia32_pmovusqw128mem_mask", IX86_BUILTIN_PMOVUSQW128_MEM, UNKNOWN, (int) VOID_FTYPE_PV8HI_V2DI_QI },
31080 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev4div4qi2_mask_store, "__builtin_ia32_pmovqb256mem_mask", IX86_BUILTIN_PMOVQB256_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V4DI_QI },
31081 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev2div2qi2_mask_store, "__builtin_ia32_pmovqb128mem_mask", IX86_BUILTIN_PMOVQB128_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V2DI_QI },
31082 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev4div4qi2_mask_store, "__builtin_ia32_pmovsqb256mem_mask", IX86_BUILTIN_PMOVSQB256_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V4DI_QI },
31083 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev2div2qi2_mask_store, "__builtin_ia32_pmovsqb128mem_mask", IX86_BUILTIN_PMOVSQB128_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V2DI_QI },
31084 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev4div4qi2_mask_store, "__builtin_ia32_pmovusqb256mem_mask", IX86_BUILTIN_PMOVUSQB256_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V4DI_QI },
31085 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev2div2qi2_mask_store, "__builtin_ia32_pmovusqb128mem_mask", IX86_BUILTIN_PMOVUSQB128_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V2DI_QI },
31086 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev8siv8qi2_mask_store, "__builtin_ia32_pmovdb256mem_mask", IX86_BUILTIN_PMOVDB256_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V8SI_QI },
31087 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev4siv4qi2_mask_store, "__builtin_ia32_pmovdb128mem_mask", IX86_BUILTIN_PMOVDB128_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V4SI_QI },
31088 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev8siv8qi2_mask_store, "__builtin_ia32_pmovsdb256mem_mask", IX86_BUILTIN_PMOVSDB256_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V8SI_QI },
31089 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev4siv4qi2_mask_store, "__builtin_ia32_pmovsdb128mem_mask", IX86_BUILTIN_PMOVSDB128_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V4SI_QI },
31090 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev8siv8qi2_mask_store, "__builtin_ia32_pmovusdb256mem_mask", IX86_BUILTIN_PMOVUSDB256_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V8SI_QI },
31091 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev4siv4qi2_mask_store, "__builtin_ia32_pmovusdb128mem_mask", IX86_BUILTIN_PMOVUSDB128_MEM, UNKNOWN, (int) VOID_FTYPE_PV16QI_V4SI_QI },
31092 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev8siv8hi2_mask_store, "__builtin_ia32_pmovdw256mem_mask", IX86_BUILTIN_PMOVDW256_MEM, UNKNOWN, (int) VOID_FTYPE_PV8HI_V8SI_QI },
31093 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev4siv4hi2_mask_store, "__builtin_ia32_pmovdw128mem_mask", IX86_BUILTIN_PMOVDW128_MEM, UNKNOWN, (int) VOID_FTYPE_PV8HI_V4SI_QI },
31094 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev8siv8hi2_mask_store, "__builtin_ia32_pmovsdw256mem_mask", IX86_BUILTIN_PMOVSDW256_MEM, UNKNOWN, (int) VOID_FTYPE_PV8HI_V8SI_QI },
31095 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev4siv4hi2_mask_store, "__builtin_ia32_pmovsdw128mem_mask", IX86_BUILTIN_PMOVSDW128_MEM, UNKNOWN, (int) VOID_FTYPE_PV8HI_V4SI_QI },
31096 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev8siv8hi2_mask_store, "__builtin_ia32_pmovusdw256mem_mask", IX86_BUILTIN_PMOVUSDW256_MEM, UNKNOWN, (int) VOID_FTYPE_PV8HI_V8SI_QI },
31097 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev4siv4hi2_mask_store, "__builtin_ia32_pmovusdw128mem_mask", IX86_BUILTIN_PMOVUSDW128_MEM, UNKNOWN, (int) VOID_FTYPE_PV8HI_V4SI_QI },
31100 { OPTION_MASK_ISA_PCOMMIT, CODE_FOR_pcommit, "__builtin_ia32_pcommit", IX86_BUILTIN_PCOMMIT, UNKNOWN, (int) VOID_FTYPE_VOID },
31103 /* Builtins with variable number of arguments. */
31104 static const struct builtin_description bdesc_args[] =
31106 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_bsr, "__builtin_ia32_bsrsi", IX86_BUILTIN_BSRSI, UNKNOWN, (int) INT_FTYPE_INT },
31107 { OPTION_MASK_ISA_64BIT, CODE_FOR_bsr_rex64, "__builtin_ia32_bsrdi", IX86_BUILTIN_BSRDI, UNKNOWN, (int) INT64_FTYPE_INT64 },
31108 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_rdpmc", IX86_BUILTIN_RDPMC, UNKNOWN, (int) UINT64_FTYPE_INT },
31109 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rotlqi3, "__builtin_ia32_rolqi", IX86_BUILTIN_ROLQI, UNKNOWN, (int) UINT8_FTYPE_UINT8_INT },
31110 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rotlhi3, "__builtin_ia32_rolhi", IX86_BUILTIN_ROLHI, UNKNOWN, (int) UINT16_FTYPE_UINT16_INT },
31111 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rotrqi3, "__builtin_ia32_rorqi", IX86_BUILTIN_RORQI, UNKNOWN, (int) UINT8_FTYPE_UINT8_INT },
31112 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rotrhi3, "__builtin_ia32_rorhi", IX86_BUILTIN_RORHI, UNKNOWN, (int) UINT16_FTYPE_UINT16_INT },
31115 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_addv8qi3, "__builtin_ia32_paddb", IX86_BUILTIN_PADDB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
31116 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_addv4hi3, "__builtin_ia32_paddw", IX86_BUILTIN_PADDW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31117 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_addv2si3, "__builtin_ia32_paddd", IX86_BUILTIN_PADDD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
31118 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_subv8qi3, "__builtin_ia32_psubb", IX86_BUILTIN_PSUBB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
31119 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_subv4hi3, "__builtin_ia32_psubw", IX86_BUILTIN_PSUBW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31120 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_subv2si3, "__builtin_ia32_psubd", IX86_BUILTIN_PSUBD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
31122 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ssaddv8qi3, "__builtin_ia32_paddsb", IX86_BUILTIN_PADDSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
31123 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ssaddv4hi3, "__builtin_ia32_paddsw", IX86_BUILTIN_PADDSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31124 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_sssubv8qi3, "__builtin_ia32_psubsb", IX86_BUILTIN_PSUBSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
31125 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_sssubv4hi3, "__builtin_ia32_psubsw", IX86_BUILTIN_PSUBSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31126 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_usaddv8qi3, "__builtin_ia32_paddusb", IX86_BUILTIN_PADDUSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
31127 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_usaddv4hi3, "__builtin_ia32_paddusw", IX86_BUILTIN_PADDUSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31128 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ussubv8qi3, "__builtin_ia32_psubusb", IX86_BUILTIN_PSUBUSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
31129 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ussubv4hi3, "__builtin_ia32_psubusw", IX86_BUILTIN_PSUBUSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31131 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_mulv4hi3, "__builtin_ia32_pmullw", IX86_BUILTIN_PMULLW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31132 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_smulv4hi3_highpart, "__builtin_ia32_pmulhw", IX86_BUILTIN_PMULHW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31134 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_andv2si3, "__builtin_ia32_pand", IX86_BUILTIN_PAND, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
31135 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_andnotv2si3, "__builtin_ia32_pandn", IX86_BUILTIN_PANDN, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
31136 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_iorv2si3, "__builtin_ia32_por", IX86_BUILTIN_POR, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
31137 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_xorv2si3, "__builtin_ia32_pxor", IX86_BUILTIN_PXOR, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
31139 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_eqv8qi3, "__builtin_ia32_pcmpeqb", IX86_BUILTIN_PCMPEQB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
31140 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_eqv4hi3, "__builtin_ia32_pcmpeqw", IX86_BUILTIN_PCMPEQW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31141 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_eqv2si3, "__builtin_ia32_pcmpeqd", IX86_BUILTIN_PCMPEQD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
31142 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_gtv8qi3, "__builtin_ia32_pcmpgtb", IX86_BUILTIN_PCMPGTB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
31143 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_gtv4hi3, "__builtin_ia32_pcmpgtw", IX86_BUILTIN_PCMPGTW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31144 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_gtv2si3, "__builtin_ia32_pcmpgtd", IX86_BUILTIN_PCMPGTD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
31146 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckhbw, "__builtin_ia32_punpckhbw", IX86_BUILTIN_PUNPCKHBW, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
31147 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckhwd, "__builtin_ia32_punpckhwd", IX86_BUILTIN_PUNPCKHWD, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31148 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckhdq, "__builtin_ia32_punpckhdq", IX86_BUILTIN_PUNPCKHDQ, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
31149 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpcklbw, "__builtin_ia32_punpcklbw", IX86_BUILTIN_PUNPCKLBW, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
31150 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpcklwd, "__builtin_ia32_punpcklwd", IX86_BUILTIN_PUNPCKLWD, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI},
31151 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckldq, "__builtin_ia32_punpckldq", IX86_BUILTIN_PUNPCKLDQ, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI},
31153 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_packsswb, "__builtin_ia32_packsswb", IX86_BUILTIN_PACKSSWB, UNKNOWN, (int) V8QI_FTYPE_V4HI_V4HI },
31154 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_packssdw, "__builtin_ia32_packssdw", IX86_BUILTIN_PACKSSDW, UNKNOWN, (int) V4HI_FTYPE_V2SI_V2SI },
31155 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_packuswb, "__builtin_ia32_packuswb", IX86_BUILTIN_PACKUSWB, UNKNOWN, (int) V8QI_FTYPE_V4HI_V4HI },
31157 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_pmaddwd, "__builtin_ia32_pmaddwd", IX86_BUILTIN_PMADDWD, UNKNOWN, (int) V2SI_FTYPE_V4HI_V4HI },
31159 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv4hi3, "__builtin_ia32_psllwi", IX86_BUILTIN_PSLLWI, UNKNOWN, (int) V4HI_FTYPE_V4HI_SI_COUNT },
31160 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv2si3, "__builtin_ia32_pslldi", IX86_BUILTIN_PSLLDI, UNKNOWN, (int) V2SI_FTYPE_V2SI_SI_COUNT },
31161 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv1di3, "__builtin_ia32_psllqi", IX86_BUILTIN_PSLLQI, UNKNOWN, (int) V1DI_FTYPE_V1DI_SI_COUNT },
31162 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv4hi3, "__builtin_ia32_psllw", IX86_BUILTIN_PSLLW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI_COUNT },
31163 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv2si3, "__builtin_ia32_pslld", IX86_BUILTIN_PSLLD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI_COUNT },
31164 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv1di3, "__builtin_ia32_psllq", IX86_BUILTIN_PSLLQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI_COUNT },
31166 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv4hi3, "__builtin_ia32_psrlwi", IX86_BUILTIN_PSRLWI, UNKNOWN, (int) V4HI_FTYPE_V4HI_SI_COUNT },
31167 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv2si3, "__builtin_ia32_psrldi", IX86_BUILTIN_PSRLDI, UNKNOWN, (int) V2SI_FTYPE_V2SI_SI_COUNT },
31168 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv1di3, "__builtin_ia32_psrlqi", IX86_BUILTIN_PSRLQI, UNKNOWN, (int) V1DI_FTYPE_V1DI_SI_COUNT },
31169 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv4hi3, "__builtin_ia32_psrlw", IX86_BUILTIN_PSRLW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI_COUNT },
31170 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv2si3, "__builtin_ia32_psrld", IX86_BUILTIN_PSRLD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI_COUNT },
31171 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv1di3, "__builtin_ia32_psrlq", IX86_BUILTIN_PSRLQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI_COUNT },
31173 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv4hi3, "__builtin_ia32_psrawi", IX86_BUILTIN_PSRAWI, UNKNOWN, (int) V4HI_FTYPE_V4HI_SI_COUNT },
31174 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv2si3, "__builtin_ia32_psradi", IX86_BUILTIN_PSRADI, UNKNOWN, (int) V2SI_FTYPE_V2SI_SI_COUNT },
31175 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv4hi3, "__builtin_ia32_psraw", IX86_BUILTIN_PSRAW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI_COUNT },
31176 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv2si3, "__builtin_ia32_psrad", IX86_BUILTIN_PSRAD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI_COUNT },
31179 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_pf2id, "__builtin_ia32_pf2id", IX86_BUILTIN_PF2ID, UNKNOWN, (int) V2SI_FTYPE_V2SF },
31180 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_floatv2si2, "__builtin_ia32_pi2fd", IX86_BUILTIN_PI2FD, UNKNOWN, (int) V2SF_FTYPE_V2SI },
31181 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rcpv2sf2, "__builtin_ia32_pfrcp", IX86_BUILTIN_PFRCP, UNKNOWN, (int) V2SF_FTYPE_V2SF },
31182 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rsqrtv2sf2, "__builtin_ia32_pfrsqrt", IX86_BUILTIN_PFRSQRT, UNKNOWN, (int) V2SF_FTYPE_V2SF },
31184 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_uavgv8qi3, "__builtin_ia32_pavgusb", IX86_BUILTIN_PAVGUSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
31185 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_haddv2sf3, "__builtin_ia32_pfacc", IX86_BUILTIN_PFACC, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
31186 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_addv2sf3, "__builtin_ia32_pfadd", IX86_BUILTIN_PFADD, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
31187 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_eqv2sf3, "__builtin_ia32_pfcmpeq", IX86_BUILTIN_PFCMPEQ, UNKNOWN, (int) V2SI_FTYPE_V2SF_V2SF },
31188 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_gev2sf3, "__builtin_ia32_pfcmpge", IX86_BUILTIN_PFCMPGE, UNKNOWN, (int) V2SI_FTYPE_V2SF_V2SF },
31189 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_gtv2sf3, "__builtin_ia32_pfcmpgt", IX86_BUILTIN_PFCMPGT, UNKNOWN, (int) V2SI_FTYPE_V2SF_V2SF },
31190 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_smaxv2sf3, "__builtin_ia32_pfmax", IX86_BUILTIN_PFMAX, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
31191 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_sminv2sf3, "__builtin_ia32_pfmin", IX86_BUILTIN_PFMIN, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
31192 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_mulv2sf3, "__builtin_ia32_pfmul", IX86_BUILTIN_PFMUL, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
31193 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rcpit1v2sf3, "__builtin_ia32_pfrcpit1", IX86_BUILTIN_PFRCPIT1, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
31194 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rcpit2v2sf3, "__builtin_ia32_pfrcpit2", IX86_BUILTIN_PFRCPIT2, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
31195 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rsqit1v2sf3, "__builtin_ia32_pfrsqit1", IX86_BUILTIN_PFRSQIT1, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
31196 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_subv2sf3, "__builtin_ia32_pfsub", IX86_BUILTIN_PFSUB, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
31197 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_subrv2sf3, "__builtin_ia32_pfsubr", IX86_BUILTIN_PFSUBR, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
31198 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_pmulhrwv4hi3, "__builtin_ia32_pmulhrw", IX86_BUILTIN_PMULHRW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31201 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pf2iw, "__builtin_ia32_pf2iw", IX86_BUILTIN_PF2IW, UNKNOWN, (int) V2SI_FTYPE_V2SF },
31202 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pi2fw, "__builtin_ia32_pi2fw", IX86_BUILTIN_PI2FW, UNKNOWN, (int) V2SF_FTYPE_V2SI },
31203 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pswapdv2si2, "__builtin_ia32_pswapdsi", IX86_BUILTIN_PSWAPDSI, UNKNOWN, (int) V2SI_FTYPE_V2SI },
31204 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pswapdv2sf2, "__builtin_ia32_pswapdsf", IX86_BUILTIN_PSWAPDSF, UNKNOWN, (int) V2SF_FTYPE_V2SF },
31205 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_hsubv2sf3, "__builtin_ia32_pfnacc", IX86_BUILTIN_PFNACC, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
31206 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_addsubv2sf3, "__builtin_ia32_pfpnacc", IX86_BUILTIN_PFPNACC, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
31209 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movmskps, "__builtin_ia32_movmskps", IX86_BUILTIN_MOVMSKPS, UNKNOWN, (int) INT_FTYPE_V4SF },
31210 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_sqrtv4sf2, "__builtin_ia32_sqrtps", IX86_BUILTIN_SQRTPS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
31211 { OPTION_MASK_ISA_SSE, CODE_FOR_sqrtv4sf2, "__builtin_ia32_sqrtps_nr", IX86_BUILTIN_SQRTPS_NR, UNKNOWN, (int) V4SF_FTYPE_V4SF },
31212 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_rsqrtv4sf2, "__builtin_ia32_rsqrtps", IX86_BUILTIN_RSQRTPS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
31213 { OPTION_MASK_ISA_SSE, CODE_FOR_rsqrtv4sf2, "__builtin_ia32_rsqrtps_nr", IX86_BUILTIN_RSQRTPS_NR, UNKNOWN, (int) V4SF_FTYPE_V4SF },
31214 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_rcpv4sf2, "__builtin_ia32_rcpps", IX86_BUILTIN_RCPPS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
31215 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtps2pi, "__builtin_ia32_cvtps2pi", IX86_BUILTIN_CVTPS2PI, UNKNOWN, (int) V2SI_FTYPE_V4SF },
31216 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtss2si, "__builtin_ia32_cvtss2si", IX86_BUILTIN_CVTSS2SI, UNKNOWN, (int) INT_FTYPE_V4SF },
31217 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvtss2siq, "__builtin_ia32_cvtss2si64", IX86_BUILTIN_CVTSS2SI64, UNKNOWN, (int) INT64_FTYPE_V4SF },
31218 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvttps2pi, "__builtin_ia32_cvttps2pi", IX86_BUILTIN_CVTTPS2PI, UNKNOWN, (int) V2SI_FTYPE_V4SF },
31219 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvttss2si, "__builtin_ia32_cvttss2si", IX86_BUILTIN_CVTTSS2SI, UNKNOWN, (int) INT_FTYPE_V4SF },
31220 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvttss2siq, "__builtin_ia32_cvttss2si64", IX86_BUILTIN_CVTTSS2SI64, UNKNOWN, (int) INT64_FTYPE_V4SF },
31222 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_shufps, "__builtin_ia32_shufps", IX86_BUILTIN_SHUFPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
31224 { OPTION_MASK_ISA_SSE, CODE_FOR_addv4sf3, "__builtin_ia32_addps", IX86_BUILTIN_ADDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31225 { OPTION_MASK_ISA_SSE, CODE_FOR_subv4sf3, "__builtin_ia32_subps", IX86_BUILTIN_SUBPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31226 { OPTION_MASK_ISA_SSE, CODE_FOR_mulv4sf3, "__builtin_ia32_mulps", IX86_BUILTIN_MULPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31227 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_divv4sf3, "__builtin_ia32_divps", IX86_BUILTIN_DIVPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31228 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmaddv4sf3, "__builtin_ia32_addss", IX86_BUILTIN_ADDSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31229 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsubv4sf3, "__builtin_ia32_subss", IX86_BUILTIN_SUBSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31230 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmulv4sf3, "__builtin_ia32_mulss", IX86_BUILTIN_MULSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31231 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmdivv4sf3, "__builtin_ia32_divss", IX86_BUILTIN_DIVSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31233 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpeqps", IX86_BUILTIN_CMPEQPS, EQ, (int) V4SF_FTYPE_V4SF_V4SF },
31234 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpltps", IX86_BUILTIN_CMPLTPS, LT, (int) V4SF_FTYPE_V4SF_V4SF },
31235 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpleps", IX86_BUILTIN_CMPLEPS, LE, (int) V4SF_FTYPE_V4SF_V4SF },
31236 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpgtps", IX86_BUILTIN_CMPGTPS, LT, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
31237 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpgeps", IX86_BUILTIN_CMPGEPS, LE, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
31238 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpunordps", IX86_BUILTIN_CMPUNORDPS, UNORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
31239 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpneqps", IX86_BUILTIN_CMPNEQPS, NE, (int) V4SF_FTYPE_V4SF_V4SF },
31240 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpnltps", IX86_BUILTIN_CMPNLTPS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF },
31241 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpnleps", IX86_BUILTIN_CMPNLEPS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF },
31242 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpngtps", IX86_BUILTIN_CMPNGTPS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
31243 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpngeps", IX86_BUILTIN_CMPNGEPS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF_SWAP},
31244 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpordps", IX86_BUILTIN_CMPORDPS, ORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
31245 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpeqss", IX86_BUILTIN_CMPEQSS, EQ, (int) V4SF_FTYPE_V4SF_V4SF },
31246 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpltss", IX86_BUILTIN_CMPLTSS, LT, (int) V4SF_FTYPE_V4SF_V4SF },
31247 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpless", IX86_BUILTIN_CMPLESS, LE, (int) V4SF_FTYPE_V4SF_V4SF },
31248 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpunordss", IX86_BUILTIN_CMPUNORDSS, UNORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
31249 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpneqss", IX86_BUILTIN_CMPNEQSS, NE, (int) V4SF_FTYPE_V4SF_V4SF },
31250 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpnltss", IX86_BUILTIN_CMPNLTSS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF },
31251 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpnless", IX86_BUILTIN_CMPNLESS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF },
31252 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpordss", IX86_BUILTIN_CMPORDSS, ORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
31254 { OPTION_MASK_ISA_SSE, CODE_FOR_sminv4sf3, "__builtin_ia32_minps", IX86_BUILTIN_MINPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31255 { OPTION_MASK_ISA_SSE, CODE_FOR_smaxv4sf3, "__builtin_ia32_maxps", IX86_BUILTIN_MAXPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31256 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsminv4sf3, "__builtin_ia32_minss", IX86_BUILTIN_MINSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31257 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsmaxv4sf3, "__builtin_ia32_maxss", IX86_BUILTIN_MAXSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31259 { OPTION_MASK_ISA_SSE, CODE_FOR_andv4sf3, "__builtin_ia32_andps", IX86_BUILTIN_ANDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31260 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_andnotv4sf3, "__builtin_ia32_andnps", IX86_BUILTIN_ANDNPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31261 { OPTION_MASK_ISA_SSE, CODE_FOR_iorv4sf3, "__builtin_ia32_orps", IX86_BUILTIN_ORPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31262 { OPTION_MASK_ISA_SSE, CODE_FOR_xorv4sf3, "__builtin_ia32_xorps", IX86_BUILTIN_XORPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31264 { OPTION_MASK_ISA_SSE, CODE_FOR_copysignv4sf3, "__builtin_ia32_copysignps", IX86_BUILTIN_CPYSGNPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31266 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movss, "__builtin_ia32_movss", IX86_BUILTIN_MOVSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31267 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movhlps_exp, "__builtin_ia32_movhlps", IX86_BUILTIN_MOVHLPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31268 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movlhps_exp, "__builtin_ia32_movlhps", IX86_BUILTIN_MOVLHPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31269 { OPTION_MASK_ISA_SSE, CODE_FOR_vec_interleave_highv4sf, "__builtin_ia32_unpckhps", IX86_BUILTIN_UNPCKHPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31270 { OPTION_MASK_ISA_SSE, CODE_FOR_vec_interleave_lowv4sf, "__builtin_ia32_unpcklps", IX86_BUILTIN_UNPCKLPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31272 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtpi2ps, "__builtin_ia32_cvtpi2ps", IX86_BUILTIN_CVTPI2PS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V2SI },
31273 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtsi2ss, "__builtin_ia32_cvtsi2ss", IX86_BUILTIN_CVTSI2SS, UNKNOWN, (int) V4SF_FTYPE_V4SF_SI },
31274 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvtsi2ssq, "__builtin_ia32_cvtsi642ss", IX86_BUILTIN_CVTSI642SS, UNKNOWN, V4SF_FTYPE_V4SF_DI },
31276 { OPTION_MASK_ISA_SSE, CODE_FOR_rsqrtsf2, "__builtin_ia32_rsqrtf", IX86_BUILTIN_RSQRTF, UNKNOWN, (int) FLOAT_FTYPE_FLOAT },
31278 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsqrtv4sf2, "__builtin_ia32_sqrtss", IX86_BUILTIN_SQRTSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_VEC_MERGE },
31279 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmrsqrtv4sf2, "__builtin_ia32_rsqrtss", IX86_BUILTIN_RSQRTSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_VEC_MERGE },
31280 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmrcpv4sf2, "__builtin_ia32_rcpss", IX86_BUILTIN_RCPSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_VEC_MERGE },
31282 { OPTION_MASK_ISA_SSE, CODE_FOR_abstf2, 0, IX86_BUILTIN_FABSQ, UNKNOWN, (int) FLOAT128_FTYPE_FLOAT128 },
31283 { OPTION_MASK_ISA_SSE, CODE_FOR_copysigntf3, 0, IX86_BUILTIN_COPYSIGNQ, UNKNOWN, (int) FLOAT128_FTYPE_FLOAT128_FLOAT128 },
31285 /* SSE MMX or 3Dnow!A */
31286 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_uavgv8qi3, "__builtin_ia32_pavgb", IX86_BUILTIN_PAVGB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
31287 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_uavgv4hi3, "__builtin_ia32_pavgw", IX86_BUILTIN_PAVGW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31288 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_umulv4hi3_highpart, "__builtin_ia32_pmulhuw", IX86_BUILTIN_PMULHUW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31290 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_umaxv8qi3, "__builtin_ia32_pmaxub", IX86_BUILTIN_PMAXUB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
31291 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_smaxv4hi3, "__builtin_ia32_pmaxsw", IX86_BUILTIN_PMAXSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31292 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_uminv8qi3, "__builtin_ia32_pminub", IX86_BUILTIN_PMINUB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
31293 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_sminv4hi3, "__builtin_ia32_pminsw", IX86_BUILTIN_PMINSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31295 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_psadbw, "__builtin_ia32_psadbw", IX86_BUILTIN_PSADBW, UNKNOWN, (int) V1DI_FTYPE_V8QI_V8QI },
31296 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pmovmskb, "__builtin_ia32_pmovmskb", IX86_BUILTIN_PMOVMSKB, UNKNOWN, (int) INT_FTYPE_V8QI },
31298 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pshufw, "__builtin_ia32_pshufw", IX86_BUILTIN_PSHUFW, UNKNOWN, (int) V4HI_FTYPE_V4HI_INT },
31301 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_shufpd, "__builtin_ia32_shufpd", IX86_BUILTIN_SHUFPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
31303 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movmskpd, "__builtin_ia32_movmskpd", IX86_BUILTIN_MOVMSKPD, UNKNOWN, (int) INT_FTYPE_V2DF },
31304 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pmovmskb, "__builtin_ia32_pmovmskb128", IX86_BUILTIN_PMOVMSKB128, UNKNOWN, (int) INT_FTYPE_V16QI },
31305 { OPTION_MASK_ISA_SSE2, CODE_FOR_sqrtv2df2, "__builtin_ia32_sqrtpd", IX86_BUILTIN_SQRTPD, UNKNOWN, (int) V2DF_FTYPE_V2DF },
31306 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtdq2pd, "__builtin_ia32_cvtdq2pd", IX86_BUILTIN_CVTDQ2PD, UNKNOWN, (int) V2DF_FTYPE_V4SI },
31307 { OPTION_MASK_ISA_SSE2, CODE_FOR_floatv4siv4sf2, "__builtin_ia32_cvtdq2ps", IX86_BUILTIN_CVTDQ2PS, UNKNOWN, (int) V4SF_FTYPE_V4SI },
31309 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpd2dq, "__builtin_ia32_cvtpd2dq", IX86_BUILTIN_CVTPD2DQ, UNKNOWN, (int) V4SI_FTYPE_V2DF },
31310 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpd2pi, "__builtin_ia32_cvtpd2pi", IX86_BUILTIN_CVTPD2PI, UNKNOWN, (int) V2SI_FTYPE_V2DF },
31311 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpd2ps, "__builtin_ia32_cvtpd2ps", IX86_BUILTIN_CVTPD2PS, UNKNOWN, (int) V4SF_FTYPE_V2DF },
31312 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvttpd2dq, "__builtin_ia32_cvttpd2dq", IX86_BUILTIN_CVTTPD2DQ, UNKNOWN, (int) V4SI_FTYPE_V2DF },
31313 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvttpd2pi, "__builtin_ia32_cvttpd2pi", IX86_BUILTIN_CVTTPD2PI, UNKNOWN, (int) V2SI_FTYPE_V2DF },
31315 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpi2pd, "__builtin_ia32_cvtpi2pd", IX86_BUILTIN_CVTPI2PD, UNKNOWN, (int) V2DF_FTYPE_V2SI },
31317 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtsd2si, "__builtin_ia32_cvtsd2si", IX86_BUILTIN_CVTSD2SI, UNKNOWN, (int) INT_FTYPE_V2DF },
31318 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvttsd2si, "__builtin_ia32_cvttsd2si", IX86_BUILTIN_CVTTSD2SI, UNKNOWN, (int) INT_FTYPE_V2DF },
31319 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvtsd2siq, "__builtin_ia32_cvtsd2si64", IX86_BUILTIN_CVTSD2SI64, UNKNOWN, (int) INT64_FTYPE_V2DF },
31320 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvttsd2siq, "__builtin_ia32_cvttsd2si64", IX86_BUILTIN_CVTTSD2SI64, UNKNOWN, (int) INT64_FTYPE_V2DF },
31322 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_fix_notruncv4sfv4si, "__builtin_ia32_cvtps2dq", IX86_BUILTIN_CVTPS2DQ, UNKNOWN, (int) V4SI_FTYPE_V4SF },
31323 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtps2pd, "__builtin_ia32_cvtps2pd", IX86_BUILTIN_CVTPS2PD, UNKNOWN, (int) V2DF_FTYPE_V4SF },
31324 { OPTION_MASK_ISA_SSE2, CODE_FOR_fix_truncv4sfv4si2, "__builtin_ia32_cvttps2dq", IX86_BUILTIN_CVTTPS2DQ, UNKNOWN, (int) V4SI_FTYPE_V4SF },
31326 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv2df3, "__builtin_ia32_addpd", IX86_BUILTIN_ADDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31327 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv2df3, "__builtin_ia32_subpd", IX86_BUILTIN_SUBPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31328 { OPTION_MASK_ISA_SSE2, CODE_FOR_mulv2df3, "__builtin_ia32_mulpd", IX86_BUILTIN_MULPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31329 { OPTION_MASK_ISA_SSE2, CODE_FOR_divv2df3, "__builtin_ia32_divpd", IX86_BUILTIN_DIVPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31330 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmaddv2df3, "__builtin_ia32_addsd", IX86_BUILTIN_ADDSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31331 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsubv2df3, "__builtin_ia32_subsd", IX86_BUILTIN_SUBSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31332 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmulv2df3, "__builtin_ia32_mulsd", IX86_BUILTIN_MULSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31333 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmdivv2df3, "__builtin_ia32_divsd", IX86_BUILTIN_DIVSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31335 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpeqpd", IX86_BUILTIN_CMPEQPD, EQ, (int) V2DF_FTYPE_V2DF_V2DF },
31336 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpltpd", IX86_BUILTIN_CMPLTPD, LT, (int) V2DF_FTYPE_V2DF_V2DF },
31337 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmplepd", IX86_BUILTIN_CMPLEPD, LE, (int) V2DF_FTYPE_V2DF_V2DF },
31338 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpgtpd", IX86_BUILTIN_CMPGTPD, LT, (int) V2DF_FTYPE_V2DF_V2DF_SWAP },
31339 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpgepd", IX86_BUILTIN_CMPGEPD, LE, (int) V2DF_FTYPE_V2DF_V2DF_SWAP},
31340 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpunordpd", IX86_BUILTIN_CMPUNORDPD, UNORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
31341 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpneqpd", IX86_BUILTIN_CMPNEQPD, NE, (int) V2DF_FTYPE_V2DF_V2DF },
31342 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpnltpd", IX86_BUILTIN_CMPNLTPD, UNGE, (int) V2DF_FTYPE_V2DF_V2DF },
31343 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpnlepd", IX86_BUILTIN_CMPNLEPD, UNGT, (int) V2DF_FTYPE_V2DF_V2DF },
31344 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpngtpd", IX86_BUILTIN_CMPNGTPD, UNGE, (int) V2DF_FTYPE_V2DF_V2DF_SWAP },
31345 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpngepd", IX86_BUILTIN_CMPNGEPD, UNGT, (int) V2DF_FTYPE_V2DF_V2DF_SWAP },
31346 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpordpd", IX86_BUILTIN_CMPORDPD, ORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
31347 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpeqsd", IX86_BUILTIN_CMPEQSD, EQ, (int) V2DF_FTYPE_V2DF_V2DF },
31348 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpltsd", IX86_BUILTIN_CMPLTSD, LT, (int) V2DF_FTYPE_V2DF_V2DF },
31349 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmplesd", IX86_BUILTIN_CMPLESD, LE, (int) V2DF_FTYPE_V2DF_V2DF },
31350 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpunordsd", IX86_BUILTIN_CMPUNORDSD, UNORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
31351 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpneqsd", IX86_BUILTIN_CMPNEQSD, NE, (int) V2DF_FTYPE_V2DF_V2DF },
31352 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpnltsd", IX86_BUILTIN_CMPNLTSD, UNGE, (int) V2DF_FTYPE_V2DF_V2DF },
31353 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpnlesd", IX86_BUILTIN_CMPNLESD, UNGT, (int) V2DF_FTYPE_V2DF_V2DF },
31354 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpordsd", IX86_BUILTIN_CMPORDSD, ORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
31356 { OPTION_MASK_ISA_SSE2, CODE_FOR_sminv2df3, "__builtin_ia32_minpd", IX86_BUILTIN_MINPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31357 { OPTION_MASK_ISA_SSE2, CODE_FOR_smaxv2df3, "__builtin_ia32_maxpd", IX86_BUILTIN_MAXPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31358 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsminv2df3, "__builtin_ia32_minsd", IX86_BUILTIN_MINSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31359 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsmaxv2df3, "__builtin_ia32_maxsd", IX86_BUILTIN_MAXSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31361 { OPTION_MASK_ISA_SSE2, CODE_FOR_andv2df3, "__builtin_ia32_andpd", IX86_BUILTIN_ANDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31362 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_andnotv2df3, "__builtin_ia32_andnpd", IX86_BUILTIN_ANDNPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31363 { OPTION_MASK_ISA_SSE2, CODE_FOR_iorv2df3, "__builtin_ia32_orpd", IX86_BUILTIN_ORPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31364 { OPTION_MASK_ISA_SSE2, CODE_FOR_xorv2df3, "__builtin_ia32_xorpd", IX86_BUILTIN_XORPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31366 { OPTION_MASK_ISA_SSE2, CODE_FOR_copysignv2df3, "__builtin_ia32_copysignpd", IX86_BUILTIN_CPYSGNPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31368 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movsd, "__builtin_ia32_movsd", IX86_BUILTIN_MOVSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31369 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv2df, "__builtin_ia32_unpckhpd", IX86_BUILTIN_UNPCKHPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31370 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv2df, "__builtin_ia32_unpcklpd", IX86_BUILTIN_UNPCKLPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31372 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_pack_sfix_v2df, "__builtin_ia32_vec_pack_sfix", IX86_BUILTIN_VEC_PACK_SFIX, UNKNOWN, (int) V4SI_FTYPE_V2DF_V2DF },
31374 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv16qi3, "__builtin_ia32_paddb128", IX86_BUILTIN_PADDB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31375 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv8hi3, "__builtin_ia32_paddw128", IX86_BUILTIN_PADDW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31376 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv4si3, "__builtin_ia32_paddd128", IX86_BUILTIN_PADDD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
31377 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv2di3, "__builtin_ia32_paddq128", IX86_BUILTIN_PADDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
31378 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv16qi3, "__builtin_ia32_psubb128", IX86_BUILTIN_PSUBB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31379 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv8hi3, "__builtin_ia32_psubw128", IX86_BUILTIN_PSUBW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31380 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv4si3, "__builtin_ia32_psubd128", IX86_BUILTIN_PSUBD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
31381 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv2di3, "__builtin_ia32_psubq128", IX86_BUILTIN_PSUBQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
31383 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ssaddv16qi3, "__builtin_ia32_paddsb128", IX86_BUILTIN_PADDSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31384 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ssaddv8hi3, "__builtin_ia32_paddsw128", IX86_BUILTIN_PADDSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31385 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_sssubv16qi3, "__builtin_ia32_psubsb128", IX86_BUILTIN_PSUBSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31386 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_sssubv8hi3, "__builtin_ia32_psubsw128", IX86_BUILTIN_PSUBSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31387 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_usaddv16qi3, "__builtin_ia32_paddusb128", IX86_BUILTIN_PADDUSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31388 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_usaddv8hi3, "__builtin_ia32_paddusw128", IX86_BUILTIN_PADDUSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31389 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ussubv16qi3, "__builtin_ia32_psubusb128", IX86_BUILTIN_PSUBUSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31390 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ussubv8hi3, "__builtin_ia32_psubusw128", IX86_BUILTIN_PSUBUSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31392 { OPTION_MASK_ISA_SSE2, CODE_FOR_mulv8hi3, "__builtin_ia32_pmullw128", IX86_BUILTIN_PMULLW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31393 { OPTION_MASK_ISA_SSE2, CODE_FOR_smulv8hi3_highpart, "__builtin_ia32_pmulhw128", IX86_BUILTIN_PMULHW128, UNKNOWN,(int) V8HI_FTYPE_V8HI_V8HI },
31395 { OPTION_MASK_ISA_SSE2, CODE_FOR_andv2di3, "__builtin_ia32_pand128", IX86_BUILTIN_PAND128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
31396 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_andnotv2di3, "__builtin_ia32_pandn128", IX86_BUILTIN_PANDN128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
31397 { OPTION_MASK_ISA_SSE2, CODE_FOR_iorv2di3, "__builtin_ia32_por128", IX86_BUILTIN_POR128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
31398 { OPTION_MASK_ISA_SSE2, CODE_FOR_xorv2di3, "__builtin_ia32_pxor128", IX86_BUILTIN_PXOR128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
31400 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_uavgv16qi3, "__builtin_ia32_pavgb128", IX86_BUILTIN_PAVGB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31401 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_uavgv8hi3, "__builtin_ia32_pavgw128", IX86_BUILTIN_PAVGW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31403 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_eqv16qi3, "__builtin_ia32_pcmpeqb128", IX86_BUILTIN_PCMPEQB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31404 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_eqv8hi3, "__builtin_ia32_pcmpeqw128", IX86_BUILTIN_PCMPEQW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31405 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_eqv4si3, "__builtin_ia32_pcmpeqd128", IX86_BUILTIN_PCMPEQD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
31406 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_gtv16qi3, "__builtin_ia32_pcmpgtb128", IX86_BUILTIN_PCMPGTB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31407 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_gtv8hi3, "__builtin_ia32_pcmpgtw128", IX86_BUILTIN_PCMPGTW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31408 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_gtv4si3, "__builtin_ia32_pcmpgtd128", IX86_BUILTIN_PCMPGTD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
31410 { OPTION_MASK_ISA_SSE2, CODE_FOR_umaxv16qi3, "__builtin_ia32_pmaxub128", IX86_BUILTIN_PMAXUB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31411 { OPTION_MASK_ISA_SSE2, CODE_FOR_smaxv8hi3, "__builtin_ia32_pmaxsw128", IX86_BUILTIN_PMAXSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31412 { OPTION_MASK_ISA_SSE2, CODE_FOR_uminv16qi3, "__builtin_ia32_pminub128", IX86_BUILTIN_PMINUB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31413 { OPTION_MASK_ISA_SSE2, CODE_FOR_sminv8hi3, "__builtin_ia32_pminsw128", IX86_BUILTIN_PMINSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31415 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv16qi, "__builtin_ia32_punpckhbw128", IX86_BUILTIN_PUNPCKHBW128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31416 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv8hi, "__builtin_ia32_punpckhwd128", IX86_BUILTIN_PUNPCKHWD128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31417 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv4si, "__builtin_ia32_punpckhdq128", IX86_BUILTIN_PUNPCKHDQ128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
31418 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv2di, "__builtin_ia32_punpckhqdq128", IX86_BUILTIN_PUNPCKHQDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
31419 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv16qi, "__builtin_ia32_punpcklbw128", IX86_BUILTIN_PUNPCKLBW128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31420 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv8hi, "__builtin_ia32_punpcklwd128", IX86_BUILTIN_PUNPCKLWD128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31421 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv4si, "__builtin_ia32_punpckldq128", IX86_BUILTIN_PUNPCKLDQ128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
31422 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv2di, "__builtin_ia32_punpcklqdq128", IX86_BUILTIN_PUNPCKLQDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
31424 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_packsswb, "__builtin_ia32_packsswb128", IX86_BUILTIN_PACKSSWB128, UNKNOWN, (int) V16QI_FTYPE_V8HI_V8HI },
31425 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_packssdw, "__builtin_ia32_packssdw128", IX86_BUILTIN_PACKSSDW128, UNKNOWN, (int) V8HI_FTYPE_V4SI_V4SI },
31426 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_packuswb, "__builtin_ia32_packuswb128", IX86_BUILTIN_PACKUSWB128, UNKNOWN, (int) V16QI_FTYPE_V8HI_V8HI },
31428 { OPTION_MASK_ISA_SSE2, CODE_FOR_umulv8hi3_highpart, "__builtin_ia32_pmulhuw128", IX86_BUILTIN_PMULHUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31429 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_psadbw, "__builtin_ia32_psadbw128", IX86_BUILTIN_PSADBW128, UNKNOWN, (int) V2DI_FTYPE_V16QI_V16QI },
31431 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_umulv1siv1di3, "__builtin_ia32_pmuludq", IX86_BUILTIN_PMULUDQ, UNKNOWN, (int) V1DI_FTYPE_V2SI_V2SI },
31432 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_widen_umult_even_v4si, "__builtin_ia32_pmuludq128", IX86_BUILTIN_PMULUDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI_V4SI },
31434 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pmaddwd, "__builtin_ia32_pmaddwd128", IX86_BUILTIN_PMADDWD128, UNKNOWN, (int) V4SI_FTYPE_V8HI_V8HI },
31436 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtsi2sd, "__builtin_ia32_cvtsi2sd", IX86_BUILTIN_CVTSI2SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_SI },
31437 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvtsi2sdq, "__builtin_ia32_cvtsi642sd", IX86_BUILTIN_CVTSI642SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_DI },
31438 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtsd2ss, "__builtin_ia32_cvtsd2ss", IX86_BUILTIN_CVTSD2SS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V2DF },
31439 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtss2sd, "__builtin_ia32_cvtss2sd", IX86_BUILTIN_CVTSS2SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V4SF },
31441 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ashlv1ti3, "__builtin_ia32_pslldqi128", IX86_BUILTIN_PSLLDQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT_CONVERT },
31442 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv8hi3, "__builtin_ia32_psllwi128", IX86_BUILTIN_PSLLWI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_SI_COUNT },
31443 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv4si3, "__builtin_ia32_pslldi128", IX86_BUILTIN_PSLLDI128, UNKNOWN, (int) V4SI_FTYPE_V4SI_SI_COUNT },
31444 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv2di3, "__builtin_ia32_psllqi128", IX86_BUILTIN_PSLLQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_SI_COUNT },
31445 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv8hi3, "__builtin_ia32_psllw128", IX86_BUILTIN_PSLLW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_COUNT },
31446 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv4si3, "__builtin_ia32_pslld128", IX86_BUILTIN_PSLLD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_COUNT },
31447 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv2di3, "__builtin_ia32_psllq128", IX86_BUILTIN_PSLLQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_COUNT },
31449 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_lshrv1ti3, "__builtin_ia32_psrldqi128", IX86_BUILTIN_PSRLDQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT_CONVERT },
31450 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv8hi3, "__builtin_ia32_psrlwi128", IX86_BUILTIN_PSRLWI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_SI_COUNT },
31451 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv4si3, "__builtin_ia32_psrldi128", IX86_BUILTIN_PSRLDI128, UNKNOWN, (int) V4SI_FTYPE_V4SI_SI_COUNT },
31452 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv2di3, "__builtin_ia32_psrlqi128", IX86_BUILTIN_PSRLQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_SI_COUNT },
31453 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv8hi3, "__builtin_ia32_psrlw128", IX86_BUILTIN_PSRLW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_COUNT },
31454 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv4si3, "__builtin_ia32_psrld128", IX86_BUILTIN_PSRLD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_COUNT },
31455 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv2di3, "__builtin_ia32_psrlq128", IX86_BUILTIN_PSRLQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_COUNT },
31457 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv8hi3, "__builtin_ia32_psrawi128", IX86_BUILTIN_PSRAWI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_SI_COUNT },
31458 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv4si3, "__builtin_ia32_psradi128", IX86_BUILTIN_PSRADI128, UNKNOWN, (int) V4SI_FTYPE_V4SI_SI_COUNT },
31459 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv8hi3, "__builtin_ia32_psraw128", IX86_BUILTIN_PSRAW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_COUNT },
31460 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv4si3, "__builtin_ia32_psrad128", IX86_BUILTIN_PSRAD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_COUNT },
31462 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pshufd, "__builtin_ia32_pshufd", IX86_BUILTIN_PSHUFD, UNKNOWN, (int) V4SI_FTYPE_V4SI_INT },
31463 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pshuflw, "__builtin_ia32_pshuflw", IX86_BUILTIN_PSHUFLW, UNKNOWN, (int) V8HI_FTYPE_V8HI_INT },
31464 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pshufhw, "__builtin_ia32_pshufhw", IX86_BUILTIN_PSHUFHW, UNKNOWN, (int) V8HI_FTYPE_V8HI_INT },
31466 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsqrtv2df2, "__builtin_ia32_sqrtsd", IX86_BUILTIN_SQRTSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_VEC_MERGE },
31468 { OPTION_MASK_ISA_SSE, CODE_FOR_sse2_movq128, "__builtin_ia32_movq128", IX86_BUILTIN_MOVQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI },
31471 { OPTION_MASK_ISA_SSE2, CODE_FOR_mmx_addv1di3, "__builtin_ia32_paddq", IX86_BUILTIN_PADDQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI },
31472 { OPTION_MASK_ISA_SSE2, CODE_FOR_mmx_subv1di3, "__builtin_ia32_psubq", IX86_BUILTIN_PSUBQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI },
31475 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_movshdup, "__builtin_ia32_movshdup", IX86_BUILTIN_MOVSHDUP, UNKNOWN, (int) V4SF_FTYPE_V4SF},
31476 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_movsldup, "__builtin_ia32_movsldup", IX86_BUILTIN_MOVSLDUP, UNKNOWN, (int) V4SF_FTYPE_V4SF },
31478 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_addsubv4sf3, "__builtin_ia32_addsubps", IX86_BUILTIN_ADDSUBPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31479 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_addsubv2df3, "__builtin_ia32_addsubpd", IX86_BUILTIN_ADDSUBPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31480 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_haddv4sf3, "__builtin_ia32_haddps", IX86_BUILTIN_HADDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31481 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_haddv2df3, "__builtin_ia32_haddpd", IX86_BUILTIN_HADDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31482 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_hsubv4sf3, "__builtin_ia32_hsubps", IX86_BUILTIN_HSUBPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
31483 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_hsubv2df3, "__builtin_ia32_hsubpd", IX86_BUILTIN_HSUBPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
31486 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv16qi2, "__builtin_ia32_pabsb128", IX86_BUILTIN_PABSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI },
31487 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv8qi2, "__builtin_ia32_pabsb", IX86_BUILTIN_PABSB, UNKNOWN, (int) V8QI_FTYPE_V8QI },
31488 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv8hi2, "__builtin_ia32_pabsw128", IX86_BUILTIN_PABSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI },
31489 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv4hi2, "__builtin_ia32_pabsw", IX86_BUILTIN_PABSW, UNKNOWN, (int) V4HI_FTYPE_V4HI },
31490 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv4si2, "__builtin_ia32_pabsd128", IX86_BUILTIN_PABSD128, UNKNOWN, (int) V4SI_FTYPE_V4SI },
31491 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv2si2, "__builtin_ia32_pabsd", IX86_BUILTIN_PABSD, UNKNOWN, (int) V2SI_FTYPE_V2SI },
31493 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddwv8hi3, "__builtin_ia32_phaddw128", IX86_BUILTIN_PHADDW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31494 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddwv4hi3, "__builtin_ia32_phaddw", IX86_BUILTIN_PHADDW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31495 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phadddv4si3, "__builtin_ia32_phaddd128", IX86_BUILTIN_PHADDD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
31496 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phadddv2si3, "__builtin_ia32_phaddd", IX86_BUILTIN_PHADDD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
31497 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddswv8hi3, "__builtin_ia32_phaddsw128", IX86_BUILTIN_PHADDSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31498 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddswv4hi3, "__builtin_ia32_phaddsw", IX86_BUILTIN_PHADDSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31499 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubwv8hi3, "__builtin_ia32_phsubw128", IX86_BUILTIN_PHSUBW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31500 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubwv4hi3, "__builtin_ia32_phsubw", IX86_BUILTIN_PHSUBW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31501 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubdv4si3, "__builtin_ia32_phsubd128", IX86_BUILTIN_PHSUBD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
31502 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubdv2si3, "__builtin_ia32_phsubd", IX86_BUILTIN_PHSUBD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
31503 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubswv8hi3, "__builtin_ia32_phsubsw128", IX86_BUILTIN_PHSUBSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31504 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubswv4hi3, "__builtin_ia32_phsubsw", IX86_BUILTIN_PHSUBSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31505 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmaddubsw128, "__builtin_ia32_pmaddubsw128", IX86_BUILTIN_PMADDUBSW128, UNKNOWN, (int) V8HI_FTYPE_V16QI_V16QI },
31506 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmaddubsw, "__builtin_ia32_pmaddubsw", IX86_BUILTIN_PMADDUBSW, UNKNOWN, (int) V4HI_FTYPE_V8QI_V8QI },
31507 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmulhrswv8hi3, "__builtin_ia32_pmulhrsw128", IX86_BUILTIN_PMULHRSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31508 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmulhrswv4hi3, "__builtin_ia32_pmulhrsw", IX86_BUILTIN_PMULHRSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31509 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pshufbv16qi3, "__builtin_ia32_pshufb128", IX86_BUILTIN_PSHUFB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31510 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pshufbv8qi3, "__builtin_ia32_pshufb", IX86_BUILTIN_PSHUFB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
31511 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv16qi3, "__builtin_ia32_psignb128", IX86_BUILTIN_PSIGNB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31512 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv8qi3, "__builtin_ia32_psignb", IX86_BUILTIN_PSIGNB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
31513 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv8hi3, "__builtin_ia32_psignw128", IX86_BUILTIN_PSIGNW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31514 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv4hi3, "__builtin_ia32_psignw", IX86_BUILTIN_PSIGNW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
31515 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv4si3, "__builtin_ia32_psignd128", IX86_BUILTIN_PSIGND128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
31516 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv2si3, "__builtin_ia32_psignd", IX86_BUILTIN_PSIGND, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
31519 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_palignrti, "__builtin_ia32_palignr128", IX86_BUILTIN_PALIGNR128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_INT_CONVERT },
31520 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_palignrdi, "__builtin_ia32_palignr", IX86_BUILTIN_PALIGNR, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI_INT_CONVERT },
31523 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendpd, "__builtin_ia32_blendpd", IX86_BUILTIN_BLENDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
31524 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendps, "__builtin_ia32_blendps", IX86_BUILTIN_BLENDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
31525 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendvpd, "__builtin_ia32_blendvpd", IX86_BUILTIN_BLENDVPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF },
31526 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendvps, "__builtin_ia32_blendvps", IX86_BUILTIN_BLENDVPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF },
31527 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_dppd, "__builtin_ia32_dppd", IX86_BUILTIN_DPPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
31528 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_dpps, "__builtin_ia32_dpps", IX86_BUILTIN_DPPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
31529 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_insertps, "__builtin_ia32_insertps128", IX86_BUILTIN_INSERTPS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
31530 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_mpsadbw, "__builtin_ia32_mpsadbw128", IX86_BUILTIN_MPSADBW128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_INT },
31531 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_pblendvb, "__builtin_ia32_pblendvb128", IX86_BUILTIN_PBLENDVB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI },
31532 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_pblendw, "__builtin_ia32_pblendw128", IX86_BUILTIN_PBLENDW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_INT },
31534 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv8qiv8hi2, "__builtin_ia32_pmovsxbw128", IX86_BUILTIN_PMOVSXBW128, UNKNOWN, (int) V8HI_FTYPE_V16QI },
31535 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv4qiv4si2, "__builtin_ia32_pmovsxbd128", IX86_BUILTIN_PMOVSXBD128, UNKNOWN, (int) V4SI_FTYPE_V16QI },
31536 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv2qiv2di2, "__builtin_ia32_pmovsxbq128", IX86_BUILTIN_PMOVSXBQ128, UNKNOWN, (int) V2DI_FTYPE_V16QI },
31537 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv4hiv4si2, "__builtin_ia32_pmovsxwd128", IX86_BUILTIN_PMOVSXWD128, UNKNOWN, (int) V4SI_FTYPE_V8HI },
31538 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv2hiv2di2, "__builtin_ia32_pmovsxwq128", IX86_BUILTIN_PMOVSXWQ128, UNKNOWN, (int) V2DI_FTYPE_V8HI },
31539 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv2siv2di2, "__builtin_ia32_pmovsxdq128", IX86_BUILTIN_PMOVSXDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI },
31540 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv8qiv8hi2, "__builtin_ia32_pmovzxbw128", IX86_BUILTIN_PMOVZXBW128, UNKNOWN, (int) V8HI_FTYPE_V16QI },
31541 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv4qiv4si2, "__builtin_ia32_pmovzxbd128", IX86_BUILTIN_PMOVZXBD128, UNKNOWN, (int) V4SI_FTYPE_V16QI },
31542 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv2qiv2di2, "__builtin_ia32_pmovzxbq128", IX86_BUILTIN_PMOVZXBQ128, UNKNOWN, (int) V2DI_FTYPE_V16QI },
31543 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv4hiv4si2, "__builtin_ia32_pmovzxwd128", IX86_BUILTIN_PMOVZXWD128, UNKNOWN, (int) V4SI_FTYPE_V8HI },
31544 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv2hiv2di2, "__builtin_ia32_pmovzxwq128", IX86_BUILTIN_PMOVZXWQ128, UNKNOWN, (int) V2DI_FTYPE_V8HI },
31545 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv2siv2di2, "__builtin_ia32_pmovzxdq128", IX86_BUILTIN_PMOVZXDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI },
31546 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_phminposuw, "__builtin_ia32_phminposuw128", IX86_BUILTIN_PHMINPOSUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI },
31548 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_packusdw, "__builtin_ia32_packusdw128", IX86_BUILTIN_PACKUSDW128, UNKNOWN, (int) V8HI_FTYPE_V4SI_V4SI },
31549 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_eqv2di3, "__builtin_ia32_pcmpeqq", IX86_BUILTIN_PCMPEQQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
31550 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_smaxv16qi3, "__builtin_ia32_pmaxsb128", IX86_BUILTIN_PMAXSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31551 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_smaxv4si3, "__builtin_ia32_pmaxsd128", IX86_BUILTIN_PMAXSD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
31552 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_umaxv4si3, "__builtin_ia32_pmaxud128", IX86_BUILTIN_PMAXUD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
31553 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_umaxv8hi3, "__builtin_ia32_pmaxuw128", IX86_BUILTIN_PMAXUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31554 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sminv16qi3, "__builtin_ia32_pminsb128", IX86_BUILTIN_PMINSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
31555 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sminv4si3, "__builtin_ia32_pminsd128", IX86_BUILTIN_PMINSD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
31556 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_uminv4si3, "__builtin_ia32_pminud128", IX86_BUILTIN_PMINUD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
31557 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_uminv8hi3, "__builtin_ia32_pminuw128", IX86_BUILTIN_PMINUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
31558 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_mulv2siv2di3, "__builtin_ia32_pmuldq128", IX86_BUILTIN_PMULDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI_V4SI },
31559 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_mulv4si3, "__builtin_ia32_pmulld128", IX86_BUILTIN_PMULLD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
31562 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_roundpd", IX86_BUILTIN_ROUNDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_INT },
31563 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_roundps", IX86_BUILTIN_ROUNDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_INT },
31564 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundsd, "__builtin_ia32_roundsd", IX86_BUILTIN_ROUNDSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
31565 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundss, "__builtin_ia32_roundss", IX86_BUILTIN_ROUNDSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
31567 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_floorpd", IX86_BUILTIN_FLOORPD, (enum rtx_code) ROUND_FLOOR, (int) V2DF_FTYPE_V2DF_ROUND },
31568 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_ceilpd", IX86_BUILTIN_CEILPD, (enum rtx_code) ROUND_CEIL, (int) V2DF_FTYPE_V2DF_ROUND },
31569 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_truncpd", IX86_BUILTIN_TRUNCPD, (enum rtx_code) ROUND_TRUNC, (int) V2DF_FTYPE_V2DF_ROUND },
31570 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_rintpd", IX86_BUILTIN_RINTPD, (enum rtx_code) ROUND_MXCSR, (int) V2DF_FTYPE_V2DF_ROUND },
31572 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd_vec_pack_sfix, "__builtin_ia32_floorpd_vec_pack_sfix", IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX, (enum rtx_code) ROUND_FLOOR, (int) V4SI_FTYPE_V2DF_V2DF_ROUND },
31573 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd_vec_pack_sfix, "__builtin_ia32_ceilpd_vec_pack_sfix", IX86_BUILTIN_CEILPD_VEC_PACK_SFIX, (enum rtx_code) ROUND_CEIL, (int) V4SI_FTYPE_V2DF_V2DF_ROUND },
31575 { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv2df2, "__builtin_ia32_roundpd_az", IX86_BUILTIN_ROUNDPD_AZ, UNKNOWN, (int) V2DF_FTYPE_V2DF },
31576 { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv2df2_vec_pack_sfix, "__builtin_ia32_roundpd_az_vec_pack_sfix", IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX, UNKNOWN, (int) V4SI_FTYPE_V2DF_V2DF },
31578 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_floorps", IX86_BUILTIN_FLOORPS, (enum rtx_code) ROUND_FLOOR, (int) V4SF_FTYPE_V4SF_ROUND },
31579 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_ceilps", IX86_BUILTIN_CEILPS, (enum rtx_code) ROUND_CEIL, (int) V4SF_FTYPE_V4SF_ROUND },
31580 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_truncps", IX86_BUILTIN_TRUNCPS, (enum rtx_code) ROUND_TRUNC, (int) V4SF_FTYPE_V4SF_ROUND },
31581 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_rintps", IX86_BUILTIN_RINTPS, (enum rtx_code) ROUND_MXCSR, (int) V4SF_FTYPE_V4SF_ROUND },
31583 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps_sfix, "__builtin_ia32_floorps_sfix", IX86_BUILTIN_FLOORPS_SFIX, (enum rtx_code) ROUND_FLOOR, (int) V4SI_FTYPE_V4SF_ROUND },
31584 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps_sfix, "__builtin_ia32_ceilps_sfix", IX86_BUILTIN_CEILPS_SFIX, (enum rtx_code) ROUND_CEIL, (int) V4SI_FTYPE_V4SF_ROUND },
31586 { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv4sf2, "__builtin_ia32_roundps_az", IX86_BUILTIN_ROUNDPS_AZ, UNKNOWN, (int) V4SF_FTYPE_V4SF },
31587 { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv4sf2_sfix, "__builtin_ia32_roundps_az_sfix", IX86_BUILTIN_ROUNDPS_AZ_SFIX, UNKNOWN, (int) V4SI_FTYPE_V4SF },
31589 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_ptest, "__builtin_ia32_ptestz128", IX86_BUILTIN_PTESTZ, EQ, (int) INT_FTYPE_V2DI_V2DI_PTEST },
31590 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_ptest, "__builtin_ia32_ptestc128", IX86_BUILTIN_PTESTC, LTU, (int) INT_FTYPE_V2DI_V2DI_PTEST },
31591 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_ptest, "__builtin_ia32_ptestnzc128", IX86_BUILTIN_PTESTNZC, GTU, (int) INT_FTYPE_V2DI_V2DI_PTEST },
31594 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_gtv2di3, "__builtin_ia32_pcmpgtq", IX86_BUILTIN_PCMPGTQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
31595 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32, CODE_FOR_sse4_2_crc32qi, "__builtin_ia32_crc32qi", IX86_BUILTIN_CRC32QI, UNKNOWN, (int) UINT_FTYPE_UINT_UCHAR },
31596 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32, CODE_FOR_sse4_2_crc32hi, "__builtin_ia32_crc32hi", IX86_BUILTIN_CRC32HI, UNKNOWN, (int) UINT_FTYPE_UINT_USHORT },
31597 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32, CODE_FOR_sse4_2_crc32si, "__builtin_ia32_crc32si", IX86_BUILTIN_CRC32SI, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
31598 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse4_2_crc32di, "__builtin_ia32_crc32di", IX86_BUILTIN_CRC32DI, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
31601 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_extrqi, "__builtin_ia32_extrqi", IX86_BUILTIN_EXTRQI, UNKNOWN, (int) V2DI_FTYPE_V2DI_UINT_UINT },
31602 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_extrq, "__builtin_ia32_extrq", IX86_BUILTIN_EXTRQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V16QI },
31603 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_insertqi, "__builtin_ia32_insertqi", IX86_BUILTIN_INSERTQI, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_UINT_UINT },
31604 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_insertq, "__builtin_ia32_insertq", IX86_BUILTIN_INSERTQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
31607 { OPTION_MASK_ISA_SSE2, CODE_FOR_aeskeygenassist, 0, IX86_BUILTIN_AESKEYGENASSIST128, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT },
31608 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesimc, 0, IX86_BUILTIN_AESIMC128, UNKNOWN, (int) V2DI_FTYPE_V2DI },
31610 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesenc, 0, IX86_BUILTIN_AESENC128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
31611 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesenclast, 0, IX86_BUILTIN_AESENCLAST128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
31612 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesdec, 0, IX86_BUILTIN_AESDEC128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
31613 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesdeclast, 0, IX86_BUILTIN_AESDECLAST128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
31616 { OPTION_MASK_ISA_SSE2, CODE_FOR_pclmulqdq, 0, IX86_BUILTIN_PCLMULQDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_INT },
31619 { OPTION_MASK_ISA_AVX, CODE_FOR_addv4df3, "__builtin_ia32_addpd256", IX86_BUILTIN_ADDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
31620 { OPTION_MASK_ISA_AVX, CODE_FOR_addv8sf3, "__builtin_ia32_addps256", IX86_BUILTIN_ADDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
31621 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_addsubv4df3, "__builtin_ia32_addsubpd256", IX86_BUILTIN_ADDSUBPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
31622 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_addsubv8sf3, "__builtin_ia32_addsubps256", IX86_BUILTIN_ADDSUBPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
31623 { OPTION_MASK_ISA_AVX, CODE_FOR_andv4df3, "__builtin_ia32_andpd256", IX86_BUILTIN_ANDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
31624 { OPTION_MASK_ISA_AVX, CODE_FOR_andv8sf3, "__builtin_ia32_andps256", IX86_BUILTIN_ANDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
31625 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_andnotv4df3, "__builtin_ia32_andnpd256", IX86_BUILTIN_ANDNPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
31626 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_andnotv8sf3, "__builtin_ia32_andnps256", IX86_BUILTIN_ANDNPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
31627 { OPTION_MASK_ISA_AVX, CODE_FOR_divv4df3, "__builtin_ia32_divpd256", IX86_BUILTIN_DIVPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
31628 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_divv8sf3, "__builtin_ia32_divps256", IX86_BUILTIN_DIVPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
31629 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_haddv4df3, "__builtin_ia32_haddpd256", IX86_BUILTIN_HADDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
31630 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_hsubv8sf3, "__builtin_ia32_hsubps256", IX86_BUILTIN_HSUBPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
31631 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_hsubv4df3, "__builtin_ia32_hsubpd256", IX86_BUILTIN_HSUBPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
31632 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_haddv8sf3, "__builtin_ia32_haddps256", IX86_BUILTIN_HADDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
31633 { OPTION_MASK_ISA_AVX, CODE_FOR_smaxv4df3, "__builtin_ia32_maxpd256", IX86_BUILTIN_MAXPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
31634 { OPTION_MASK_ISA_AVX, CODE_FOR_smaxv8sf3, "__builtin_ia32_maxps256", IX86_BUILTIN_MAXPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
31635 { OPTION_MASK_ISA_AVX, CODE_FOR_sminv4df3, "__builtin_ia32_minpd256", IX86_BUILTIN_MINPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
31636 { OPTION_MASK_ISA_AVX, CODE_FOR_sminv8sf3, "__builtin_ia32_minps256", IX86_BUILTIN_MINPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
31637 { OPTION_MASK_ISA_AVX, CODE_FOR_mulv4df3, "__builtin_ia32_mulpd256", IX86_BUILTIN_MULPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
31638 { OPTION_MASK_ISA_AVX, CODE_FOR_mulv8sf3, "__builtin_ia32_mulps256", IX86_BUILTIN_MULPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
31639 { OPTION_MASK_ISA_AVX, CODE_FOR_iorv4df3, "__builtin_ia32_orpd256", IX86_BUILTIN_ORPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
31640 { OPTION_MASK_ISA_AVX, CODE_FOR_iorv8sf3, "__builtin_ia32_orps256", IX86_BUILTIN_ORPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
31641 { OPTION_MASK_ISA_AVX, CODE_FOR_subv4df3, "__builtin_ia32_subpd256", IX86_BUILTIN_SUBPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
31642 { OPTION_MASK_ISA_AVX, CODE_FOR_subv8sf3, "__builtin_ia32_subps256", IX86_BUILTIN_SUBPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
31643 { OPTION_MASK_ISA_AVX, CODE_FOR_xorv4df3, "__builtin_ia32_xorpd256", IX86_BUILTIN_XORPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
31644 { OPTION_MASK_ISA_AVX, CODE_FOR_xorv8sf3, "__builtin_ia32_xorps256", IX86_BUILTIN_XORPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
31646 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv2df3, "__builtin_ia32_vpermilvarpd", IX86_BUILTIN_VPERMILVARPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DI },
31647 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv4sf3, "__builtin_ia32_vpermilvarps", IX86_BUILTIN_VPERMILVARPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SI },
31648 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv4df3, "__builtin_ia32_vpermilvarpd256", IX86_BUILTIN_VPERMILVARPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DI },
31649 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv8sf3, "__builtin_ia32_vpermilvarps256", IX86_BUILTIN_VPERMILVARPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SI },
31651 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendpd256, "__builtin_ia32_blendpd256", IX86_BUILTIN_BLENDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
31652 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendps256, "__builtin_ia32_blendps256", IX86_BUILTIN_BLENDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
31653 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendvpd256, "__builtin_ia32_blendvpd256", IX86_BUILTIN_BLENDVPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF },
31654 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendvps256, "__builtin_ia32_blendvps256", IX86_BUILTIN_BLENDVPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF },
31655 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_dpps256, "__builtin_ia32_dpps256", IX86_BUILTIN_DPPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
31656 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_shufpd256, "__builtin_ia32_shufpd256", IX86_BUILTIN_SHUFPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
31657 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_shufps256, "__builtin_ia32_shufps256", IX86_BUILTIN_SHUFPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
31658 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vmcmpv2df3, "__builtin_ia32_cmpsd", IX86_BUILTIN_CMPSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
31659 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vmcmpv4sf3, "__builtin_ia32_cmpss", IX86_BUILTIN_CMPSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
31660 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpv2df3, "__builtin_ia32_cmppd", IX86_BUILTIN_CMPPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
31661 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpv4sf3, "__builtin_ia32_cmpps", IX86_BUILTIN_CMPPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
31662 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpv4df3, "__builtin_ia32_cmppd256", IX86_BUILTIN_CMPPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
31663 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpv8sf3, "__builtin_ia32_cmpps256", IX86_BUILTIN_CMPPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
31664 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vextractf128v4df, "__builtin_ia32_vextractf128_pd256", IX86_BUILTIN_EXTRACTF128PD256, UNKNOWN, (int) V2DF_FTYPE_V4DF_INT },
31665 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vextractf128v8sf, "__builtin_ia32_vextractf128_ps256", IX86_BUILTIN_EXTRACTF128PS256, UNKNOWN, (int) V4SF_FTYPE_V8SF_INT },
31666 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vextractf128v8si, "__builtin_ia32_vextractf128_si256", IX86_BUILTIN_EXTRACTF128SI256, UNKNOWN, (int) V4SI_FTYPE_V8SI_INT },
31667 { OPTION_MASK_ISA_AVX, CODE_FOR_floatv4siv4df2, "__builtin_ia32_cvtdq2pd256", IX86_BUILTIN_CVTDQ2PD256, UNKNOWN, (int) V4DF_FTYPE_V4SI },
31668 { OPTION_MASK_ISA_AVX, CODE_FOR_floatv8siv8sf2, "__builtin_ia32_cvtdq2ps256", IX86_BUILTIN_CVTDQ2PS256, UNKNOWN, (int) V8SF_FTYPE_V8SI },
31669 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtpd2ps256, "__builtin_ia32_cvtpd2ps256", IX86_BUILTIN_CVTPD2PS256, UNKNOWN, (int) V4SF_FTYPE_V4DF },
31670 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_fix_notruncv8sfv8si, "__builtin_ia32_cvtps2dq256", IX86_BUILTIN_CVTPS2DQ256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
31671 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtps2pd256, "__builtin_ia32_cvtps2pd256", IX86_BUILTIN_CVTPS2PD256, UNKNOWN, (int) V4DF_FTYPE_V4SF },
31672 { OPTION_MASK_ISA_AVX, CODE_FOR_fix_truncv4dfv4si2, "__builtin_ia32_cvttpd2dq256", IX86_BUILTIN_CVTTPD2DQ256, UNKNOWN, (int) V4SI_FTYPE_V4DF },
31673 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtpd2dq256, "__builtin_ia32_cvtpd2dq256", IX86_BUILTIN_CVTPD2DQ256, UNKNOWN, (int) V4SI_FTYPE_V4DF },
31674 { OPTION_MASK_ISA_AVX, CODE_FOR_fix_truncv8sfv8si2, "__builtin_ia32_cvttps2dq256", IX86_BUILTIN_CVTTPS2DQ256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
31675 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vperm2f128v4df3, "__builtin_ia32_vperm2f128_pd256", IX86_BUILTIN_VPERM2F128PD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
31676 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vperm2f128v8sf3, "__builtin_ia32_vperm2f128_ps256", IX86_BUILTIN_VPERM2F128PS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
31677 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vperm2f128v8si3, "__builtin_ia32_vperm2f128_si256", IX86_BUILTIN_VPERM2F128SI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_INT },
31678 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv2df, "__builtin_ia32_vpermilpd", IX86_BUILTIN_VPERMILPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_INT },
31679 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv4sf, "__builtin_ia32_vpermilps", IX86_BUILTIN_VPERMILPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_INT },
31680 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv4df, "__builtin_ia32_vpermilpd256", IX86_BUILTIN_VPERMILPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT },
31681 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv8sf, "__builtin_ia32_vpermilps256", IX86_BUILTIN_VPERMILPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_INT },
31682 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vinsertf128v4df, "__builtin_ia32_vinsertf128_pd256", IX86_BUILTIN_VINSERTF128PD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V2DF_INT },
31683 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vinsertf128v8sf, "__builtin_ia32_vinsertf128_ps256", IX86_BUILTIN_VINSERTF128PS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V4SF_INT },
31684 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vinsertf128v8si, "__builtin_ia32_vinsertf128_si256", IX86_BUILTIN_VINSERTF128SI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_INT },
31686 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movshdup256, "__builtin_ia32_movshdup256", IX86_BUILTIN_MOVSHDUP256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
31687 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movsldup256, "__builtin_ia32_movsldup256", IX86_BUILTIN_MOVSLDUP256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
31688 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movddup256, "__builtin_ia32_movddup256", IX86_BUILTIN_MOVDDUP256, UNKNOWN, (int) V4DF_FTYPE_V4DF },
31690 { OPTION_MASK_ISA_AVX, CODE_FOR_sqrtv4df2, "__builtin_ia32_sqrtpd256", IX86_BUILTIN_SQRTPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF },
31691 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_sqrtv8sf2, "__builtin_ia32_sqrtps256", IX86_BUILTIN_SQRTPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
31692 { OPTION_MASK_ISA_AVX, CODE_FOR_sqrtv8sf2, "__builtin_ia32_sqrtps_nr256", IX86_BUILTIN_SQRTPS_NR256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
31693 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_rsqrtv8sf2, "__builtin_ia32_rsqrtps256", IX86_BUILTIN_RSQRTPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
31694 { OPTION_MASK_ISA_AVX, CODE_FOR_rsqrtv8sf2, "__builtin_ia32_rsqrtps_nr256", IX86_BUILTIN_RSQRTPS_NR256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
31696 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_rcpv8sf2, "__builtin_ia32_rcpps256", IX86_BUILTIN_RCPPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
31698 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_roundpd256", IX86_BUILTIN_ROUNDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT },
31699 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_roundps256", IX86_BUILTIN_ROUNDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_INT },
31701 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_floorpd256", IX86_BUILTIN_FLOORPD256, (enum rtx_code) ROUND_FLOOR, (int) V4DF_FTYPE_V4DF_ROUND },
31702 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_ceilpd256", IX86_BUILTIN_CEILPD256, (enum rtx_code) ROUND_CEIL, (int) V4DF_FTYPE_V4DF_ROUND },
31703 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_truncpd256", IX86_BUILTIN_TRUNCPD256, (enum rtx_code) ROUND_TRUNC, (int) V4DF_FTYPE_V4DF_ROUND },
31704 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_rintpd256", IX86_BUILTIN_RINTPD256, (enum rtx_code) ROUND_MXCSR, (int) V4DF_FTYPE_V4DF_ROUND },
31706 { OPTION_MASK_ISA_AVX, CODE_FOR_roundv4df2, "__builtin_ia32_roundpd_az256", IX86_BUILTIN_ROUNDPD_AZ256, UNKNOWN, (int) V4DF_FTYPE_V4DF },
31707 { OPTION_MASK_ISA_AVX, CODE_FOR_roundv4df2_vec_pack_sfix, "__builtin_ia32_roundpd_az_vec_pack_sfix256", IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX256, UNKNOWN, (int) V8SI_FTYPE_V4DF_V4DF },
31709 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd_vec_pack_sfix256, "__builtin_ia32_floorpd_vec_pack_sfix256", IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX256, (enum rtx_code) ROUND_FLOOR, (int) V8SI_FTYPE_V4DF_V4DF_ROUND },
31710 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd_vec_pack_sfix256, "__builtin_ia32_ceilpd_vec_pack_sfix256", IX86_BUILTIN_CEILPD_VEC_PACK_SFIX256, (enum rtx_code) ROUND_CEIL, (int) V8SI_FTYPE_V4DF_V4DF_ROUND },
31712 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_floorps256", IX86_BUILTIN_FLOORPS256, (enum rtx_code) ROUND_FLOOR, (int) V8SF_FTYPE_V8SF_ROUND },
31713 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_ceilps256", IX86_BUILTIN_CEILPS256, (enum rtx_code) ROUND_CEIL, (int) V8SF_FTYPE_V8SF_ROUND },
31714 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_truncps256", IX86_BUILTIN_TRUNCPS256, (enum rtx_code) ROUND_TRUNC, (int) V8SF_FTYPE_V8SF_ROUND },
31715 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_rintps256", IX86_BUILTIN_RINTPS256, (enum rtx_code) ROUND_MXCSR, (int) V8SF_FTYPE_V8SF_ROUND },
31717 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps_sfix256, "__builtin_ia32_floorps_sfix256", IX86_BUILTIN_FLOORPS_SFIX256, (enum rtx_code) ROUND_FLOOR, (int) V8SI_FTYPE_V8SF_ROUND },
31718 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps_sfix256, "__builtin_ia32_ceilps_sfix256", IX86_BUILTIN_CEILPS_SFIX256, (enum rtx_code) ROUND_CEIL, (int) V8SI_FTYPE_V8SF_ROUND },
31720 { OPTION_MASK_ISA_AVX, CODE_FOR_roundv8sf2, "__builtin_ia32_roundps_az256", IX86_BUILTIN_ROUNDPS_AZ256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
31721 { OPTION_MASK_ISA_AVX, CODE_FOR_roundv8sf2_sfix, "__builtin_ia32_roundps_az_sfix256", IX86_BUILTIN_ROUNDPS_AZ_SFIX256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
31723 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpckhpd256, "__builtin_ia32_unpckhpd256", IX86_BUILTIN_UNPCKHPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
31724 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpcklpd256, "__builtin_ia32_unpcklpd256", IX86_BUILTIN_UNPCKLPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
31725 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpckhps256, "__builtin_ia32_unpckhps256", IX86_BUILTIN_UNPCKHPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
31726 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpcklps256, "__builtin_ia32_unpcklps256", IX86_BUILTIN_UNPCKLPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
31728 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_si256_si, "__builtin_ia32_si256_si", IX86_BUILTIN_SI256_SI, UNKNOWN, (int) V8SI_FTYPE_V4SI },
31729 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ps256_ps, "__builtin_ia32_ps256_ps", IX86_BUILTIN_PS256_PS, UNKNOWN, (int) V8SF_FTYPE_V4SF },
31730 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_pd256_pd, "__builtin_ia32_pd256_pd", IX86_BUILTIN_PD256_PD, UNKNOWN, (int) V4DF_FTYPE_V2DF },
31731 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_extract_lo_v8si, "__builtin_ia32_si_si256", IX86_BUILTIN_SI_SI256, UNKNOWN, (int) V4SI_FTYPE_V8SI },
31732 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_extract_lo_v8sf, "__builtin_ia32_ps_ps256", IX86_BUILTIN_PS_PS256, UNKNOWN, (int) V4SF_FTYPE_V8SF },
31733 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_extract_lo_v4df, "__builtin_ia32_pd_pd256", IX86_BUILTIN_PD_PD256, UNKNOWN, (int) V2DF_FTYPE_V4DF },
31735 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd, "__builtin_ia32_vtestzpd", IX86_BUILTIN_VTESTZPD, EQ, (int) INT_FTYPE_V2DF_V2DF_PTEST },
31736 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd, "__builtin_ia32_vtestcpd", IX86_BUILTIN_VTESTCPD, LTU, (int) INT_FTYPE_V2DF_V2DF_PTEST },
31737 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd, "__builtin_ia32_vtestnzcpd", IX86_BUILTIN_VTESTNZCPD, GTU, (int) INT_FTYPE_V2DF_V2DF_PTEST },
31738 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps, "__builtin_ia32_vtestzps", IX86_BUILTIN_VTESTZPS, EQ, (int) INT_FTYPE_V4SF_V4SF_PTEST },
31739 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps, "__builtin_ia32_vtestcps", IX86_BUILTIN_VTESTCPS, LTU, (int) INT_FTYPE_V4SF_V4SF_PTEST },
31740 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps, "__builtin_ia32_vtestnzcps", IX86_BUILTIN_VTESTNZCPS, GTU, (int) INT_FTYPE_V4SF_V4SF_PTEST },
31741 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd256, "__builtin_ia32_vtestzpd256", IX86_BUILTIN_VTESTZPD256, EQ, (int) INT_FTYPE_V4DF_V4DF_PTEST },
31742 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd256, "__builtin_ia32_vtestcpd256", IX86_BUILTIN_VTESTCPD256, LTU, (int) INT_FTYPE_V4DF_V4DF_PTEST },
31743 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd256, "__builtin_ia32_vtestnzcpd256", IX86_BUILTIN_VTESTNZCPD256, GTU, (int) INT_FTYPE_V4DF_V4DF_PTEST },
31744 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps256, "__builtin_ia32_vtestzps256", IX86_BUILTIN_VTESTZPS256, EQ, (int) INT_FTYPE_V8SF_V8SF_PTEST },
31745 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps256, "__builtin_ia32_vtestcps256", IX86_BUILTIN_VTESTCPS256, LTU, (int) INT_FTYPE_V8SF_V8SF_PTEST },
31746 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps256, "__builtin_ia32_vtestnzcps256", IX86_BUILTIN_VTESTNZCPS256, GTU, (int) INT_FTYPE_V8SF_V8SF_PTEST },
31747 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ptest256, "__builtin_ia32_ptestz256", IX86_BUILTIN_PTESTZ256, EQ, (int) INT_FTYPE_V4DI_V4DI_PTEST },
31748 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ptest256, "__builtin_ia32_ptestc256", IX86_BUILTIN_PTESTC256, LTU, (int) INT_FTYPE_V4DI_V4DI_PTEST },
31749 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ptest256, "__builtin_ia32_ptestnzc256", IX86_BUILTIN_PTESTNZC256, GTU, (int) INT_FTYPE_V4DI_V4DI_PTEST },
31751 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movmskpd256, "__builtin_ia32_movmskpd256", IX86_BUILTIN_MOVMSKPD256, UNKNOWN, (int) INT_FTYPE_V4DF },
31752 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movmskps256, "__builtin_ia32_movmskps256", IX86_BUILTIN_MOVMSKPS256, UNKNOWN, (int) INT_FTYPE_V8SF },
31754 { OPTION_MASK_ISA_AVX, CODE_FOR_copysignv8sf3, "__builtin_ia32_copysignps256", IX86_BUILTIN_CPYSGNPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
31755 { OPTION_MASK_ISA_AVX, CODE_FOR_copysignv4df3, "__builtin_ia32_copysignpd256", IX86_BUILTIN_CPYSGNPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
31757 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_pack_sfix_v4df, "__builtin_ia32_vec_pack_sfix256 ", IX86_BUILTIN_VEC_PACK_SFIX256, UNKNOWN, (int) V8SI_FTYPE_V4DF_V4DF },
31760 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_mpsadbw, "__builtin_ia32_mpsadbw256", IX86_BUILTIN_MPSADBW256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_INT },
31761 { OPTION_MASK_ISA_AVX2, CODE_FOR_absv32qi2, "__builtin_ia32_pabsb256", IX86_BUILTIN_PABSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI },
31762 { OPTION_MASK_ISA_AVX2, CODE_FOR_absv16hi2, "__builtin_ia32_pabsw256", IX86_BUILTIN_PABSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI },
31763 { OPTION_MASK_ISA_AVX2, CODE_FOR_absv8si2, "__builtin_ia32_pabsd256", IX86_BUILTIN_PABSD256, UNKNOWN, (int) V8SI_FTYPE_V8SI },
31764 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_packssdw, "__builtin_ia32_packssdw256", IX86_BUILTIN_PACKSSDW256, UNKNOWN, (int) V16HI_FTYPE_V8SI_V8SI },
31765 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_packsswb, "__builtin_ia32_packsswb256", IX86_BUILTIN_PACKSSWB256, UNKNOWN, (int) V32QI_FTYPE_V16HI_V16HI },
31766 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_packusdw, "__builtin_ia32_packusdw256", IX86_BUILTIN_PACKUSDW256, UNKNOWN, (int) V16HI_FTYPE_V8SI_V8SI },
31767 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_packuswb, "__builtin_ia32_packuswb256", IX86_BUILTIN_PACKUSWB256, UNKNOWN, (int) V32QI_FTYPE_V16HI_V16HI },
31768 { OPTION_MASK_ISA_AVX2, CODE_FOR_addv32qi3, "__builtin_ia32_paddb256", IX86_BUILTIN_PADDB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31769 { OPTION_MASK_ISA_AVX2, CODE_FOR_addv16hi3, "__builtin_ia32_paddw256", IX86_BUILTIN_PADDW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31770 { OPTION_MASK_ISA_AVX2, CODE_FOR_addv8si3, "__builtin_ia32_paddd256", IX86_BUILTIN_PADDD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
31771 { OPTION_MASK_ISA_AVX2, CODE_FOR_addv4di3, "__builtin_ia32_paddq256", IX86_BUILTIN_PADDQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
31772 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ssaddv32qi3, "__builtin_ia32_paddsb256", IX86_BUILTIN_PADDSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31773 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ssaddv16hi3, "__builtin_ia32_paddsw256", IX86_BUILTIN_PADDSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31774 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_usaddv32qi3, "__builtin_ia32_paddusb256", IX86_BUILTIN_PADDUSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31775 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_usaddv16hi3, "__builtin_ia32_paddusw256", IX86_BUILTIN_PADDUSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31776 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_palignrv2ti, "__builtin_ia32_palignr256", IX86_BUILTIN_PALIGNR256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_INT_CONVERT },
31777 { OPTION_MASK_ISA_AVX2, CODE_FOR_andv4di3, "__builtin_ia32_andsi256", IX86_BUILTIN_AND256I, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
31778 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_andnotv4di3, "__builtin_ia32_andnotsi256", IX86_BUILTIN_ANDNOT256I, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
31779 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_uavgv32qi3, "__builtin_ia32_pavgb256", IX86_BUILTIN_PAVGB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31780 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_uavgv16hi3, "__builtin_ia32_pavgw256", IX86_BUILTIN_PAVGW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31781 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pblendvb, "__builtin_ia32_pblendvb256", IX86_BUILTIN_PBLENDVB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI },
31782 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pblendw, "__builtin_ia32_pblendw256", IX86_BUILTIN_PBLENDVW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_INT },
31783 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_eqv32qi3, "__builtin_ia32_pcmpeqb256", IX86_BUILTIN_PCMPEQB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31784 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_eqv16hi3, "__builtin_ia32_pcmpeqw256", IX86_BUILTIN_PCMPEQW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31785 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_eqv8si3, "__builtin_ia32_pcmpeqd256", IX86_BUILTIN_PCMPEQD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
31786 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_eqv4di3, "__builtin_ia32_pcmpeqq256", IX86_BUILTIN_PCMPEQQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
31787 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_gtv32qi3, "__builtin_ia32_pcmpgtb256", IX86_BUILTIN_PCMPGTB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31788 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_gtv16hi3, "__builtin_ia32_pcmpgtw256", IX86_BUILTIN_PCMPGTW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31789 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_gtv8si3, "__builtin_ia32_pcmpgtd256", IX86_BUILTIN_PCMPGTD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
31790 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_gtv4di3, "__builtin_ia32_pcmpgtq256", IX86_BUILTIN_PCMPGTQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
31791 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phaddwv16hi3, "__builtin_ia32_phaddw256", IX86_BUILTIN_PHADDW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31792 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phadddv8si3, "__builtin_ia32_phaddd256", IX86_BUILTIN_PHADDD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
31793 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phaddswv16hi3, "__builtin_ia32_phaddsw256", IX86_BUILTIN_PHADDSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31794 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phsubwv16hi3, "__builtin_ia32_phsubw256", IX86_BUILTIN_PHSUBW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31795 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phsubdv8si3, "__builtin_ia32_phsubd256", IX86_BUILTIN_PHSUBD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
31796 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phsubswv16hi3, "__builtin_ia32_phsubsw256", IX86_BUILTIN_PHSUBSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31797 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pmaddubsw256, "__builtin_ia32_pmaddubsw256", IX86_BUILTIN_PMADDUBSW256, UNKNOWN, (int) V16HI_FTYPE_V32QI_V32QI },
31798 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pmaddwd, "__builtin_ia32_pmaddwd256", IX86_BUILTIN_PMADDWD256, UNKNOWN, (int) V8SI_FTYPE_V16HI_V16HI },
31799 { OPTION_MASK_ISA_AVX2, CODE_FOR_smaxv32qi3, "__builtin_ia32_pmaxsb256", IX86_BUILTIN_PMAXSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31800 { OPTION_MASK_ISA_AVX2, CODE_FOR_smaxv16hi3, "__builtin_ia32_pmaxsw256", IX86_BUILTIN_PMAXSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31801 { OPTION_MASK_ISA_AVX2, CODE_FOR_smaxv8si3 , "__builtin_ia32_pmaxsd256", IX86_BUILTIN_PMAXSD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
31802 { OPTION_MASK_ISA_AVX2, CODE_FOR_umaxv32qi3, "__builtin_ia32_pmaxub256", IX86_BUILTIN_PMAXUB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31803 { OPTION_MASK_ISA_AVX2, CODE_FOR_umaxv16hi3, "__builtin_ia32_pmaxuw256", IX86_BUILTIN_PMAXUW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31804 { OPTION_MASK_ISA_AVX2, CODE_FOR_umaxv8si3 , "__builtin_ia32_pmaxud256", IX86_BUILTIN_PMAXUD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
31805 { OPTION_MASK_ISA_AVX2, CODE_FOR_sminv32qi3, "__builtin_ia32_pminsb256", IX86_BUILTIN_PMINSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31806 { OPTION_MASK_ISA_AVX2, CODE_FOR_sminv16hi3, "__builtin_ia32_pminsw256", IX86_BUILTIN_PMINSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31807 { OPTION_MASK_ISA_AVX2, CODE_FOR_sminv8si3 , "__builtin_ia32_pminsd256", IX86_BUILTIN_PMINSD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
31808 { OPTION_MASK_ISA_AVX2, CODE_FOR_uminv32qi3, "__builtin_ia32_pminub256", IX86_BUILTIN_PMINUB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31809 { OPTION_MASK_ISA_AVX2, CODE_FOR_uminv16hi3, "__builtin_ia32_pminuw256", IX86_BUILTIN_PMINUW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31810 { OPTION_MASK_ISA_AVX2, CODE_FOR_uminv8si3 , "__builtin_ia32_pminud256", IX86_BUILTIN_PMINUD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
31811 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pmovmskb, "__builtin_ia32_pmovmskb256", IX86_BUILTIN_PMOVMSKB256, UNKNOWN, (int) INT_FTYPE_V32QI },
31812 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv16qiv16hi2, "__builtin_ia32_pmovsxbw256", IX86_BUILTIN_PMOVSXBW256, UNKNOWN, (int) V16HI_FTYPE_V16QI },
31813 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv8qiv8si2 , "__builtin_ia32_pmovsxbd256", IX86_BUILTIN_PMOVSXBD256, UNKNOWN, (int) V8SI_FTYPE_V16QI },
31814 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv4qiv4di2 , "__builtin_ia32_pmovsxbq256", IX86_BUILTIN_PMOVSXBQ256, UNKNOWN, (int) V4DI_FTYPE_V16QI },
31815 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv8hiv8si2 , "__builtin_ia32_pmovsxwd256", IX86_BUILTIN_PMOVSXWD256, UNKNOWN, (int) V8SI_FTYPE_V8HI },
31816 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv4hiv4di2 , "__builtin_ia32_pmovsxwq256", IX86_BUILTIN_PMOVSXWQ256, UNKNOWN, (int) V4DI_FTYPE_V8HI },
31817 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv4siv4di2 , "__builtin_ia32_pmovsxdq256", IX86_BUILTIN_PMOVSXDQ256, UNKNOWN, (int) V4DI_FTYPE_V4SI },
31818 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv16qiv16hi2, "__builtin_ia32_pmovzxbw256", IX86_BUILTIN_PMOVZXBW256, UNKNOWN, (int) V16HI_FTYPE_V16QI },
31819 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv8qiv8si2 , "__builtin_ia32_pmovzxbd256", IX86_BUILTIN_PMOVZXBD256, UNKNOWN, (int) V8SI_FTYPE_V16QI },
31820 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv4qiv4di2 , "__builtin_ia32_pmovzxbq256", IX86_BUILTIN_PMOVZXBQ256, UNKNOWN, (int) V4DI_FTYPE_V16QI },
31821 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv8hiv8si2 , "__builtin_ia32_pmovzxwd256", IX86_BUILTIN_PMOVZXWD256, UNKNOWN, (int) V8SI_FTYPE_V8HI },
31822 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv4hiv4di2 , "__builtin_ia32_pmovzxwq256", IX86_BUILTIN_PMOVZXWQ256, UNKNOWN, (int) V4DI_FTYPE_V8HI },
31823 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv4siv4di2 , "__builtin_ia32_pmovzxdq256", IX86_BUILTIN_PMOVZXDQ256, UNKNOWN, (int) V4DI_FTYPE_V4SI },
31824 { OPTION_MASK_ISA_AVX2, CODE_FOR_vec_widen_smult_even_v8si, "__builtin_ia32_pmuldq256", IX86_BUILTIN_PMULDQ256, UNKNOWN, (int) V4DI_FTYPE_V8SI_V8SI },
31825 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pmulhrswv16hi3 , "__builtin_ia32_pmulhrsw256", IX86_BUILTIN_PMULHRSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31826 { OPTION_MASK_ISA_AVX2, CODE_FOR_umulv16hi3_highpart, "__builtin_ia32_pmulhuw256" , IX86_BUILTIN_PMULHUW256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31827 { OPTION_MASK_ISA_AVX2, CODE_FOR_smulv16hi3_highpart, "__builtin_ia32_pmulhw256" , IX86_BUILTIN_PMULHW256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31828 { OPTION_MASK_ISA_AVX2, CODE_FOR_mulv16hi3, "__builtin_ia32_pmullw256" , IX86_BUILTIN_PMULLW256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31829 { OPTION_MASK_ISA_AVX2, CODE_FOR_mulv8si3, "__builtin_ia32_pmulld256" , IX86_BUILTIN_PMULLD256 , UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
31830 { OPTION_MASK_ISA_AVX2, CODE_FOR_vec_widen_umult_even_v8si, "__builtin_ia32_pmuludq256", IX86_BUILTIN_PMULUDQ256, UNKNOWN, (int) V4DI_FTYPE_V8SI_V8SI },
31831 { OPTION_MASK_ISA_AVX2, CODE_FOR_iorv4di3, "__builtin_ia32_por256", IX86_BUILTIN_POR256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
31832 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psadbw, "__builtin_ia32_psadbw256", IX86_BUILTIN_PSADBW256, UNKNOWN, (int) V16HI_FTYPE_V32QI_V32QI },
31833 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pshufbv32qi3, "__builtin_ia32_pshufb256", IX86_BUILTIN_PSHUFB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31834 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pshufdv3, "__builtin_ia32_pshufd256", IX86_BUILTIN_PSHUFD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_INT },
31835 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pshufhwv3, "__builtin_ia32_pshufhw256", IX86_BUILTIN_PSHUFHW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_INT },
31836 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pshuflwv3, "__builtin_ia32_pshuflw256", IX86_BUILTIN_PSHUFLW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_INT },
31837 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psignv32qi3, "__builtin_ia32_psignb256", IX86_BUILTIN_PSIGNB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31838 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psignv16hi3, "__builtin_ia32_psignw256", IX86_BUILTIN_PSIGNW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31839 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psignv8si3 , "__builtin_ia32_psignd256", IX86_BUILTIN_PSIGND256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
31840 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlv2ti3, "__builtin_ia32_pslldqi256", IX86_BUILTIN_PSLLDQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_CONVERT },
31841 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv16hi3, "__builtin_ia32_psllwi256", IX86_BUILTIN_PSLLWI256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_SI_COUNT },
31842 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv16hi3, "__builtin_ia32_psllw256", IX86_BUILTIN_PSLLW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_COUNT },
31843 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv8si3, "__builtin_ia32_pslldi256", IX86_BUILTIN_PSLLDI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_SI_COUNT },
31844 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv8si3, "__builtin_ia32_pslld256", IX86_BUILTIN_PSLLD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_COUNT },
31845 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv4di3, "__builtin_ia32_psllqi256", IX86_BUILTIN_PSLLQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_COUNT },
31846 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv4di3, "__builtin_ia32_psllq256", IX86_BUILTIN_PSLLQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_COUNT },
31847 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv16hi3, "__builtin_ia32_psrawi256", IX86_BUILTIN_PSRAWI256, UNKNOWN, (int) V16HI_FTYPE_V16HI_SI_COUNT },
31848 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv16hi3, "__builtin_ia32_psraw256", IX86_BUILTIN_PSRAW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_COUNT },
31849 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv8si3, "__builtin_ia32_psradi256", IX86_BUILTIN_PSRADI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_SI_COUNT },
31850 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv8si3, "__builtin_ia32_psrad256", IX86_BUILTIN_PSRAD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_COUNT },
31851 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrv2ti3, "__builtin_ia32_psrldqi256", IX86_BUILTIN_PSRLDQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_CONVERT },
31852 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv16hi3, "__builtin_ia32_psrlwi256", IX86_BUILTIN_PSRLWI256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_SI_COUNT },
31853 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv16hi3, "__builtin_ia32_psrlw256", IX86_BUILTIN_PSRLW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_COUNT },
31854 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv8si3, "__builtin_ia32_psrldi256", IX86_BUILTIN_PSRLDI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_SI_COUNT },
31855 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv8si3, "__builtin_ia32_psrld256", IX86_BUILTIN_PSRLD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_COUNT },
31856 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv4di3, "__builtin_ia32_psrlqi256", IX86_BUILTIN_PSRLQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_COUNT },
31857 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv4di3, "__builtin_ia32_psrlq256", IX86_BUILTIN_PSRLQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_COUNT },
31858 { OPTION_MASK_ISA_AVX2, CODE_FOR_subv32qi3, "__builtin_ia32_psubb256", IX86_BUILTIN_PSUBB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31859 { OPTION_MASK_ISA_AVX2, CODE_FOR_subv16hi3, "__builtin_ia32_psubw256", IX86_BUILTIN_PSUBW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31860 { OPTION_MASK_ISA_AVX2, CODE_FOR_subv8si3, "__builtin_ia32_psubd256", IX86_BUILTIN_PSUBD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
31861 { OPTION_MASK_ISA_AVX2, CODE_FOR_subv4di3, "__builtin_ia32_psubq256", IX86_BUILTIN_PSUBQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
31862 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sssubv32qi3, "__builtin_ia32_psubsb256", IX86_BUILTIN_PSUBSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31863 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sssubv16hi3, "__builtin_ia32_psubsw256", IX86_BUILTIN_PSUBSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31864 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ussubv32qi3, "__builtin_ia32_psubusb256", IX86_BUILTIN_PSUBUSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31865 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ussubv16hi3, "__builtin_ia32_psubusw256", IX86_BUILTIN_PSUBUSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31866 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_highv32qi, "__builtin_ia32_punpckhbw256", IX86_BUILTIN_PUNPCKHBW256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31867 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_highv16hi, "__builtin_ia32_punpckhwd256", IX86_BUILTIN_PUNPCKHWD256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31868 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_highv8si, "__builtin_ia32_punpckhdq256", IX86_BUILTIN_PUNPCKHDQ256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
31869 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_highv4di, "__builtin_ia32_punpckhqdq256", IX86_BUILTIN_PUNPCKHQDQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
31870 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_lowv32qi, "__builtin_ia32_punpcklbw256", IX86_BUILTIN_PUNPCKLBW256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
31871 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_lowv16hi, "__builtin_ia32_punpcklwd256", IX86_BUILTIN_PUNPCKLWD256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
31872 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_lowv8si, "__builtin_ia32_punpckldq256", IX86_BUILTIN_PUNPCKLDQ256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
31873 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_lowv4di, "__builtin_ia32_punpcklqdq256", IX86_BUILTIN_PUNPCKLQDQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
31874 { OPTION_MASK_ISA_AVX2, CODE_FOR_xorv4di3, "__builtin_ia32_pxor256", IX86_BUILTIN_PXOR256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
31875 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_vec_dupv4sf, "__builtin_ia32_vbroadcastss_ps", IX86_BUILTIN_VBROADCASTSS_PS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
31876 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_vec_dupv8sf, "__builtin_ia32_vbroadcastss_ps256", IX86_BUILTIN_VBROADCASTSS_PS256, UNKNOWN, (int) V8SF_FTYPE_V4SF },
31877 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_vec_dupv4df, "__builtin_ia32_vbroadcastsd_pd256", IX86_BUILTIN_VBROADCASTSD_PD256, UNKNOWN, (int) V4DF_FTYPE_V2DF },
31878 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_vbroadcasti128_v4di, "__builtin_ia32_vbroadcastsi256", IX86_BUILTIN_VBROADCASTSI256, UNKNOWN, (int) V4DI_FTYPE_V2DI },
31879 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pblenddv4si, "__builtin_ia32_pblendd128", IX86_BUILTIN_PBLENDD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_INT },
31880 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pblenddv8si, "__builtin_ia32_pblendd256", IX86_BUILTIN_PBLENDD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_INT },
31881 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv32qi, "__builtin_ia32_pbroadcastb256", IX86_BUILTIN_PBROADCASTB256, UNKNOWN, (int) V32QI_FTYPE_V16QI },
31882 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv16hi, "__builtin_ia32_pbroadcastw256", IX86_BUILTIN_PBROADCASTW256, UNKNOWN, (int) V16HI_FTYPE_V8HI },
31883 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv8si, "__builtin_ia32_pbroadcastd256", IX86_BUILTIN_PBROADCASTD256, UNKNOWN, (int) V8SI_FTYPE_V4SI },
31884 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv4di, "__builtin_ia32_pbroadcastq256", IX86_BUILTIN_PBROADCASTQ256, UNKNOWN, (int) V4DI_FTYPE_V2DI },
31885 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv16qi, "__builtin_ia32_pbroadcastb128", IX86_BUILTIN_PBROADCASTB128, UNKNOWN, (int) V16QI_FTYPE_V16QI },
31886 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv8hi, "__builtin_ia32_pbroadcastw128", IX86_BUILTIN_PBROADCASTW128, UNKNOWN, (int) V8HI_FTYPE_V8HI },
31887 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv4si, "__builtin_ia32_pbroadcastd128", IX86_BUILTIN_PBROADCASTD128, UNKNOWN, (int) V4SI_FTYPE_V4SI },
31888 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv2di, "__builtin_ia32_pbroadcastq128", IX86_BUILTIN_PBROADCASTQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI },
31889 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permvarv8si, "__builtin_ia32_permvarsi256", IX86_BUILTIN_VPERMVARSI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
31890 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permvarv8sf, "__builtin_ia32_permvarsf256", IX86_BUILTIN_VPERMVARSF256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SI },
31891 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permv4df, "__builtin_ia32_permdf256", IX86_BUILTIN_VPERMDF256, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT },
31892 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permv4di, "__builtin_ia32_permdi256", IX86_BUILTIN_VPERMDI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT },
31893 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permv2ti, "__builtin_ia32_permti256", IX86_BUILTIN_VPERMTI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_INT },
31894 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx_vextractf128v4di, "__builtin_ia32_extract128i256", IX86_BUILTIN_VEXTRACT128I256, UNKNOWN, (int) V2DI_FTYPE_V4DI_INT },
31895 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx_vinsertf128v4di, "__builtin_ia32_insert128i256", IX86_BUILTIN_VINSERT128I256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_INT },
31896 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv4di, "__builtin_ia32_psllv4di", IX86_BUILTIN_PSLLVV4DI, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
31897 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv2di, "__builtin_ia32_psllv2di", IX86_BUILTIN_PSLLVV2DI, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
31898 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv8si, "__builtin_ia32_psllv8si", IX86_BUILTIN_PSLLVV8SI, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
31899 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv4si, "__builtin_ia32_psllv4si", IX86_BUILTIN_PSLLVV4SI, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
31900 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashrvv8si, "__builtin_ia32_psrav8si", IX86_BUILTIN_PSRAVV8SI, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
31901 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashrvv4si, "__builtin_ia32_psrav4si", IX86_BUILTIN_PSRAVV4SI, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
31902 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrvv4di, "__builtin_ia32_psrlv4di", IX86_BUILTIN_PSRLVV4DI, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
31903 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrvv2di, "__builtin_ia32_psrlv2di", IX86_BUILTIN_PSRLVV2DI, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
31904 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrvv8si, "__builtin_ia32_psrlv8si", IX86_BUILTIN_PSRLVV8SI, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
31905 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrvv4si, "__builtin_ia32_psrlv4si", IX86_BUILTIN_PSRLVV4SI, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
31907 { OPTION_MASK_ISA_LZCNT, CODE_FOR_clzhi2_lzcnt, "__builtin_clzs", IX86_BUILTIN_CLZS, UNKNOWN, (int) UINT16_FTYPE_UINT16 },
31910 { OPTION_MASK_ISA_BMI, CODE_FOR_bmi_bextr_si, "__builtin_ia32_bextr_u32", IX86_BUILTIN_BEXTR32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
31911 { OPTION_MASK_ISA_BMI, CODE_FOR_bmi_bextr_di, "__builtin_ia32_bextr_u64", IX86_BUILTIN_BEXTR64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
31912 { OPTION_MASK_ISA_BMI, CODE_FOR_ctzhi2, "__builtin_ctzs", IX86_BUILTIN_CTZS, UNKNOWN, (int) UINT16_FTYPE_UINT16 },
31915 { OPTION_MASK_ISA_TBM, CODE_FOR_tbm_bextri_si, "__builtin_ia32_bextri_u32", IX86_BUILTIN_BEXTRI32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
31916 { OPTION_MASK_ISA_TBM, CODE_FOR_tbm_bextri_di, "__builtin_ia32_bextri_u64", IX86_BUILTIN_BEXTRI64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
31919 { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtph2ps, "__builtin_ia32_vcvtph2ps", IX86_BUILTIN_CVTPH2PS, UNKNOWN, (int) V4SF_FTYPE_V8HI },
31920 { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtph2ps256, "__builtin_ia32_vcvtph2ps256", IX86_BUILTIN_CVTPH2PS256, UNKNOWN, (int) V8SF_FTYPE_V8HI },
31921 { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtps2ph, "__builtin_ia32_vcvtps2ph", IX86_BUILTIN_CVTPS2PH, UNKNOWN, (int) V8HI_FTYPE_V4SF_INT },
31922 { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtps2ph256, "__builtin_ia32_vcvtps2ph256", IX86_BUILTIN_CVTPS2PH256, UNKNOWN, (int) V8HI_FTYPE_V8SF_INT },
31925 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_bzhi_si3, "__builtin_ia32_bzhi_si", IX86_BUILTIN_BZHI32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
31926 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_bzhi_di3, "__builtin_ia32_bzhi_di", IX86_BUILTIN_BZHI64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
31927 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_pdep_si3, "__builtin_ia32_pdep_si", IX86_BUILTIN_PDEP32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
31928 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_pdep_di3, "__builtin_ia32_pdep_di", IX86_BUILTIN_PDEP64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
31929 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_pext_si3, "__builtin_ia32_pext_si", IX86_BUILTIN_PEXT32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
31930 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_pext_di3, "__builtin_ia32_pext_di", IX86_BUILTIN_PEXT64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
31933 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_si512_256si, "__builtin_ia32_si512_256si", IX86_BUILTIN_SI512_SI256, UNKNOWN, (int) V16SI_FTYPE_V8SI },
31934 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ps512_256ps, "__builtin_ia32_ps512_256ps", IX86_BUILTIN_PS512_PS256, UNKNOWN, (int) V16SF_FTYPE_V8SF },
31935 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_pd512_256pd, "__builtin_ia32_pd512_256pd", IX86_BUILTIN_PD512_PD256, UNKNOWN, (int) V8DF_FTYPE_V4DF },
31936 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_si512_si, "__builtin_ia32_si512_si", IX86_BUILTIN_SI512_SI, UNKNOWN, (int) V16SI_FTYPE_V4SI },
31937 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ps512_ps, "__builtin_ia32_ps512_ps", IX86_BUILTIN_PS512_PS, UNKNOWN, (int) V16SF_FTYPE_V4SF },
31938 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_pd512_pd, "__builtin_ia32_pd512_pd", IX86_BUILTIN_PD512_PD, UNKNOWN, (int) V8DF_FTYPE_V2DF },
31939 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_alignv16si_mask, "__builtin_ia32_alignd512_mask", IX86_BUILTIN_ALIGND512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_INT_V16SI_HI },
31940 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_alignv8di_mask, "__builtin_ia32_alignq512_mask", IX86_BUILTIN_ALIGNQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_INT_V8DI_QI },
31941 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_blendmv16si, "__builtin_ia32_blendmd_512_mask", IX86_BUILTIN_BLENDMD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_HI },
31942 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_blendmv8df, "__builtin_ia32_blendmpd_512_mask", IX86_BUILTIN_BLENDMPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_QI },
31943 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_blendmv16sf, "__builtin_ia32_blendmps_512_mask", IX86_BUILTIN_BLENDMPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_HI },
31944 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_blendmv8di, "__builtin_ia32_blendmq_512_mask", IX86_BUILTIN_BLENDMQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_QI },
31945 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_broadcastv16sf_mask, "__builtin_ia32_broadcastf32x4_512", IX86_BUILTIN_BROADCASTF32X4_512, UNKNOWN, (int) V16SF_FTYPE_V4SF_V16SF_HI },
31946 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_broadcastv8df_mask, "__builtin_ia32_broadcastf64x4_512", IX86_BUILTIN_BROADCASTF64X4_512, UNKNOWN, (int) V8DF_FTYPE_V4DF_V8DF_QI },
31947 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_broadcastv16si_mask, "__builtin_ia32_broadcasti32x4_512", IX86_BUILTIN_BROADCASTI32X4_512, UNKNOWN, (int) V16SI_FTYPE_V4SI_V16SI_HI },
31948 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_broadcastv8di_mask, "__builtin_ia32_broadcasti64x4_512", IX86_BUILTIN_BROADCASTI64X4_512, UNKNOWN, (int) V8DI_FTYPE_V4DI_V8DI_QI },
31949 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vec_dupv8df_mask, "__builtin_ia32_broadcastsd512", IX86_BUILTIN_BROADCASTSD512, UNKNOWN, (int) V8DF_FTYPE_V2DF_V8DF_QI },
31950 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vec_dupv16sf_mask, "__builtin_ia32_broadcastss512", IX86_BUILTIN_BROADCASTSS512, UNKNOWN, (int) V16SF_FTYPE_V4SF_V16SF_HI },
31951 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_cmpv16si3_mask, "__builtin_ia32_cmpd512_mask", IX86_BUILTIN_CMPD512, UNKNOWN, (int) HI_FTYPE_V16SI_V16SI_INT_HI },
31952 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_cmpv8di3_mask, "__builtin_ia32_cmpq512_mask", IX86_BUILTIN_CMPQ512, UNKNOWN, (int) QI_FTYPE_V8DI_V8DI_INT_QI },
31953 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_compressv8df_mask, "__builtin_ia32_compressdf512_mask", IX86_BUILTIN_COMPRESSPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_QI },
31954 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_compressv16sf_mask, "__builtin_ia32_compresssf512_mask", IX86_BUILTIN_COMPRESSPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_HI },
31955 { OPTION_MASK_ISA_AVX512F, CODE_FOR_floatv8siv8df2_mask, "__builtin_ia32_cvtdq2pd512_mask", IX86_BUILTIN_CVTDQ2PD512, UNKNOWN, (int) V8DF_FTYPE_V8SI_V8DF_QI },
31956 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vcvtps2ph512_mask, "__builtin_ia32_vcvtps2ph512_mask", IX86_BUILTIN_CVTPS2PH512, UNKNOWN, (int) V16HI_FTYPE_V16SF_INT_V16HI_HI },
31957 { OPTION_MASK_ISA_AVX512F, CODE_FOR_ufloatv8siv8df2_mask, "__builtin_ia32_cvtudq2pd512_mask", IX86_BUILTIN_CVTUDQ2PD512, UNKNOWN, (int) V8DF_FTYPE_V8SI_V8DF_QI },
31958 { OPTION_MASK_ISA_AVX512F, CODE_FOR_cvtusi2sd32, "__builtin_ia32_cvtusi2sd32", IX86_BUILTIN_CVTUSI2SD32, UNKNOWN, (int) V2DF_FTYPE_V2DF_UINT },
31959 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_expandv8df_mask, "__builtin_ia32_expanddf512_mask", IX86_BUILTIN_EXPANDPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_QI },
31960 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_expandv8df_maskz, "__builtin_ia32_expanddf512_maskz", IX86_BUILTIN_EXPANDPD512Z, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_QI },
31961 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_expandv16sf_mask, "__builtin_ia32_expandsf512_mask", IX86_BUILTIN_EXPANDPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_HI },
31962 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_expandv16sf_maskz, "__builtin_ia32_expandsf512_maskz", IX86_BUILTIN_EXPANDPS512Z, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_HI },
31963 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vextractf32x4_mask, "__builtin_ia32_extractf32x4_mask", IX86_BUILTIN_EXTRACTF32X4, UNKNOWN, (int) V4SF_FTYPE_V16SF_INT_V4SF_QI },
31964 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vextractf64x4_mask, "__builtin_ia32_extractf64x4_mask", IX86_BUILTIN_EXTRACTF64X4, UNKNOWN, (int) V4DF_FTYPE_V8DF_INT_V4DF_QI },
31965 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vextracti32x4_mask, "__builtin_ia32_extracti32x4_mask", IX86_BUILTIN_EXTRACTI32X4, UNKNOWN, (int) V4SI_FTYPE_V16SI_INT_V4SI_QI },
31966 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vextracti64x4_mask, "__builtin_ia32_extracti64x4_mask", IX86_BUILTIN_EXTRACTI64X4, UNKNOWN, (int) V4DI_FTYPE_V8DI_INT_V4DI_QI },
31967 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vinsertf32x4_mask, "__builtin_ia32_insertf32x4_mask", IX86_BUILTIN_INSERTF32X4, UNKNOWN, (int) V16SF_FTYPE_V16SF_V4SF_INT_V16SF_HI },
31968 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vinsertf64x4_mask, "__builtin_ia32_insertf64x4_mask", IX86_BUILTIN_INSERTF64X4, UNKNOWN, (int) V8DF_FTYPE_V8DF_V4DF_INT_V8DF_QI },
31969 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vinserti32x4_mask, "__builtin_ia32_inserti32x4_mask", IX86_BUILTIN_INSERTI32X4, UNKNOWN, (int) V16SI_FTYPE_V16SI_V4SI_INT_V16SI_HI },
31970 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vinserti64x4_mask, "__builtin_ia32_inserti64x4_mask", IX86_BUILTIN_INSERTI64X4, UNKNOWN, (int) V8DI_FTYPE_V8DI_V4DI_INT_V8DI_QI },
31971 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_loadv8df_mask, "__builtin_ia32_movapd512_mask", IX86_BUILTIN_MOVAPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_QI },
31972 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_loadv16sf_mask, "__builtin_ia32_movaps512_mask", IX86_BUILTIN_MOVAPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_HI },
31973 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_movddup512_mask, "__builtin_ia32_movddup512_mask", IX86_BUILTIN_MOVDDUP512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_QI },
31974 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_loadv16si_mask, "__builtin_ia32_movdqa32_512_mask", IX86_BUILTIN_MOVDQA32_512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_HI },
31975 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_loadv8di_mask, "__builtin_ia32_movdqa64_512_mask", IX86_BUILTIN_MOVDQA64_512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_QI },
31976 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_movshdup512_mask, "__builtin_ia32_movshdup512_mask", IX86_BUILTIN_MOVSHDUP512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_HI },
31977 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_movsldup512_mask, "__builtin_ia32_movsldup512_mask", IX86_BUILTIN_MOVSLDUP512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_HI },
31978 { OPTION_MASK_ISA_AVX512F, CODE_FOR_absv16si2_mask, "__builtin_ia32_pabsd512_mask", IX86_BUILTIN_PABSD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_HI },
31979 { OPTION_MASK_ISA_AVX512F, CODE_FOR_absv8di2_mask, "__builtin_ia32_pabsq512_mask", IX86_BUILTIN_PABSQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_QI },
31980 { OPTION_MASK_ISA_AVX512F, CODE_FOR_addv16si3_mask, "__builtin_ia32_paddd512_mask", IX86_BUILTIN_PADDD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
31981 { OPTION_MASK_ISA_AVX512F, CODE_FOR_addv8di3_mask, "__builtin_ia32_paddq512_mask", IX86_BUILTIN_PADDQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
31982 { OPTION_MASK_ISA_AVX512F, CODE_FOR_andv16si3_mask, "__builtin_ia32_pandd512_mask", IX86_BUILTIN_PANDD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
31983 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_andnotv16si3_mask, "__builtin_ia32_pandnd512_mask", IX86_BUILTIN_PANDND512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
31984 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_andnotv8di3_mask, "__builtin_ia32_pandnq512_mask", IX86_BUILTIN_PANDNQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
31985 { OPTION_MASK_ISA_AVX512F, CODE_FOR_andv8di3_mask, "__builtin_ia32_pandq512_mask", IX86_BUILTIN_PANDQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
31986 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vec_dupv16si_mask, "__builtin_ia32_pbroadcastd512", IX86_BUILTIN_PBROADCASTD512, UNKNOWN, (int) V16SI_FTYPE_V4SI_V16SI_HI },
31987 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vec_dup_gprv16si_mask, "__builtin_ia32_pbroadcastd512_gpr_mask", IX86_BUILTIN_PBROADCASTD512_GPR, UNKNOWN, (int) V16SI_FTYPE_SI_V16SI_HI },
31988 { OPTION_MASK_ISA_AVX512CD, CODE_FOR_avx512cd_maskb_vec_dupv8di, "__builtin_ia32_broadcastmb512", IX86_BUILTIN_PBROADCASTMB512, UNKNOWN, (int) V8DI_FTYPE_QI },
31989 { OPTION_MASK_ISA_AVX512CD, CODE_FOR_avx512cd_maskw_vec_dupv16si, "__builtin_ia32_broadcastmw512", IX86_BUILTIN_PBROADCASTMW512, UNKNOWN, (int) V16SI_FTYPE_HI },
31990 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vec_dupv8di_mask, "__builtin_ia32_pbroadcastq512", IX86_BUILTIN_PBROADCASTQ512, UNKNOWN, (int) V8DI_FTYPE_V2DI_V8DI_QI },
31991 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vec_dup_gprv8di_mask, "__builtin_ia32_pbroadcastq512_gpr_mask", IX86_BUILTIN_PBROADCASTQ512_GPR, UNKNOWN, (int) V8DI_FTYPE_DI_V8DI_QI },
31992 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_eqv16si3_mask, "__builtin_ia32_pcmpeqd512_mask", IX86_BUILTIN_PCMPEQD512_MASK, UNKNOWN, (int) HI_FTYPE_V16SI_V16SI_HI },
31993 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_eqv8di3_mask, "__builtin_ia32_pcmpeqq512_mask", IX86_BUILTIN_PCMPEQQ512_MASK, UNKNOWN, (int) QI_FTYPE_V8DI_V8DI_QI },
31994 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_gtv16si3_mask, "__builtin_ia32_pcmpgtd512_mask", IX86_BUILTIN_PCMPGTD512_MASK, UNKNOWN, (int) HI_FTYPE_V16SI_V16SI_HI },
31995 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_gtv8di3_mask, "__builtin_ia32_pcmpgtq512_mask", IX86_BUILTIN_PCMPGTQ512_MASK, UNKNOWN, (int) QI_FTYPE_V8DI_V8DI_QI },
31996 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_compressv16si_mask, "__builtin_ia32_compresssi512_mask", IX86_BUILTIN_PCOMPRESSD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_HI },
31997 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_compressv8di_mask, "__builtin_ia32_compressdi512_mask", IX86_BUILTIN_PCOMPRESSQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_QI },
31998 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_expandv16si_mask, "__builtin_ia32_expandsi512_mask", IX86_BUILTIN_PEXPANDD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_HI },
31999 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_expandv16si_maskz, "__builtin_ia32_expandsi512_maskz", IX86_BUILTIN_PEXPANDD512Z, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_HI },
32000 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_expandv8di_mask, "__builtin_ia32_expanddi512_mask", IX86_BUILTIN_PEXPANDQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_QI },
32001 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_expandv8di_maskz, "__builtin_ia32_expanddi512_maskz", IX86_BUILTIN_PEXPANDQ512Z, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_QI },
32002 { OPTION_MASK_ISA_AVX512F, CODE_FOR_smaxv16si3_mask, "__builtin_ia32_pmaxsd512_mask", IX86_BUILTIN_PMAXSD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32003 { OPTION_MASK_ISA_AVX512F, CODE_FOR_smaxv8di3_mask, "__builtin_ia32_pmaxsq512_mask", IX86_BUILTIN_PMAXSQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32004 { OPTION_MASK_ISA_AVX512F, CODE_FOR_umaxv16si3_mask, "__builtin_ia32_pmaxud512_mask", IX86_BUILTIN_PMAXUD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32005 { OPTION_MASK_ISA_AVX512F, CODE_FOR_umaxv8di3_mask, "__builtin_ia32_pmaxuq512_mask", IX86_BUILTIN_PMAXUQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32006 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sminv16si3_mask, "__builtin_ia32_pminsd512_mask", IX86_BUILTIN_PMINSD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32007 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sminv8di3_mask, "__builtin_ia32_pminsq512_mask", IX86_BUILTIN_PMINSQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32008 { OPTION_MASK_ISA_AVX512F, CODE_FOR_uminv16si3_mask, "__builtin_ia32_pminud512_mask", IX86_BUILTIN_PMINUD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32009 { OPTION_MASK_ISA_AVX512F, CODE_FOR_uminv8di3_mask, "__builtin_ia32_pminuq512_mask", IX86_BUILTIN_PMINUQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32010 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_truncatev16siv16qi2_mask, "__builtin_ia32_pmovdb512_mask", IX86_BUILTIN_PMOVDB512, UNKNOWN, (int) V16QI_FTYPE_V16SI_V16QI_HI },
32011 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_truncatev16siv16hi2_mask, "__builtin_ia32_pmovdw512_mask", IX86_BUILTIN_PMOVDW512, UNKNOWN, (int) V16HI_FTYPE_V16SI_V16HI_HI },
32012 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_truncatev8div16qi2_mask, "__builtin_ia32_pmovqb512_mask", IX86_BUILTIN_PMOVQB512, UNKNOWN, (int) V16QI_FTYPE_V8DI_V16QI_QI },
32013 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_truncatev8div8si2_mask, "__builtin_ia32_pmovqd512_mask", IX86_BUILTIN_PMOVQD512, UNKNOWN, (int) V8SI_FTYPE_V8DI_V8SI_QI },
32014 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_truncatev8div8hi2_mask, "__builtin_ia32_pmovqw512_mask", IX86_BUILTIN_PMOVQW512, UNKNOWN, (int) V8HI_FTYPE_V8DI_V8HI_QI },
32015 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ss_truncatev16siv16qi2_mask, "__builtin_ia32_pmovsdb512_mask", IX86_BUILTIN_PMOVSDB512, UNKNOWN, (int) V16QI_FTYPE_V16SI_V16QI_HI },
32016 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ss_truncatev16siv16hi2_mask, "__builtin_ia32_pmovsdw512_mask", IX86_BUILTIN_PMOVSDW512, UNKNOWN, (int) V16HI_FTYPE_V16SI_V16HI_HI },
32017 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ss_truncatev8div16qi2_mask, "__builtin_ia32_pmovsqb512_mask", IX86_BUILTIN_PMOVSQB512, UNKNOWN, (int) V16QI_FTYPE_V8DI_V16QI_QI },
32018 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ss_truncatev8div8si2_mask, "__builtin_ia32_pmovsqd512_mask", IX86_BUILTIN_PMOVSQD512, UNKNOWN, (int) V8SI_FTYPE_V8DI_V8SI_QI },
32019 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ss_truncatev8div8hi2_mask, "__builtin_ia32_pmovsqw512_mask", IX86_BUILTIN_PMOVSQW512, UNKNOWN, (int) V8HI_FTYPE_V8DI_V8HI_QI },
32020 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_sign_extendv16qiv16si2_mask, "__builtin_ia32_pmovsxbd512_mask", IX86_BUILTIN_PMOVSXBD512, UNKNOWN, (int) V16SI_FTYPE_V16QI_V16SI_HI },
32021 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_sign_extendv8qiv8di2_mask, "__builtin_ia32_pmovsxbq512_mask", IX86_BUILTIN_PMOVSXBQ512, UNKNOWN, (int) V8DI_FTYPE_V16QI_V8DI_QI },
32022 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_sign_extendv8siv8di2_mask, "__builtin_ia32_pmovsxdq512_mask", IX86_BUILTIN_PMOVSXDQ512, UNKNOWN, (int) V8DI_FTYPE_V8SI_V8DI_QI },
32023 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_sign_extendv16hiv16si2_mask, "__builtin_ia32_pmovsxwd512_mask", IX86_BUILTIN_PMOVSXWD512, UNKNOWN, (int) V16SI_FTYPE_V16HI_V16SI_HI },
32024 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_sign_extendv8hiv8di2_mask, "__builtin_ia32_pmovsxwq512_mask", IX86_BUILTIN_PMOVSXWQ512, UNKNOWN, (int) V8DI_FTYPE_V8HI_V8DI_QI },
32025 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_us_truncatev16siv16qi2_mask, "__builtin_ia32_pmovusdb512_mask", IX86_BUILTIN_PMOVUSDB512, UNKNOWN, (int) V16QI_FTYPE_V16SI_V16QI_HI },
32026 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_us_truncatev16siv16hi2_mask, "__builtin_ia32_pmovusdw512_mask", IX86_BUILTIN_PMOVUSDW512, UNKNOWN, (int) V16HI_FTYPE_V16SI_V16HI_HI },
32027 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_us_truncatev8div16qi2_mask, "__builtin_ia32_pmovusqb512_mask", IX86_BUILTIN_PMOVUSQB512, UNKNOWN, (int) V16QI_FTYPE_V8DI_V16QI_QI },
32028 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_us_truncatev8div8si2_mask, "__builtin_ia32_pmovusqd512_mask", IX86_BUILTIN_PMOVUSQD512, UNKNOWN, (int) V8SI_FTYPE_V8DI_V8SI_QI },
32029 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_us_truncatev8div8hi2_mask, "__builtin_ia32_pmovusqw512_mask", IX86_BUILTIN_PMOVUSQW512, UNKNOWN, (int) V8HI_FTYPE_V8DI_V8HI_QI },
32030 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_zero_extendv16qiv16si2_mask, "__builtin_ia32_pmovzxbd512_mask", IX86_BUILTIN_PMOVZXBD512, UNKNOWN, (int) V16SI_FTYPE_V16QI_V16SI_HI },
32031 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_zero_extendv8qiv8di2_mask, "__builtin_ia32_pmovzxbq512_mask", IX86_BUILTIN_PMOVZXBQ512, UNKNOWN, (int) V8DI_FTYPE_V16QI_V8DI_QI },
32032 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_zero_extendv8siv8di2_mask, "__builtin_ia32_pmovzxdq512_mask", IX86_BUILTIN_PMOVZXDQ512, UNKNOWN, (int) V8DI_FTYPE_V8SI_V8DI_QI },
32033 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_zero_extendv16hiv16si2_mask, "__builtin_ia32_pmovzxwd512_mask", IX86_BUILTIN_PMOVZXWD512, UNKNOWN, (int) V16SI_FTYPE_V16HI_V16SI_HI },
32034 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_zero_extendv8hiv8di2_mask, "__builtin_ia32_pmovzxwq512_mask", IX86_BUILTIN_PMOVZXWQ512, UNKNOWN, (int) V8DI_FTYPE_V8HI_V8DI_QI },
32035 { OPTION_MASK_ISA_AVX512F, CODE_FOR_vec_widen_smult_even_v16si_mask, "__builtin_ia32_pmuldq512_mask", IX86_BUILTIN_PMULDQ512, UNKNOWN, (int) V8DI_FTYPE_V16SI_V16SI_V8DI_QI },
32036 { OPTION_MASK_ISA_AVX512F, CODE_FOR_mulv16si3_mask, "__builtin_ia32_pmulld512_mask" , IX86_BUILTIN_PMULLD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32037 { OPTION_MASK_ISA_AVX512F, CODE_FOR_vec_widen_umult_even_v16si_mask, "__builtin_ia32_pmuludq512_mask", IX86_BUILTIN_PMULUDQ512, UNKNOWN, (int) V8DI_FTYPE_V16SI_V16SI_V8DI_QI },
32038 { OPTION_MASK_ISA_AVX512F, CODE_FOR_iorv16si3_mask, "__builtin_ia32_pord512_mask", IX86_BUILTIN_PORD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32039 { OPTION_MASK_ISA_AVX512F, CODE_FOR_iorv8di3_mask, "__builtin_ia32_porq512_mask", IX86_BUILTIN_PORQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32040 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_rolv16si_mask, "__builtin_ia32_prold512_mask", IX86_BUILTIN_PROLD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_INT_V16SI_HI },
32041 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_rolv8di_mask, "__builtin_ia32_prolq512_mask", IX86_BUILTIN_PROLQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_INT_V8DI_QI },
32042 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_rolvv16si_mask, "__builtin_ia32_prolvd512_mask", IX86_BUILTIN_PROLVD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32043 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_rolvv8di_mask, "__builtin_ia32_prolvq512_mask", IX86_BUILTIN_PROLVQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32044 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_rorv16si_mask, "__builtin_ia32_prord512_mask", IX86_BUILTIN_PRORD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_INT_V16SI_HI },
32045 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_rorv8di_mask, "__builtin_ia32_prorq512_mask", IX86_BUILTIN_PRORQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_INT_V8DI_QI },
32046 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_rorvv16si_mask, "__builtin_ia32_prorvd512_mask", IX86_BUILTIN_PRORVD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32047 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_rorvv8di_mask, "__builtin_ia32_prorvq512_mask", IX86_BUILTIN_PRORVQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32048 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_pshufdv3_mask, "__builtin_ia32_pshufd512_mask", IX86_BUILTIN_PSHUFD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_INT_V16SI_HI },
32049 { OPTION_MASK_ISA_AVX512F, CODE_FOR_ashlv16si3_mask, "__builtin_ia32_pslld512_mask", IX86_BUILTIN_PSLLD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V4SI_V16SI_HI },
32050 { OPTION_MASK_ISA_AVX512F, CODE_FOR_ashlv16si3_mask, "__builtin_ia32_pslldi512_mask", IX86_BUILTIN_PSLLDI512, UNKNOWN, (int) V16SI_FTYPE_V16SI_INT_V16SI_HI },
32051 { OPTION_MASK_ISA_AVX512F, CODE_FOR_ashlv8di3_mask, "__builtin_ia32_psllq512_mask", IX86_BUILTIN_PSLLQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V2DI_V8DI_QI },
32052 { OPTION_MASK_ISA_AVX512F, CODE_FOR_ashlv8di3_mask, "__builtin_ia32_psllqi512_mask", IX86_BUILTIN_PSLLQI512, UNKNOWN, (int) V8DI_FTYPE_V8DI_INT_V8DI_QI },
32053 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ashlvv16si_mask, "__builtin_ia32_psllv16si_mask", IX86_BUILTIN_PSLLVV16SI, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32054 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ashlvv8di_mask, "__builtin_ia32_psllv8di_mask", IX86_BUILTIN_PSLLVV8DI, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32055 { OPTION_MASK_ISA_AVX512F, CODE_FOR_ashrv16si3_mask, "__builtin_ia32_psrad512_mask", IX86_BUILTIN_PSRAD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V4SI_V16SI_HI },
32056 { OPTION_MASK_ISA_AVX512F, CODE_FOR_ashrv16si3_mask, "__builtin_ia32_psradi512_mask", IX86_BUILTIN_PSRADI512, UNKNOWN, (int) V16SI_FTYPE_V16SI_INT_V16SI_HI },
32057 { OPTION_MASK_ISA_AVX512F, CODE_FOR_ashrv8di3_mask, "__builtin_ia32_psraq512_mask", IX86_BUILTIN_PSRAQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V2DI_V8DI_QI },
32058 { OPTION_MASK_ISA_AVX512F, CODE_FOR_ashrv8di3_mask, "__builtin_ia32_psraqi512_mask", IX86_BUILTIN_PSRAQI512, UNKNOWN, (int) V8DI_FTYPE_V8DI_INT_V8DI_QI },
32059 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ashrvv16si_mask, "__builtin_ia32_psrav16si_mask", IX86_BUILTIN_PSRAVV16SI, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32060 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ashrvv8di_mask, "__builtin_ia32_psrav8di_mask", IX86_BUILTIN_PSRAVV8DI, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32061 { OPTION_MASK_ISA_AVX512F, CODE_FOR_lshrv16si3_mask, "__builtin_ia32_psrld512_mask", IX86_BUILTIN_PSRLD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V4SI_V16SI_HI },
32062 { OPTION_MASK_ISA_AVX512F, CODE_FOR_lshrv16si3_mask, "__builtin_ia32_psrldi512_mask", IX86_BUILTIN_PSRLDI512, UNKNOWN, (int) V16SI_FTYPE_V16SI_INT_V16SI_HI },
32063 { OPTION_MASK_ISA_AVX512F, CODE_FOR_lshrv8di3_mask, "__builtin_ia32_psrlq512_mask", IX86_BUILTIN_PSRLQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V2DI_V8DI_QI },
32064 { OPTION_MASK_ISA_AVX512F, CODE_FOR_lshrv8di3_mask, "__builtin_ia32_psrlqi512_mask", IX86_BUILTIN_PSRLQI512, UNKNOWN, (int) V8DI_FTYPE_V8DI_INT_V8DI_QI },
32065 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_lshrvv16si_mask, "__builtin_ia32_psrlv16si_mask", IX86_BUILTIN_PSRLVV16SI, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32066 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_lshrvv8di_mask, "__builtin_ia32_psrlv8di_mask", IX86_BUILTIN_PSRLVV8DI, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32067 { OPTION_MASK_ISA_AVX512F, CODE_FOR_subv16si3_mask, "__builtin_ia32_psubd512_mask", IX86_BUILTIN_PSUBD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32068 { OPTION_MASK_ISA_AVX512F, CODE_FOR_subv8di3_mask, "__builtin_ia32_psubq512_mask", IX86_BUILTIN_PSUBQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32069 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_testmv16si3_mask, "__builtin_ia32_ptestmd512", IX86_BUILTIN_PTESTMD512, UNKNOWN, (int) HI_FTYPE_V16SI_V16SI_HI },
32070 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_testmv8di3_mask, "__builtin_ia32_ptestmq512", IX86_BUILTIN_PTESTMQ512, UNKNOWN, (int) QI_FTYPE_V8DI_V8DI_QI },
32071 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_testnmv16si3_mask, "__builtin_ia32_ptestnmd512", IX86_BUILTIN_PTESTNMD512, UNKNOWN, (int) HI_FTYPE_V16SI_V16SI_HI },
32072 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_testnmv8di3_mask, "__builtin_ia32_ptestnmq512", IX86_BUILTIN_PTESTNMQ512, UNKNOWN, (int) QI_FTYPE_V8DI_V8DI_QI },
32073 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_interleave_highv16si_mask, "__builtin_ia32_punpckhdq512_mask", IX86_BUILTIN_PUNPCKHDQ512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32074 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_interleave_highv8di_mask, "__builtin_ia32_punpckhqdq512_mask", IX86_BUILTIN_PUNPCKHQDQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32075 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_interleave_lowv16si_mask, "__builtin_ia32_punpckldq512_mask", IX86_BUILTIN_PUNPCKLDQ512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32076 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_interleave_lowv8di_mask, "__builtin_ia32_punpcklqdq512_mask", IX86_BUILTIN_PUNPCKLQDQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32077 { OPTION_MASK_ISA_AVX512F, CODE_FOR_xorv16si3_mask, "__builtin_ia32_pxord512_mask", IX86_BUILTIN_PXORD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32078 { OPTION_MASK_ISA_AVX512F, CODE_FOR_xorv8di3_mask, "__builtin_ia32_pxorq512_mask", IX86_BUILTIN_PXORQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32079 { OPTION_MASK_ISA_AVX512F, CODE_FOR_rcp14v8df_mask, "__builtin_ia32_rcp14pd512_mask", IX86_BUILTIN_RCP14PD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_QI },
32080 { OPTION_MASK_ISA_AVX512F, CODE_FOR_rcp14v16sf_mask, "__builtin_ia32_rcp14ps512_mask", IX86_BUILTIN_RCP14PS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_HI },
32081 { OPTION_MASK_ISA_AVX512F, CODE_FOR_srcp14v2df, "__builtin_ia32_rcp14sd", IX86_BUILTIN_RCP14SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
32082 { OPTION_MASK_ISA_AVX512F, CODE_FOR_srcp14v4sf, "__builtin_ia32_rcp14ss", IX86_BUILTIN_RCP14SS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
32083 { OPTION_MASK_ISA_AVX512F, CODE_FOR_rsqrt14v8df_mask, "__builtin_ia32_rsqrt14pd512_mask", IX86_BUILTIN_RSQRT14PD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_QI },
32084 { OPTION_MASK_ISA_AVX512F, CODE_FOR_rsqrt14v16sf_mask, "__builtin_ia32_rsqrt14ps512_mask", IX86_BUILTIN_RSQRT14PS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_HI },
32085 { OPTION_MASK_ISA_AVX512F, CODE_FOR_rsqrt14v2df, "__builtin_ia32_rsqrt14sd", IX86_BUILTIN_RSQRT14SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
32086 { OPTION_MASK_ISA_AVX512F, CODE_FOR_rsqrt14v4sf, "__builtin_ia32_rsqrt14ss", IX86_BUILTIN_RSQRT14SS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
32087 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_shufpd512_mask, "__builtin_ia32_shufpd512_mask", IX86_BUILTIN_SHUFPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_INT_V8DF_QI },
32088 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_shufps512_mask, "__builtin_ia32_shufps512_mask", IX86_BUILTIN_SHUFPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_INT_V16SF_HI },
32089 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_shuf_f32x4_mask, "__builtin_ia32_shuf_f32x4_mask", IX86_BUILTIN_SHUF_F32x4, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_INT_V16SF_HI },
32090 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_shuf_f64x2_mask, "__builtin_ia32_shuf_f64x2_mask", IX86_BUILTIN_SHUF_F64x2, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_INT_V8DF_QI },
32091 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_shuf_i32x4_mask, "__builtin_ia32_shuf_i32x4_mask", IX86_BUILTIN_SHUF_I32x4, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_INT_V16SI_HI },
32092 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_shuf_i64x2_mask, "__builtin_ia32_shuf_i64x2_mask", IX86_BUILTIN_SHUF_I64x2, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_INT_V8DI_QI },
32093 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ucmpv16si3_mask, "__builtin_ia32_ucmpd512_mask", IX86_BUILTIN_UCMPD512, UNKNOWN, (int) HI_FTYPE_V16SI_V16SI_INT_HI },
32094 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ucmpv8di3_mask, "__builtin_ia32_ucmpq512_mask", IX86_BUILTIN_UCMPQ512, UNKNOWN, (int) QI_FTYPE_V8DI_V8DI_INT_QI },
32095 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_unpckhpd512_mask, "__builtin_ia32_unpckhpd512_mask", IX86_BUILTIN_UNPCKHPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI },
32096 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_unpckhps512_mask, "__builtin_ia32_unpckhps512_mask", IX86_BUILTIN_UNPCKHPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI },
32097 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_unpcklpd512_mask, "__builtin_ia32_unpcklpd512_mask", IX86_BUILTIN_UNPCKLPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI },
32098 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_unpcklps512_mask, "__builtin_ia32_unpcklps512_mask", IX86_BUILTIN_UNPCKLPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI },
32099 { OPTION_MASK_ISA_AVX512CD, CODE_FOR_clzv16si2_mask, "__builtin_ia32_vplzcntd_512_mask", IX86_BUILTIN_VPCLZCNTD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_HI },
32100 { OPTION_MASK_ISA_AVX512CD, CODE_FOR_clzv8di2_mask, "__builtin_ia32_vplzcntq_512_mask", IX86_BUILTIN_VPCLZCNTQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_QI },
32101 { OPTION_MASK_ISA_AVX512CD, CODE_FOR_conflictv16si_mask, "__builtin_ia32_vpconflictsi_512_mask", IX86_BUILTIN_VPCONFLICTD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_HI },
32102 { OPTION_MASK_ISA_AVX512CD, CODE_FOR_conflictv8di_mask, "__builtin_ia32_vpconflictdi_512_mask", IX86_BUILTIN_VPCONFLICTQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_QI },
32103 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_permv8df_mask, "__builtin_ia32_permdf512_mask", IX86_BUILTIN_VPERMDF512, UNKNOWN, (int) V8DF_FTYPE_V8DF_INT_V8DF_QI },
32104 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_permv8di_mask, "__builtin_ia32_permdi512_mask", IX86_BUILTIN_VPERMDI512, UNKNOWN, (int) V8DI_FTYPE_V8DI_INT_V8DI_QI },
32105 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vpermi2varv16si3_mask, "__builtin_ia32_vpermi2vard512_mask", IX86_BUILTIN_VPERMI2VARD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32106 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vpermi2varv8df3_mask, "__builtin_ia32_vpermi2varpd512_mask", IX86_BUILTIN_VPERMI2VARPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DI_V8DF_QI },
32107 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vpermi2varv16sf3_mask, "__builtin_ia32_vpermi2varps512_mask", IX86_BUILTIN_VPERMI2VARPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SI_V16SF_HI },
32108 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vpermi2varv8di3_mask, "__builtin_ia32_vpermi2varq512_mask", IX86_BUILTIN_VPERMI2VARQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32109 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vpermilv8df_mask, "__builtin_ia32_vpermilpd512_mask", IX86_BUILTIN_VPERMILPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_INT_V8DF_QI },
32110 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vpermilv16sf_mask, "__builtin_ia32_vpermilps512_mask", IX86_BUILTIN_VPERMILPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_INT_V16SF_HI },
32111 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vpermilvarv8df3_mask, "__builtin_ia32_vpermilvarpd512_mask", IX86_BUILTIN_VPERMILVARPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DI_V8DF_QI },
32112 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vpermilvarv16sf3_mask, "__builtin_ia32_vpermilvarps512_mask", IX86_BUILTIN_VPERMILVARPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SI_V16SF_HI },
32113 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vpermt2varv16si3_mask, "__builtin_ia32_vpermt2vard512_mask", IX86_BUILTIN_VPERMT2VARD512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32114 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vpermt2varv16si3_maskz, "__builtin_ia32_vpermt2vard512_maskz", IX86_BUILTIN_VPERMT2VARD512_MASKZ, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32115 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vpermt2varv8df3_mask, "__builtin_ia32_vpermt2varpd512_mask", IX86_BUILTIN_VPERMT2VARPD512, UNKNOWN, (int) V8DF_FTYPE_V8DI_V8DF_V8DF_QI },
32116 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vpermt2varv8df3_maskz, "__builtin_ia32_vpermt2varpd512_maskz", IX86_BUILTIN_VPERMT2VARPD512_MASKZ, UNKNOWN, (int) V8DF_FTYPE_V8DI_V8DF_V8DF_QI },
32117 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vpermt2varv16sf3_mask, "__builtin_ia32_vpermt2varps512_mask", IX86_BUILTIN_VPERMT2VARPS512, UNKNOWN, (int) V16SF_FTYPE_V16SI_V16SF_V16SF_HI },
32118 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vpermt2varv16sf3_maskz, "__builtin_ia32_vpermt2varps512_maskz", IX86_BUILTIN_VPERMT2VARPS512_MASKZ, UNKNOWN, (int) V16SF_FTYPE_V16SI_V16SF_V16SF_HI },
32119 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vpermt2varv8di3_mask, "__builtin_ia32_vpermt2varq512_mask", IX86_BUILTIN_VPERMT2VARQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32120 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vpermt2varv8di3_maskz, "__builtin_ia32_vpermt2varq512_maskz", IX86_BUILTIN_VPERMT2VARQ512_MASKZ, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32121 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_permvarv8df_mask, "__builtin_ia32_permvardf512_mask", IX86_BUILTIN_VPERMVARDF512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DI_V8DF_QI },
32122 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_permvarv8di_mask, "__builtin_ia32_permvardi512_mask", IX86_BUILTIN_VPERMVARDI512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32123 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_permvarv16sf_mask, "__builtin_ia32_permvarsf512_mask", IX86_BUILTIN_VPERMVARSF512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SI_V16SF_HI },
32124 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_permvarv16si_mask, "__builtin_ia32_permvarsi512_mask", IX86_BUILTIN_VPERMVARSI512, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_HI },
32125 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vternlogv16si_mask, "__builtin_ia32_pternlogd512_mask", IX86_BUILTIN_VTERNLOGD512_MASK, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_INT_HI },
32126 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vternlogv16si_maskz, "__builtin_ia32_pternlogd512_maskz", IX86_BUILTIN_VTERNLOGD512_MASKZ, UNKNOWN, (int) V16SI_FTYPE_V16SI_V16SI_V16SI_INT_HI },
32127 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vternlogv8di_mask, "__builtin_ia32_pternlogq512_mask", IX86_BUILTIN_VTERNLOGQ512_MASK, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_INT_QI },
32128 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vternlogv8di_maskz, "__builtin_ia32_pternlogq512_maskz", IX86_BUILTIN_VTERNLOGQ512_MASKZ, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_INT_QI },
32130 { OPTION_MASK_ISA_AVX512F, CODE_FOR_copysignv16sf3, "__builtin_ia32_copysignps512", IX86_BUILTIN_CPYSGNPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF },
32131 { OPTION_MASK_ISA_AVX512F, CODE_FOR_copysignv8df3, "__builtin_ia32_copysignpd512", IX86_BUILTIN_CPYSGNPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF },
32132 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_sqrtv8df2, "__builtin_ia32_sqrtpd512", IX86_BUILTIN_SQRTPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF },
32133 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sqrtv16sf2, "__builtin_ia32_sqrtps512", IX86_BUILTIN_SQRTPS_NR512, UNKNOWN, (int) V16SF_FTYPE_V16SF },
32134 { OPTION_MASK_ISA_AVX512ER, CODE_FOR_avx512er_exp2v16sf, "__builtin_ia32_exp2ps", IX86_BUILTIN_EXP2PS, UNKNOWN, (int) V16SF_FTYPE_V16SF },
32135 { OPTION_MASK_ISA_AVX512F, CODE_FOR_roundv8df2_vec_pack_sfix, "__builtin_ia32_roundpd_az_vec_pack_sfix512", IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX512, UNKNOWN, (int) V16SI_FTYPE_V8DF_V8DF },
32136 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_roundpd_vec_pack_sfix512, "__builtin_ia32_floorpd_vec_pack_sfix512", IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX512, (enum rtx_code) ROUND_FLOOR, (int) V16SI_FTYPE_V8DF_V8DF_ROUND },
32137 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_roundpd_vec_pack_sfix512, "__builtin_ia32_ceilpd_vec_pack_sfix512", IX86_BUILTIN_CEILPD_VEC_PACK_SFIX512, (enum rtx_code) ROUND_CEIL, (int) V16SI_FTYPE_V8DF_V8DF_ROUND },
32139 /* Mask arithmetic operations */
32140 { OPTION_MASK_ISA_AVX512F, CODE_FOR_andhi3, "__builtin_ia32_kandhi", IX86_BUILTIN_KAND16, UNKNOWN, (int) HI_FTYPE_HI_HI },
32141 { OPTION_MASK_ISA_AVX512F, CODE_FOR_kandnhi, "__builtin_ia32_kandnhi", IX86_BUILTIN_KANDN16, UNKNOWN, (int) HI_FTYPE_HI_HI },
32142 { OPTION_MASK_ISA_AVX512F, CODE_FOR_one_cmplhi2, "__builtin_ia32_knothi", IX86_BUILTIN_KNOT16, UNKNOWN, (int) HI_FTYPE_HI },
32143 { OPTION_MASK_ISA_AVX512F, CODE_FOR_iorhi3, "__builtin_ia32_korhi", IX86_BUILTIN_KOR16, UNKNOWN, (int) HI_FTYPE_HI_HI },
32144 { OPTION_MASK_ISA_AVX512F, CODE_FOR_kortestchi, "__builtin_ia32_kortestchi", IX86_BUILTIN_KORTESTC16, UNKNOWN, (int) HI_FTYPE_HI_HI },
32145 { OPTION_MASK_ISA_AVX512F, CODE_FOR_kortestzhi, "__builtin_ia32_kortestzhi", IX86_BUILTIN_KORTESTZ16, UNKNOWN, (int) HI_FTYPE_HI_HI },
32146 { OPTION_MASK_ISA_AVX512F, CODE_FOR_kunpckhi, "__builtin_ia32_kunpckhi", IX86_BUILTIN_KUNPCKBW, UNKNOWN, (int) HI_FTYPE_HI_HI },
32147 { OPTION_MASK_ISA_AVX512F, CODE_FOR_kxnorhi, "__builtin_ia32_kxnorhi", IX86_BUILTIN_KXNOR16, UNKNOWN, (int) HI_FTYPE_HI_HI },
32148 { OPTION_MASK_ISA_AVX512F, CODE_FOR_xorhi3, "__builtin_ia32_kxorhi", IX86_BUILTIN_KXOR16, UNKNOWN, (int) HI_FTYPE_HI_HI },
32149 { OPTION_MASK_ISA_AVX512F, CODE_FOR_kmovw, "__builtin_ia32_kmov16", IX86_BUILTIN_KMOV16, UNKNOWN, (int) HI_FTYPE_HI },
32152 { OPTION_MASK_ISA_SSE2, CODE_FOR_sha1msg1, 0, IX86_BUILTIN_SHA1MSG1, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
32153 { OPTION_MASK_ISA_SSE2, CODE_FOR_sha1msg2, 0, IX86_BUILTIN_SHA1MSG2, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
32154 { OPTION_MASK_ISA_SSE2, CODE_FOR_sha1nexte, 0, IX86_BUILTIN_SHA1NEXTE, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
32155 { OPTION_MASK_ISA_SSE2, CODE_FOR_sha1rnds4, 0, IX86_BUILTIN_SHA1RNDS4, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_INT },
32156 { OPTION_MASK_ISA_SSE2, CODE_FOR_sha256msg1, 0, IX86_BUILTIN_SHA256MSG1, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
32157 { OPTION_MASK_ISA_SSE2, CODE_FOR_sha256msg2, 0, IX86_BUILTIN_SHA256MSG2, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
32158 { OPTION_MASK_ISA_SSE2, CODE_FOR_sha256rnds2, 0, IX86_BUILTIN_SHA256RNDS2, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI },
32161 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_palignrv32qi_mask, "__builtin_ia32_palignr256_mask", IX86_BUILTIN_PALIGNR256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_INT_V4DI_SI_CONVERT },
32162 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ssse3_palignrv16qi_mask, "__builtin_ia32_palignr128_mask", IX86_BUILTIN_PALIGNR128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_INT_V2DI_HI_CONVERT },
32163 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loadv4di_mask, "__builtin_ia32_movdqa64_256_mask", IX86_BUILTIN_MOVDQA64_256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_QI },
32164 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loadv2di_mask, "__builtin_ia32_movdqa64_128_mask", IX86_BUILTIN_MOVDQA64_128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_QI },
32165 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loadv8si_mask, "__builtin_ia32_movdqa32_256_mask", IX86_BUILTIN_MOVDQA32_256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_QI },
32166 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loadv4si_mask, "__builtin_ia32_movdqa32_128_mask", IX86_BUILTIN_MOVDQA32_128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_QI },
32167 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loadv4df_mask, "__builtin_ia32_movapd256_mask", IX86_BUILTIN_MOVAPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_QI },
32168 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loadv2df_mask, "__builtin_ia32_movapd128_mask", IX86_BUILTIN_MOVAPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_QI },
32169 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loadv8sf_mask, "__builtin_ia32_movaps256_mask", IX86_BUILTIN_MOVAPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_QI },
32170 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loadv4sf_mask, "__builtin_ia32_movaps128_mask", IX86_BUILTIN_MOVAPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_QI },
32171 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loaddquv16hi_mask, "__builtin_ia32_movdquhi256_mask", IX86_BUILTIN_MOVDQUHI256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_HI },
32172 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_loaddquv8hi_mask, "__builtin_ia32_movdquhi128_mask", IX86_BUILTIN_MOVDQUHI128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_QI },
32173 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_loaddquv32qi_mask, "__builtin_ia32_movdquqi256_mask", IX86_BUILTIN_MOVDQUQI256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_SI },
32174 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_loaddquv16qi_mask, "__builtin_ia32_movdquqi128_mask", IX86_BUILTIN_MOVDQUQI128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_HI },
32175 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sminv4sf3_mask, "__builtin_ia32_minps_mask", IX86_BUILTIN_MINPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32176 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_smaxv4sf3_mask, "__builtin_ia32_maxps_mask", IX86_BUILTIN_MAXPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32177 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sminv2df3_mask, "__builtin_ia32_minpd_mask", IX86_BUILTIN_MINPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32178 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_smaxv2df3_mask, "__builtin_ia32_maxpd_mask", IX86_BUILTIN_MAXPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32179 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_smaxv4df3_mask, "__builtin_ia32_maxpd256_mask", IX86_BUILTIN_MAXPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32180 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_smaxv8sf3_mask, "__builtin_ia32_maxps256_mask", IX86_BUILTIN_MAXPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32181 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sminv4df3_mask, "__builtin_ia32_minpd256_mask", IX86_BUILTIN_MINPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32182 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sminv8sf3_mask, "__builtin_ia32_minps256_mask", IX86_BUILTIN_MINPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32183 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_mulv4sf3_mask, "__builtin_ia32_mulps_mask", IX86_BUILTIN_MULPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32184 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse_divv4sf3_mask, "__builtin_ia32_divps_mask", IX86_BUILTIN_DIVPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32185 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_mulv2df3_mask, "__builtin_ia32_mulpd_mask", IX86_BUILTIN_MULPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32186 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_divv2df3_mask, "__builtin_ia32_divpd_mask", IX86_BUILTIN_DIVPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32187 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_divv4df3_mask, "__builtin_ia32_divpd256_mask", IX86_BUILTIN_DIVPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32188 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_divv8sf3_mask, "__builtin_ia32_divps256_mask", IX86_BUILTIN_DIVPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32189 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_mulv4df3_mask, "__builtin_ia32_mulpd256_mask", IX86_BUILTIN_MULPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32190 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_mulv8sf3_mask, "__builtin_ia32_mulps256_mask", IX86_BUILTIN_MULPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32191 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_addv2df3_mask, "__builtin_ia32_addpd128_mask", IX86_BUILTIN_ADDPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32192 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_addv4df3_mask, "__builtin_ia32_addpd256_mask", IX86_BUILTIN_ADDPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32193 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_addv4sf3_mask, "__builtin_ia32_addps128_mask", IX86_BUILTIN_ADDPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32194 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_addv8sf3_mask, "__builtin_ia32_addps256_mask", IX86_BUILTIN_ADDPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32195 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_subv2df3_mask, "__builtin_ia32_subpd128_mask", IX86_BUILTIN_SUBPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32196 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_subv4df3_mask, "__builtin_ia32_subpd256_mask", IX86_BUILTIN_SUBPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32197 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_subv4sf3_mask, "__builtin_ia32_subps128_mask", IX86_BUILTIN_SUBPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32198 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_subv8sf3_mask, "__builtin_ia32_subps256_mask", IX86_BUILTIN_SUBPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32199 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_xorv4df3_mask, "__builtin_ia32_xorpd256_mask", IX86_BUILTIN_XORPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32200 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_xorv2df3_mask, "__builtin_ia32_xorpd128_mask", IX86_BUILTIN_XORPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32201 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_xorv8sf3_mask, "__builtin_ia32_xorps256_mask", IX86_BUILTIN_XORPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32202 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_xorv4sf3_mask, "__builtin_ia32_xorps128_mask", IX86_BUILTIN_XORPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32203 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_iorv4df3_mask, "__builtin_ia32_orpd256_mask", IX86_BUILTIN_ORPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32204 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_iorv2df3_mask, "__builtin_ia32_orpd128_mask", IX86_BUILTIN_ORPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32205 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_iorv8sf3_mask, "__builtin_ia32_orps256_mask", IX86_BUILTIN_ORPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32206 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_iorv4sf3_mask, "__builtin_ia32_orps128_mask", IX86_BUILTIN_ORPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32207 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_broadcastv8sf_mask, "__builtin_ia32_broadcastf32x2_256_mask", IX86_BUILTIN_BROADCASTF32x2_256, UNKNOWN, (int) V8SF_FTYPE_V4SF_V8SF_QI },
32208 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_broadcastv8si_mask, "__builtin_ia32_broadcasti32x2_256_mask", IX86_BUILTIN_BROADCASTI32x2_256, UNKNOWN, (int) V8SI_FTYPE_V4SI_V8SI_QI },
32209 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_broadcastv4si_mask, "__builtin_ia32_broadcasti32x2_128_mask", IX86_BUILTIN_BROADCASTI32x2_128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_QI },
32210 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_broadcastv4df_mask_1, "__builtin_ia32_broadcastf64x2_256_mask", IX86_BUILTIN_BROADCASTF64X2_256, UNKNOWN, (int) V4DF_FTYPE_V2DF_V4DF_QI },
32211 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_broadcastv4di_mask_1, "__builtin_ia32_broadcasti64x2_256_mask", IX86_BUILTIN_BROADCASTI64X2_256, UNKNOWN, (int) V4DI_FTYPE_V2DI_V4DI_QI },
32212 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_broadcastv8sf_mask_1, "__builtin_ia32_broadcastf32x4_256_mask", IX86_BUILTIN_BROADCASTF32X4_256, UNKNOWN, (int) V8SF_FTYPE_V4SF_V8SF_QI },
32213 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_broadcastv8si_mask_1, "__builtin_ia32_broadcasti32x4_256_mask", IX86_BUILTIN_BROADCASTI32X4_256, UNKNOWN, (int) V8SI_FTYPE_V4SI_V8SI_QI },
32214 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vextractf128v8sf, "__builtin_ia32_extractf32x4_256_mask", IX86_BUILTIN_EXTRACTF32X4_256, UNKNOWN, (int) V4SF_FTYPE_V8SF_INT_V4SF_QI },
32215 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vextractf128v8si, "__builtin_ia32_extracti32x4_256_mask", IX86_BUILTIN_EXTRACTI32X4_256, UNKNOWN, (int) V4SI_FTYPE_V8SI_INT_V4SI_QI },
32216 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512bw_dbpsadbwv16hi_mask, "__builtin_ia32_dbpsadbw256_mask", IX86_BUILTIN_DBPSADBW256, UNKNOWN, (int) V16HI_FTYPE_V32QI_V32QI_INT_V16HI_HI },
32217 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512bw_dbpsadbwv8hi_mask, "__builtin_ia32_dbpsadbw128_mask", IX86_BUILTIN_DBPSADBW128, UNKNOWN, (int) V8HI_FTYPE_V16QI_V16QI_INT_V8HI_QI },
32218 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_fix_truncv4dfv4di2_mask, "__builtin_ia32_cvttpd2qq256_mask", IX86_BUILTIN_CVTTPD2QQ256, UNKNOWN, (int) V4DI_FTYPE_V4DF_V4DI_QI },
32219 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_fix_truncv2dfv2di2_mask, "__builtin_ia32_cvttpd2qq128_mask", IX86_BUILTIN_CVTTPD2QQ128, UNKNOWN, (int) V2DI_FTYPE_V2DF_V2DI_QI },
32220 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufix_truncv4dfv4di2_mask, "__builtin_ia32_cvttpd2uqq256_mask", IX86_BUILTIN_CVTTPD2UQQ256, UNKNOWN, (int) V4DI_FTYPE_V4DF_V4DI_QI },
32221 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufix_truncv2dfv2di2_mask, "__builtin_ia32_cvttpd2uqq128_mask", IX86_BUILTIN_CVTTPD2UQQ128, UNKNOWN, (int) V2DI_FTYPE_V2DF_V2DI_QI },
32222 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_fix_notruncv4dfv4di2_mask, "__builtin_ia32_cvtpd2qq256_mask", IX86_BUILTIN_CVTPD2QQ256, UNKNOWN, (int) V4DI_FTYPE_V4DF_V4DI_QI },
32223 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_fix_notruncv2dfv2di2_mask, "__builtin_ia32_cvtpd2qq128_mask", IX86_BUILTIN_CVTPD2QQ128, UNKNOWN, (int) V2DI_FTYPE_V2DF_V2DI_QI },
32224 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufix_notruncv4dfv4di2_mask, "__builtin_ia32_cvtpd2uqq256_mask", IX86_BUILTIN_CVTPD2UQQ256, UNKNOWN, (int) V4DI_FTYPE_V4DF_V4DI_QI },
32225 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufix_notruncv2dfv2di2_mask, "__builtin_ia32_cvtpd2uqq128_mask", IX86_BUILTIN_CVTPD2UQQ128, UNKNOWN, (int) V2DI_FTYPE_V2DF_V2DI_QI },
32226 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufix_notruncv4dfv4si2_mask, "__builtin_ia32_cvtpd2udq256_mask", IX86_BUILTIN_CVTPD2UDQ256_MASK, UNKNOWN, (int) V4SI_FTYPE_V4DF_V4SI_QI },
32227 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufix_notruncv2dfv2si2_mask, "__builtin_ia32_cvtpd2udq128_mask", IX86_BUILTIN_CVTPD2UDQ128_MASK, UNKNOWN, (int) V4SI_FTYPE_V2DF_V4SI_QI },
32228 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_fix_truncv4sfv4di2_mask, "__builtin_ia32_cvttps2qq256_mask", IX86_BUILTIN_CVTTPS2QQ256, UNKNOWN, (int) V4DI_FTYPE_V4SF_V4DI_QI },
32229 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_fix_truncv2sfv2di2_mask, "__builtin_ia32_cvttps2qq128_mask", IX86_BUILTIN_CVTTPS2QQ128, UNKNOWN, (int) V2DI_FTYPE_V4SF_V2DI_QI },
32230 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufix_truncv4sfv4di2_mask, "__builtin_ia32_cvttps2uqq256_mask", IX86_BUILTIN_CVTTPS2UQQ256, UNKNOWN, (int) V4DI_FTYPE_V4SF_V4DI_QI },
32231 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufix_truncv2sfv2di2_mask, "__builtin_ia32_cvttps2uqq128_mask", IX86_BUILTIN_CVTTPS2UQQ128, UNKNOWN, (int) V2DI_FTYPE_V4SF_V2DI_QI },
32232 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_fix_truncv8sfv8si2_mask, "__builtin_ia32_cvttps2dq256_mask", IX86_BUILTIN_CVTTPS2DQ256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SF_V8SI_QI },
32233 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_fix_truncv4sfv4si2_mask, "__builtin_ia32_cvttps2dq128_mask", IX86_BUILTIN_CVTTPS2DQ128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SF_V4SI_QI },
32234 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufix_truncv8sfv8si2_mask, "__builtin_ia32_cvttps2udq256_mask", IX86_BUILTIN_CVTTPS2UDQ256, UNKNOWN, (int) V8SI_FTYPE_V8SF_V8SI_QI },
32235 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufix_truncv4sfv4si2_mask, "__builtin_ia32_cvttps2udq128_mask", IX86_BUILTIN_CVTTPS2UDQ128, UNKNOWN, (int) V4SI_FTYPE_V4SF_V4SI_QI },
32236 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_fix_truncv4dfv4si2_mask, "__builtin_ia32_cvttpd2dq256_mask", IX86_BUILTIN_CVTTPD2DQ256_MASK, UNKNOWN, (int) V4SI_FTYPE_V4DF_V4SI_QI },
32237 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_cvttpd2dq_mask, "__builtin_ia32_cvttpd2dq128_mask", IX86_BUILTIN_CVTTPD2DQ128_MASK, UNKNOWN, (int) V4SI_FTYPE_V2DF_V4SI_QI },
32238 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufix_truncv4dfv4si2_mask, "__builtin_ia32_cvttpd2udq256_mask", IX86_BUILTIN_CVTTPD2UDQ256_MASK, UNKNOWN, (int) V4SI_FTYPE_V4DF_V4SI_QI },
32239 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufix_truncv2dfv2si2_mask, "__builtin_ia32_cvttpd2udq128_mask", IX86_BUILTIN_CVTTPD2UDQ128_MASK, UNKNOWN, (int) V4SI_FTYPE_V2DF_V4SI_QI },
32240 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_cvtpd2dq256_mask, "__builtin_ia32_cvtpd2dq256_mask", IX86_BUILTIN_CVTPD2DQ256_MASK, UNKNOWN, (int) V4SI_FTYPE_V4DF_V4SI_QI },
32241 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_cvtpd2dq_mask, "__builtin_ia32_cvtpd2dq128_mask", IX86_BUILTIN_CVTPD2DQ128_MASK, UNKNOWN, (int) V4SI_FTYPE_V2DF_V4SI_QI },
32242 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_floatv4siv4df2_mask, "__builtin_ia32_cvtdq2pd256_mask", IX86_BUILTIN_CVTDQ2PD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4SI_V4DF_QI },
32243 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_cvtdq2pd_mask, "__builtin_ia32_cvtdq2pd128_mask", IX86_BUILTIN_CVTDQ2PD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V4SI_V2DF_QI },
32244 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufloatv4siv4df2_mask, "__builtin_ia32_cvtudq2pd256_mask", IX86_BUILTIN_CVTUDQ2PD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4SI_V4DF_QI },
32245 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufloatv2siv2df2_mask, "__builtin_ia32_cvtudq2pd128_mask", IX86_BUILTIN_CVTUDQ2PD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V4SI_V2DF_QI },
32246 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_floatv8siv8sf2_mask, "__builtin_ia32_cvtdq2ps256_mask", IX86_BUILTIN_CVTDQ2PS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SI_V8SF_QI },
32247 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_floatv4siv4sf2_mask, "__builtin_ia32_cvtdq2ps128_mask", IX86_BUILTIN_CVTDQ2PS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SI_V4SF_QI },
32248 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufloatv8siv8sf2_mask, "__builtin_ia32_cvtudq2ps256_mask", IX86_BUILTIN_CVTUDQ2PS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SI_V8SF_QI },
32249 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufloatv4siv4sf2_mask, "__builtin_ia32_cvtudq2ps128_mask", IX86_BUILTIN_CVTUDQ2PS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SI_V4SF_QI },
32250 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_cvtps2pd256_mask, "__builtin_ia32_cvtps2pd256_mask", IX86_BUILTIN_CVTPS2PD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4SF_V4DF_QI },
32251 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_cvtps2pd_mask, "__builtin_ia32_cvtps2pd128_mask", IX86_BUILTIN_CVTPS2PD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V4SF_V2DF_QI },
32252 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dupv32qi_mask, "__builtin_ia32_pbroadcastb256_mask", IX86_BUILTIN_PBROADCASTB256_MASK, UNKNOWN, (int) V32QI_FTYPE_V16QI_V32QI_SI },
32253 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dup_gprv32qi_mask, "__builtin_ia32_pbroadcastb256_gpr_mask", IX86_BUILTIN_PBROADCASTB256_GPR_MASK, UNKNOWN, (int) V32QI_FTYPE_QI_V32QI_SI },
32254 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dupv16qi_mask, "__builtin_ia32_pbroadcastb128_mask", IX86_BUILTIN_PBROADCASTB128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_HI },
32255 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dup_gprv16qi_mask, "__builtin_ia32_pbroadcastb128_gpr_mask", IX86_BUILTIN_PBROADCASTB128_GPR_MASK, UNKNOWN, (int) V16QI_FTYPE_QI_V16QI_HI },
32256 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dupv16hi_mask, "__builtin_ia32_pbroadcastw256_mask", IX86_BUILTIN_PBROADCASTW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V8HI_V16HI_HI },
32257 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dup_gprv16hi_mask, "__builtin_ia32_pbroadcastw256_gpr_mask", IX86_BUILTIN_PBROADCASTW256_GPR_MASK, UNKNOWN, (int) V16HI_FTYPE_HI_V16HI_HI },
32258 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dupv8hi_mask, "__builtin_ia32_pbroadcastw128_mask", IX86_BUILTIN_PBROADCASTW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_QI },
32259 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dup_gprv8hi_mask, "__builtin_ia32_pbroadcastw128_gpr_mask", IX86_BUILTIN_PBROADCASTW128_GPR_MASK, UNKNOWN, (int) V8HI_FTYPE_HI_V8HI_QI },
32260 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dupv8si_mask, "__builtin_ia32_pbroadcastd256_mask", IX86_BUILTIN_PBROADCASTD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V4SI_V8SI_QI },
32261 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dup_gprv8si_mask, "__builtin_ia32_pbroadcastd256_gpr_mask", IX86_BUILTIN_PBROADCASTD256_GPR_MASK, UNKNOWN, (int) V8SI_FTYPE_SI_V8SI_QI },
32262 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dupv4si_mask, "__builtin_ia32_pbroadcastd128_mask", IX86_BUILTIN_PBROADCASTD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_QI },
32263 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dup_gprv4si_mask, "__builtin_ia32_pbroadcastd128_gpr_mask", IX86_BUILTIN_PBROADCASTD128_GPR_MASK, UNKNOWN, (int) V4SI_FTYPE_SI_V4SI_QI },
32264 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dupv4di_mask, "__builtin_ia32_pbroadcastq256_mask", IX86_BUILTIN_PBROADCASTQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V2DI_V4DI_QI },
32265 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dup_gprv4di_mask, "__builtin_ia32_pbroadcastq256_gpr_mask", IX86_BUILTIN_PBROADCASTQ256_GPR_MASK, UNKNOWN, (int) V4DI_FTYPE_DI_V4DI_QI },
32266 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dupv2di_mask, "__builtin_ia32_pbroadcastq128_mask", IX86_BUILTIN_PBROADCASTQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_QI },
32267 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dup_gprv2di_mask, "__builtin_ia32_pbroadcastq128_gpr_mask", IX86_BUILTIN_PBROADCASTQ128_GPR_MASK, UNKNOWN, (int) V2DI_FTYPE_DI_V2DI_QI },
32268 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dupv8sf_mask, "__builtin_ia32_broadcastss256_mask", IX86_BUILTIN_BROADCASTSS256, UNKNOWN, (int) V8SF_FTYPE_V4SF_V8SF_QI },
32269 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dupv4sf_mask, "__builtin_ia32_broadcastss128_mask", IX86_BUILTIN_BROADCASTSS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_QI },
32270 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vec_dupv4df_mask, "__builtin_ia32_broadcastsd256_mask", IX86_BUILTIN_BROADCASTSD256, UNKNOWN, (int) V4DF_FTYPE_V2DF_V4DF_QI },
32271 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vextractf128v4df, "__builtin_ia32_extractf64x2_256_mask", IX86_BUILTIN_EXTRACTF64X2_256, UNKNOWN, (int) V2DF_FTYPE_V4DF_INT_V2DF_QI },
32272 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vextractf128v4di, "__builtin_ia32_extracti64x2_256_mask", IX86_BUILTIN_EXTRACTI64X2_256, UNKNOWN, (int) V2DI_FTYPE_V4DI_INT_V2DI_QI },
32273 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vinsertv8sf, "__builtin_ia32_insertf32x4_256_mask", IX86_BUILTIN_INSERTF32X4_256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V4SF_INT_V8SF_QI },
32274 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vinsertv8si, "__builtin_ia32_inserti32x4_256_mask", IX86_BUILTIN_INSERTI32X4_256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_INT_V8SI_QI },
32275 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_sign_extendv16qiv16hi2_mask, "__builtin_ia32_pmovsxbw256_mask", IX86_BUILTIN_PMOVSXBW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16QI_V16HI_HI },
32276 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse4_1_sign_extendv8qiv8hi2_mask, "__builtin_ia32_pmovsxbw128_mask", IX86_BUILTIN_PMOVSXBW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V16QI_V8HI_QI },
32277 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_sign_extendv8qiv8si2_mask, "__builtin_ia32_pmovsxbd256_mask", IX86_BUILTIN_PMOVSXBD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V16QI_V8SI_QI },
32278 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse4_1_sign_extendv4qiv4si2_mask, "__builtin_ia32_pmovsxbd128_mask", IX86_BUILTIN_PMOVSXBD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V16QI_V4SI_QI },
32279 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_sign_extendv4qiv4di2_mask, "__builtin_ia32_pmovsxbq256_mask", IX86_BUILTIN_PMOVSXBQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V16QI_V4DI_QI },
32280 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse4_1_sign_extendv2qiv2di2_mask, "__builtin_ia32_pmovsxbq128_mask", IX86_BUILTIN_PMOVSXBQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V16QI_V2DI_QI },
32281 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_sign_extendv8hiv8si2_mask, "__builtin_ia32_pmovsxwd256_mask", IX86_BUILTIN_PMOVSXWD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8HI_V8SI_QI },
32282 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse4_1_sign_extendv4hiv4si2_mask, "__builtin_ia32_pmovsxwd128_mask", IX86_BUILTIN_PMOVSXWD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V8HI_V4SI_QI },
32283 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_sign_extendv4hiv4di2_mask, "__builtin_ia32_pmovsxwq256_mask", IX86_BUILTIN_PMOVSXWQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V8HI_V4DI_QI },
32284 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse4_1_sign_extendv2hiv2di2_mask, "__builtin_ia32_pmovsxwq128_mask", IX86_BUILTIN_PMOVSXWQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V8HI_V2DI_QI },
32285 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_sign_extendv4siv4di2_mask, "__builtin_ia32_pmovsxdq256_mask", IX86_BUILTIN_PMOVSXDQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4SI_V4DI_QI },
32286 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse4_1_sign_extendv2siv2di2_mask, "__builtin_ia32_pmovsxdq128_mask", IX86_BUILTIN_PMOVSXDQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V4SI_V2DI_QI },
32287 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_zero_extendv16qiv16hi2_mask, "__builtin_ia32_pmovzxbw256_mask", IX86_BUILTIN_PMOVZXBW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16QI_V16HI_HI },
32288 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse4_1_zero_extendv8qiv8hi2_mask, "__builtin_ia32_pmovzxbw128_mask", IX86_BUILTIN_PMOVZXBW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V16QI_V8HI_QI },
32289 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_zero_extendv8qiv8si2_mask, "__builtin_ia32_pmovzxbd256_mask", IX86_BUILTIN_PMOVZXBD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V16QI_V8SI_QI },
32290 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse4_1_zero_extendv4qiv4si2_mask, "__builtin_ia32_pmovzxbd128_mask", IX86_BUILTIN_PMOVZXBD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V16QI_V4SI_QI },
32291 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_zero_extendv4qiv4di2_mask, "__builtin_ia32_pmovzxbq256_mask", IX86_BUILTIN_PMOVZXBQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V16QI_V4DI_QI },
32292 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse4_1_zero_extendv2qiv2di2_mask, "__builtin_ia32_pmovzxbq128_mask", IX86_BUILTIN_PMOVZXBQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V16QI_V2DI_QI },
32293 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_zero_extendv8hiv8si2_mask, "__builtin_ia32_pmovzxwd256_mask", IX86_BUILTIN_PMOVZXWD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8HI_V8SI_QI },
32294 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse4_1_zero_extendv4hiv4si2_mask, "__builtin_ia32_pmovzxwd128_mask", IX86_BUILTIN_PMOVZXWD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V8HI_V4SI_QI },
32295 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_zero_extendv4hiv4di2_mask, "__builtin_ia32_pmovzxwq256_mask", IX86_BUILTIN_PMOVZXWQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V8HI_V4DI_QI },
32296 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse4_1_zero_extendv2hiv2di2_mask, "__builtin_ia32_pmovzxwq128_mask", IX86_BUILTIN_PMOVZXWQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V8HI_V2DI_QI },
32297 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_zero_extendv4siv4di2_mask, "__builtin_ia32_pmovzxdq256_mask", IX86_BUILTIN_PMOVZXDQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4SI_V4DI_QI },
32298 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse4_1_zero_extendv2siv2di2_mask, "__builtin_ia32_pmovzxdq128_mask", IX86_BUILTIN_PMOVZXDQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V4SI_V2DI_QI },
32299 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_reducepv4df_mask, "__builtin_ia32_reducepd256_mask", IX86_BUILTIN_REDUCEPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT_V4DF_QI },
32300 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_reducepv2df_mask, "__builtin_ia32_reducepd128_mask", IX86_BUILTIN_REDUCEPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_INT_V2DF_QI },
32301 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_reducepv8sf_mask, "__builtin_ia32_reduceps256_mask", IX86_BUILTIN_REDUCEPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_INT_V8SF_QI },
32302 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_reducepv4sf_mask, "__builtin_ia32_reduceps128_mask", IX86_BUILTIN_REDUCEPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_INT_V4SF_QI },
32303 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_reducesv2df, "__builtin_ia32_reducesd", IX86_BUILTIN_REDUCESD_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
32304 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_reducesv4sf, "__builtin_ia32_reducess", IX86_BUILTIN_REDUCESS_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
32305 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_permvarv16hi_mask, "__builtin_ia32_permvarhi256_mask", IX86_BUILTIN_VPERMVARHI256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32306 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_permvarv8hi_mask, "__builtin_ia32_permvarhi128_mask", IX86_BUILTIN_VPERMVARHI128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32307 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv16hi3_mask, "__builtin_ia32_vpermt2varhi256_mask", IX86_BUILTIN_VPERMT2VARHI256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32308 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv16hi3_maskz, "__builtin_ia32_vpermt2varhi256_maskz", IX86_BUILTIN_VPERMT2VARHI256_MASKZ, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32309 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv8hi3_mask, "__builtin_ia32_vpermt2varhi128_mask", IX86_BUILTIN_VPERMT2VARHI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32310 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv8hi3_maskz, "__builtin_ia32_vpermt2varhi128_maskz", IX86_BUILTIN_VPERMT2VARHI128_MASKZ, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32311 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermi2varv16hi3_mask, "__builtin_ia32_vpermi2varhi256_mask", IX86_BUILTIN_VPERMI2VARHI256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32312 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermi2varv8hi3_mask, "__builtin_ia32_vpermi2varhi128_mask", IX86_BUILTIN_VPERMI2VARHI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32313 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_rcp14v4df_mask, "__builtin_ia32_rcp14pd256_mask", IX86_BUILTIN_RCP14PD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_QI },
32314 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_rcp14v2df_mask, "__builtin_ia32_rcp14pd128_mask", IX86_BUILTIN_RCP14PD128, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_QI },
32315 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_rcp14v8sf_mask, "__builtin_ia32_rcp14ps256_mask", IX86_BUILTIN_RCP14PS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_QI },
32316 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_rcp14v4sf_mask, "__builtin_ia32_rcp14ps128_mask", IX86_BUILTIN_RCP14PS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_QI },
32317 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_rsqrt14v4df_mask, "__builtin_ia32_rsqrt14pd256_mask", IX86_BUILTIN_RSQRT14PD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_QI },
32318 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_rsqrt14v2df_mask, "__builtin_ia32_rsqrt14pd128_mask", IX86_BUILTIN_RSQRT14PD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_QI },
32319 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_rsqrt14v8sf_mask, "__builtin_ia32_rsqrt14ps256_mask", IX86_BUILTIN_RSQRT14PS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_QI },
32320 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_rsqrt14v4sf_mask, "__builtin_ia32_rsqrt14ps128_mask", IX86_BUILTIN_RSQRT14PS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_QI },
32321 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_sqrtv4df2_mask, "__builtin_ia32_sqrtpd256_mask", IX86_BUILTIN_SQRTPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_QI },
32322 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_sqrtv2df2_mask, "__builtin_ia32_sqrtpd128_mask", IX86_BUILTIN_SQRTPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_QI },
32323 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_sqrtv8sf2_mask, "__builtin_ia32_sqrtps256_mask", IX86_BUILTIN_SQRTPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_QI },
32324 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse_sqrtv4sf2_mask, "__builtin_ia32_sqrtps128_mask", IX86_BUILTIN_SQRTPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_QI },
32325 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_addv16qi3_mask, "__builtin_ia32_paddb128_mask", IX86_BUILTIN_PADDB128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
32326 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_addv8hi3_mask, "__builtin_ia32_paddw128_mask", IX86_BUILTIN_PADDW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32327 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_addv4si3_mask, "__builtin_ia32_paddd128_mask", IX86_BUILTIN_PADDD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32328 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_addv2di3_mask, "__builtin_ia32_paddq128_mask", IX86_BUILTIN_PADDQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32329 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_subv16qi3_mask, "__builtin_ia32_psubb128_mask", IX86_BUILTIN_PSUBB128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
32330 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_subv8hi3_mask, "__builtin_ia32_psubw128_mask", IX86_BUILTIN_PSUBW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32331 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_subv4si3_mask, "__builtin_ia32_psubd128_mask", IX86_BUILTIN_PSUBD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32332 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_subv2di3_mask, "__builtin_ia32_psubq128_mask", IX86_BUILTIN_PSUBQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32333 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_ssaddv16qi3_mask, "__builtin_ia32_paddsb128_mask", IX86_BUILTIN_PADDSB128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
32334 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_ssaddv8hi3_mask, "__builtin_ia32_paddsw128_mask", IX86_BUILTIN_PADDSW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32335 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_sssubv16qi3_mask, "__builtin_ia32_psubsb128_mask", IX86_BUILTIN_PSUBSB128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
32336 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_sssubv8hi3_mask, "__builtin_ia32_psubsw128_mask", IX86_BUILTIN_PSUBSW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32337 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_usaddv16qi3_mask, "__builtin_ia32_paddusb128_mask", IX86_BUILTIN_PADDUSB128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
32338 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_usaddv8hi3_mask, "__builtin_ia32_paddusw128_mask", IX86_BUILTIN_PADDUSW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32339 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_ussubv16qi3_mask, "__builtin_ia32_psubusb128_mask", IX86_BUILTIN_PSUBUSB128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
32340 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_ussubv8hi3_mask, "__builtin_ia32_psubusw128_mask", IX86_BUILTIN_PSUBUSW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32341 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_addv32qi3_mask, "__builtin_ia32_paddb256_mask", IX86_BUILTIN_PADDB256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
32342 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_addv16hi3_mask, "__builtin_ia32_paddw256_mask", IX86_BUILTIN_PADDW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32343 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_addv8si3_mask, "__builtin_ia32_paddd256_mask", IX86_BUILTIN_PADDD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32344 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_addv4di3_mask, "__builtin_ia32_paddq256_mask", IX86_BUILTIN_PADDQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32345 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_ssaddv32qi3_mask, "__builtin_ia32_paddsb256_mask", IX86_BUILTIN_PADDSB256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
32346 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_ssaddv16hi3_mask, "__builtin_ia32_paddsw256_mask", IX86_BUILTIN_PADDSW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32347 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_usaddv32qi3_mask, "__builtin_ia32_paddusb256_mask", IX86_BUILTIN_PADDUSB256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
32348 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_usaddv16hi3_mask, "__builtin_ia32_paddusw256_mask", IX86_BUILTIN_PADDUSW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32349 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_subv32qi3_mask, "__builtin_ia32_psubb256_mask", IX86_BUILTIN_PSUBB256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
32350 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_subv16hi3_mask, "__builtin_ia32_psubw256_mask", IX86_BUILTIN_PSUBW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32351 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_subv8si3_mask, "__builtin_ia32_psubd256_mask", IX86_BUILTIN_PSUBD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32352 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_subv4di3_mask, "__builtin_ia32_psubq256_mask", IX86_BUILTIN_PSUBQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32353 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_sssubv32qi3_mask, "__builtin_ia32_psubsb256_mask", IX86_BUILTIN_PSUBSB256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
32354 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_sssubv16hi3_mask, "__builtin_ia32_psubsw256_mask", IX86_BUILTIN_PSUBSW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32355 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_ussubv32qi3_mask, "__builtin_ia32_psubusb256_mask", IX86_BUILTIN_PSUBUSB256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
32356 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_ussubv16hi3_mask, "__builtin_ia32_psubusw256_mask", IX86_BUILTIN_PSUBUSW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32357 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_shuf_f64x2_mask, "__builtin_ia32_shuf_f64x2_256_mask", IX86_BUILTIN_SHUF_F64x2_256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT_V4DF_QI },
32358 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_shuf_i64x2_mask, "__builtin_ia32_shuf_i64x2_256_mask", IX86_BUILTIN_SHUF_I64x2_256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_INT_V4DI_QI },
32359 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_shuf_i32x4_mask, "__builtin_ia32_shuf_i32x4_256_mask", IX86_BUILTIN_SHUF_I32x4_256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_INT_V8SI_QI },
32360 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_shuf_f32x4_mask, "__builtin_ia32_shuf_f32x4_256_mask", IX86_BUILTIN_SHUF_F32x4_256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT_V8SF_QI },
32361 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev8hiv8qi2_mask, "__builtin_ia32_pmovwb128_mask", IX86_BUILTIN_PMOVWB128, UNKNOWN, (int) V16QI_FTYPE_V8HI_V16QI_QI },
32362 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev16hiv16qi2_mask, "__builtin_ia32_pmovwb256_mask", IX86_BUILTIN_PMOVWB256, UNKNOWN, (int) V16QI_FTYPE_V16HI_V16QI_HI },
32363 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev8hiv8qi2_mask, "__builtin_ia32_pmovswb128_mask", IX86_BUILTIN_PMOVSWB128, UNKNOWN, (int) V16QI_FTYPE_V8HI_V16QI_QI },
32364 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev16hiv16qi2_mask, "__builtin_ia32_pmovswb256_mask", IX86_BUILTIN_PMOVSWB256, UNKNOWN, (int) V16QI_FTYPE_V16HI_V16QI_HI },
32365 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev8hiv8qi2_mask, "__builtin_ia32_pmovuswb128_mask", IX86_BUILTIN_PMOVUSWB128, UNKNOWN, (int) V16QI_FTYPE_V8HI_V16QI_QI },
32366 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev16hiv16qi2_mask, "__builtin_ia32_pmovuswb256_mask", IX86_BUILTIN_PMOVUSWB256, UNKNOWN, (int) V16QI_FTYPE_V16HI_V16QI_HI },
32367 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev4siv4qi2_mask, "__builtin_ia32_pmovdb128_mask", IX86_BUILTIN_PMOVDB128, UNKNOWN, (int) V16QI_FTYPE_V4SI_V16QI_QI },
32368 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev8siv8qi2_mask, "__builtin_ia32_pmovdb256_mask", IX86_BUILTIN_PMOVDB256, UNKNOWN, (int) V16QI_FTYPE_V8SI_V16QI_QI },
32369 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev4siv4qi2_mask, "__builtin_ia32_pmovsdb128_mask", IX86_BUILTIN_PMOVSDB128, UNKNOWN, (int) V16QI_FTYPE_V4SI_V16QI_QI },
32370 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev8siv8qi2_mask, "__builtin_ia32_pmovsdb256_mask", IX86_BUILTIN_PMOVSDB256, UNKNOWN, (int) V16QI_FTYPE_V8SI_V16QI_QI },
32371 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev4siv4qi2_mask, "__builtin_ia32_pmovusdb128_mask", IX86_BUILTIN_PMOVUSDB128, UNKNOWN, (int) V16QI_FTYPE_V4SI_V16QI_QI },
32372 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev8siv8qi2_mask, "__builtin_ia32_pmovusdb256_mask", IX86_BUILTIN_PMOVUSDB256, UNKNOWN, (int) V16QI_FTYPE_V8SI_V16QI_QI },
32373 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev4siv4hi2_mask, "__builtin_ia32_pmovdw128_mask", IX86_BUILTIN_PMOVDW128, UNKNOWN, (int) V8HI_FTYPE_V4SI_V8HI_QI },
32374 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev8siv8hi2_mask, "__builtin_ia32_pmovdw256_mask", IX86_BUILTIN_PMOVDW256, UNKNOWN, (int) V8HI_FTYPE_V8SI_V8HI_QI },
32375 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev4siv4hi2_mask, "__builtin_ia32_pmovsdw128_mask", IX86_BUILTIN_PMOVSDW128, UNKNOWN, (int) V8HI_FTYPE_V4SI_V8HI_QI },
32376 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev8siv8hi2_mask, "__builtin_ia32_pmovsdw256_mask", IX86_BUILTIN_PMOVSDW256, UNKNOWN, (int) V8HI_FTYPE_V8SI_V8HI_QI },
32377 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev4siv4hi2_mask, "__builtin_ia32_pmovusdw128_mask", IX86_BUILTIN_PMOVUSDW128, UNKNOWN, (int) V8HI_FTYPE_V4SI_V8HI_QI },
32378 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev8siv8hi2_mask, "__builtin_ia32_pmovusdw256_mask", IX86_BUILTIN_PMOVUSDW256, UNKNOWN, (int) V8HI_FTYPE_V8SI_V8HI_QI },
32379 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev2div2qi2_mask, "__builtin_ia32_pmovqb128_mask", IX86_BUILTIN_PMOVQB128, UNKNOWN, (int) V16QI_FTYPE_V2DI_V16QI_QI },
32380 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev4div4qi2_mask, "__builtin_ia32_pmovqb256_mask", IX86_BUILTIN_PMOVQB256, UNKNOWN, (int) V16QI_FTYPE_V4DI_V16QI_QI },
32381 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev2div2qi2_mask, "__builtin_ia32_pmovsqb128_mask", IX86_BUILTIN_PMOVSQB128, UNKNOWN, (int) V16QI_FTYPE_V2DI_V16QI_QI },
32382 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev4div4qi2_mask, "__builtin_ia32_pmovsqb256_mask", IX86_BUILTIN_PMOVSQB256, UNKNOWN, (int) V16QI_FTYPE_V4DI_V16QI_QI },
32383 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev2div2qi2_mask, "__builtin_ia32_pmovusqb128_mask", IX86_BUILTIN_PMOVUSQB128, UNKNOWN, (int) V16QI_FTYPE_V2DI_V16QI_QI },
32384 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev4div4qi2_mask, "__builtin_ia32_pmovusqb256_mask", IX86_BUILTIN_PMOVUSQB256, UNKNOWN, (int) V16QI_FTYPE_V4DI_V16QI_QI },
32385 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev2div2hi2_mask, "__builtin_ia32_pmovqw128_mask", IX86_BUILTIN_PMOVQW128, UNKNOWN, (int) V8HI_FTYPE_V2DI_V8HI_QI },
32386 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev4div4hi2_mask, "__builtin_ia32_pmovqw256_mask", IX86_BUILTIN_PMOVQW256, UNKNOWN, (int) V8HI_FTYPE_V4DI_V8HI_QI },
32387 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev2div2hi2_mask, "__builtin_ia32_pmovsqw128_mask", IX86_BUILTIN_PMOVSQW128, UNKNOWN, (int) V8HI_FTYPE_V2DI_V8HI_QI },
32388 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev4div4hi2_mask, "__builtin_ia32_pmovsqw256_mask", IX86_BUILTIN_PMOVSQW256, UNKNOWN, (int) V8HI_FTYPE_V4DI_V8HI_QI },
32389 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev2div2hi2_mask, "__builtin_ia32_pmovusqw128_mask", IX86_BUILTIN_PMOVUSQW128, UNKNOWN, (int) V8HI_FTYPE_V2DI_V8HI_QI },
32390 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev4div4hi2_mask, "__builtin_ia32_pmovusqw256_mask", IX86_BUILTIN_PMOVUSQW256, UNKNOWN, (int) V8HI_FTYPE_V4DI_V8HI_QI },
32391 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev2div2si2_mask, "__builtin_ia32_pmovqd128_mask", IX86_BUILTIN_PMOVQD128, UNKNOWN, (int) V4SI_FTYPE_V2DI_V4SI_QI },
32392 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_truncatev4div4si2_mask, "__builtin_ia32_pmovqd256_mask", IX86_BUILTIN_PMOVQD256, UNKNOWN, (int) V4SI_FTYPE_V4DI_V4SI_QI },
32393 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev2div2si2_mask, "__builtin_ia32_pmovsqd128_mask", IX86_BUILTIN_PMOVSQD128, UNKNOWN, (int) V4SI_FTYPE_V2DI_V4SI_QI },
32394 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ss_truncatev4div4si2_mask, "__builtin_ia32_pmovsqd256_mask", IX86_BUILTIN_PMOVSQD256, UNKNOWN, (int) V4SI_FTYPE_V4DI_V4SI_QI },
32395 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev2div2si2_mask, "__builtin_ia32_pmovusqd128_mask", IX86_BUILTIN_PMOVUSQD128, UNKNOWN, (int) V4SI_FTYPE_V2DI_V4SI_QI },
32396 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_us_truncatev4div4si2_mask, "__builtin_ia32_pmovusqd256_mask", IX86_BUILTIN_PMOVUSQD256, UNKNOWN, (int) V4SI_FTYPE_V4DI_V4SI_QI },
32397 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_rangepv4df_mask, "__builtin_ia32_rangepd256_mask", IX86_BUILTIN_RANGEPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT_V4DF_QI },
32398 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_rangepv2df_mask, "__builtin_ia32_rangepd128_mask", IX86_BUILTIN_RANGEPD128, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT_V2DF_QI },
32399 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_rangepv8sf_mask, "__builtin_ia32_rangeps256_mask", IX86_BUILTIN_RANGEPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT_V8SF_QI },
32400 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_rangepv4sf_mask, "__builtin_ia32_rangeps128_mask", IX86_BUILTIN_RANGEPS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT_V4SF_QI },
32401 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_getexpv8sf_mask, "__builtin_ia32_getexpps256_mask", IX86_BUILTIN_GETEXPPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_QI },
32402 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_getexpv4df_mask, "__builtin_ia32_getexppd256_mask", IX86_BUILTIN_GETEXPPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_QI },
32403 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_getexpv4sf_mask, "__builtin_ia32_getexpps128_mask", IX86_BUILTIN_GETEXPPS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_QI },
32404 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_getexpv2df_mask, "__builtin_ia32_getexppd128_mask", IX86_BUILTIN_GETEXPPD128, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_QI },
32405 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fixupimmv4df_mask, "__builtin_ia32_fixupimmpd256_mask", IX86_BUILTIN_FIXUPIMMPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DI_INT_QI },
32406 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fixupimmv4df_maskz, "__builtin_ia32_fixupimmpd256_maskz", IX86_BUILTIN_FIXUPIMMPD256_MASKZ, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DI_INT_QI },
32407 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fixupimmv8sf_mask, "__builtin_ia32_fixupimmps256_mask", IX86_BUILTIN_FIXUPIMMPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SI_INT_QI },
32408 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fixupimmv8sf_maskz, "__builtin_ia32_fixupimmps256_maskz", IX86_BUILTIN_FIXUPIMMPS256_MASKZ, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SI_INT_QI },
32409 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fixupimmv2df_mask, "__builtin_ia32_fixupimmpd128_mask", IX86_BUILTIN_FIXUPIMMPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DI_INT_QI },
32410 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fixupimmv2df_maskz, "__builtin_ia32_fixupimmpd128_maskz", IX86_BUILTIN_FIXUPIMMPD128_MASKZ, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DI_INT_QI },
32411 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fixupimmv4sf_mask, "__builtin_ia32_fixupimmps128_mask", IX86_BUILTIN_FIXUPIMMPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SI_INT_QI },
32412 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fixupimmv4sf_maskz, "__builtin_ia32_fixupimmps128_maskz", IX86_BUILTIN_FIXUPIMMPS128_MASKZ, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SI_INT_QI },
32413 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_absv4di2_mask, "__builtin_ia32_pabsq256_mask", IX86_BUILTIN_PABSQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_QI },
32414 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_absv2di2_mask, "__builtin_ia32_pabsq128_mask", IX86_BUILTIN_PABSQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_QI },
32415 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_absv8si2_mask, "__builtin_ia32_pabsd256_mask", IX86_BUILTIN_PABSD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_QI },
32416 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_absv4si2_mask, "__builtin_ia32_pabsd128_mask", IX86_BUILTIN_PABSD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_QI },
32417 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_pmulhrswv16hi3_mask , "__builtin_ia32_pmulhrsw256_mask", IX86_BUILTIN_PMULHRSW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32418 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ssse3_pmulhrswv8hi3_mask, "__builtin_ia32_pmulhrsw128_mask", IX86_BUILTIN_PMULHRSW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32419 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_umulv8hi3_highpart_mask, "__builtin_ia32_pmulhuw128_mask", IX86_BUILTIN_PMULHUW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32420 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_umulv16hi3_highpart_mask, "__builtin_ia32_pmulhuw256_mask" , IX86_BUILTIN_PMULHUW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32421 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_smulv16hi3_highpart_mask, "__builtin_ia32_pmulhw256_mask" , IX86_BUILTIN_PMULHW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32422 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_smulv8hi3_highpart_mask, "__builtin_ia32_pmulhw128_mask", IX86_BUILTIN_PMULHW128_MASK, UNKNOWN,(int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32423 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_mulv16hi3_mask, "__builtin_ia32_pmullw256_mask" , IX86_BUILTIN_PMULLW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32424 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_mulv8hi3_mask, "__builtin_ia32_pmullw128_mask", IX86_BUILTIN_PMULLW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32425 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_mulv4di3_mask, "__builtin_ia32_pmullq256_mask", IX86_BUILTIN_PMULLQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32426 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_mulv2di3_mask, "__builtin_ia32_pmullq128_mask", IX86_BUILTIN_PMULLQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32427 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_andv4df3_mask, "__builtin_ia32_andpd256_mask", IX86_BUILTIN_ANDPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32428 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_andv2df3_mask, "__builtin_ia32_andpd128_mask", IX86_BUILTIN_ANDPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32429 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_andv8sf3_mask, "__builtin_ia32_andps256_mask", IX86_BUILTIN_ANDPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32430 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_andv4sf3_mask, "__builtin_ia32_andps128_mask", IX86_BUILTIN_ANDPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32431 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_andnotv4df3_mask, "__builtin_ia32_andnpd256_mask", IX86_BUILTIN_ANDNPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32432 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_andnotv2df3_mask, "__builtin_ia32_andnpd128_mask", IX86_BUILTIN_ANDNPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32433 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_andnotv8sf3_mask, "__builtin_ia32_andnps256_mask", IX86_BUILTIN_ANDNPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32434 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse_andnotv4sf3_mask, "__builtin_ia32_andnps128_mask", IX86_BUILTIN_ANDNPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32435 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashlv8hi3_mask, "__builtin_ia32_psllwi128_mask", IX86_BUILTIN_PSLLWI128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_INT_V8HI_QI },
32436 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashlv4si3_mask, "__builtin_ia32_pslldi128_mask", IX86_BUILTIN_PSLLDI128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_INT_V4SI_QI },
32437 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashlv2di3_mask, "__builtin_ia32_psllqi128_mask", IX86_BUILTIN_PSLLQI128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT_V2DI_QI },
32438 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashlv8hi3_mask, "__builtin_ia32_psllw128_mask", IX86_BUILTIN_PSLLW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32439 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashlv4si3_mask, "__builtin_ia32_pslld128_mask", IX86_BUILTIN_PSLLD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32440 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashlv2di3_mask, "__builtin_ia32_psllq128_mask", IX86_BUILTIN_PSLLQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32441 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashlv16hi3_mask, "__builtin_ia32_psllwi256_mask", IX86_BUILTIN_PSLLWI256_MASK , UNKNOWN, (int) V16HI_FTYPE_V16HI_INT_V16HI_HI },
32442 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashlv16hi3_mask, "__builtin_ia32_psllw256_mask", IX86_BUILTIN_PSLLW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_V16HI_HI },
32443 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashlv8si3_mask, "__builtin_ia32_pslldi256_mask", IX86_BUILTIN_PSLLDI256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_INT_V8SI_QI },
32444 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashlv8si3_mask, "__builtin_ia32_pslld256_mask", IX86_BUILTIN_PSLLD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_V8SI_QI },
32445 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashlv4di3_mask, "__builtin_ia32_psllqi256_mask", IX86_BUILTIN_PSLLQI256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_V4DI_QI },
32446 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashlv4di3_mask, "__builtin_ia32_psllq256_mask", IX86_BUILTIN_PSLLQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_V4DI_QI },
32447 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashrv4si3_mask, "__builtin_ia32_psradi128_mask", IX86_BUILTIN_PSRADI128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_INT_V4SI_QI },
32448 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashrv4si3_mask, "__builtin_ia32_psrad128_mask", IX86_BUILTIN_PSRAD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32449 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashrv8si3_mask, "__builtin_ia32_psradi256_mask", IX86_BUILTIN_PSRADI256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_INT_V8SI_QI },
32450 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashrv8si3_mask, "__builtin_ia32_psrad256_mask", IX86_BUILTIN_PSRAD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_V8SI_QI },
32451 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashrv2di3_mask, "__builtin_ia32_psraqi128_mask", IX86_BUILTIN_PSRAQI128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT_V2DI_QI },
32452 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashrv2di3_mask, "__builtin_ia32_psraq128_mask", IX86_BUILTIN_PSRAQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32453 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashrv4di3_mask, "__builtin_ia32_psraqi256_mask", IX86_BUILTIN_PSRAQI256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_V4DI_QI },
32454 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashrv4di3_mask, "__builtin_ia32_psraq256_mask", IX86_BUILTIN_PSRAQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_V4DI_QI },
32455 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_andv8si3_mask, "__builtin_ia32_pandd256_mask", IX86_BUILTIN_PANDD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32456 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_andv4si3_mask, "__builtin_ia32_pandd128_mask", IX86_BUILTIN_PANDD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32457 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_lshrv4si3_mask, "__builtin_ia32_psrldi128_mask", IX86_BUILTIN_PSRLDI128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_INT_V4SI_QI },
32458 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_lshrv4si3_mask, "__builtin_ia32_psrld128_mask", IX86_BUILTIN_PSRLD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32459 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_lshrv8si3_mask, "__builtin_ia32_psrldi256_mask", IX86_BUILTIN_PSRLDI256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_INT_V8SI_QI },
32460 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_lshrv8si3_mask, "__builtin_ia32_psrld256_mask", IX86_BUILTIN_PSRLD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_V8SI_QI },
32461 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_lshrv2di3_mask, "__builtin_ia32_psrlqi128_mask", IX86_BUILTIN_PSRLQI128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT_V2DI_QI },
32462 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_lshrv2di3_mask, "__builtin_ia32_psrlq128_mask", IX86_BUILTIN_PSRLQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32463 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_lshrv4di3_mask, "__builtin_ia32_psrlqi256_mask", IX86_BUILTIN_PSRLQI256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_V4DI_QI },
32464 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_lshrv4di3_mask, "__builtin_ia32_psrlq256_mask", IX86_BUILTIN_PSRLQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_V4DI_QI },
32465 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_andv4di3_mask, "__builtin_ia32_pandq256_mask", IX86_BUILTIN_PANDQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32466 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_andv2di3_mask, "__builtin_ia32_pandq128_mask", IX86_BUILTIN_PANDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32467 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_andnotv8si3_mask, "__builtin_ia32_pandnd256_mask", IX86_BUILTIN_PANDND256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32468 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_andnotv4si3_mask, "__builtin_ia32_pandnd128_mask", IX86_BUILTIN_PANDND128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32469 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_andnotv4di3_mask, "__builtin_ia32_pandnq256_mask", IX86_BUILTIN_PANDNQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32470 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_andnotv2di3_mask, "__builtin_ia32_pandnq128_mask", IX86_BUILTIN_PANDNQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32471 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_iorv8si3_mask, "__builtin_ia32_pord256_mask", IX86_BUILTIN_PORD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32472 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_iorv4si3_mask, "__builtin_ia32_pord128_mask", IX86_BUILTIN_PORD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32473 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_iorv4di3_mask, "__builtin_ia32_porq256_mask", IX86_BUILTIN_PORQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32474 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_iorv2di3_mask, "__builtin_ia32_porq128_mask", IX86_BUILTIN_PORQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32475 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_xorv8si3_mask, "__builtin_ia32_pxord256_mask", IX86_BUILTIN_PXORD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32476 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_xorv4si3_mask, "__builtin_ia32_pxord128_mask", IX86_BUILTIN_PXORD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32477 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_xorv4di3_mask, "__builtin_ia32_pxorq256_mask", IX86_BUILTIN_PXORQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32478 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_xorv2di3_mask, "__builtin_ia32_pxorq128_mask", IX86_BUILTIN_PXORQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32479 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_packsswb_mask, "__builtin_ia32_packsswb256_mask", IX86_BUILTIN_PACKSSWB256_MASK, UNKNOWN, (int) V32QI_FTYPE_V16HI_V16HI_V32QI_SI },
32480 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_packsswb_mask, "__builtin_ia32_packsswb128_mask", IX86_BUILTIN_PACKSSWB128_MASK, UNKNOWN, (int) V16QI_FTYPE_V8HI_V8HI_V16QI_HI },
32481 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_packuswb_mask, "__builtin_ia32_packuswb256_mask", IX86_BUILTIN_PACKUSWB256_MASK, UNKNOWN, (int) V32QI_FTYPE_V16HI_V16HI_V32QI_SI },
32482 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_packuswb_mask, "__builtin_ia32_packuswb128_mask", IX86_BUILTIN_PACKUSWB128_MASK, UNKNOWN, (int) V16QI_FTYPE_V8HI_V8HI_V16QI_HI },
32483 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rndscalev8sf_mask, "__builtin_ia32_rndscaleps_256_mask", IX86_BUILTIN_RNDSCALEPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_INT_V8SF_QI },
32484 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rndscalev4df_mask, "__builtin_ia32_rndscalepd_256_mask", IX86_BUILTIN_RNDSCALEPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT_V4DF_QI },
32485 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rndscalev4sf_mask, "__builtin_ia32_rndscaleps_128_mask", IX86_BUILTIN_RNDSCALEPS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_INT_V4SF_QI },
32486 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rndscalev2df_mask, "__builtin_ia32_rndscalepd_128_mask", IX86_BUILTIN_RNDSCALEPD128, UNKNOWN, (int) V2DF_FTYPE_V2DF_INT_V2DF_QI },
32487 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vternlogv4di_mask, "__builtin_ia32_pternlogq256_mask", IX86_BUILTIN_VTERNLOGQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_INT_QI },
32488 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vternlogv4di_maskz, "__builtin_ia32_pternlogq256_maskz", IX86_BUILTIN_VTERNLOGQ256_MASKZ, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_INT_QI },
32489 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vternlogv8si_mask, "__builtin_ia32_pternlogd256_mask", IX86_BUILTIN_VTERNLOGD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_INT_QI },
32490 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vternlogv8si_maskz, "__builtin_ia32_pternlogd256_maskz", IX86_BUILTIN_VTERNLOGD256_MASKZ, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_INT_QI },
32491 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vternlogv2di_mask, "__builtin_ia32_pternlogq128_mask", IX86_BUILTIN_VTERNLOGQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_INT_QI },
32492 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vternlogv2di_maskz, "__builtin_ia32_pternlogq128_maskz", IX86_BUILTIN_VTERNLOGQ128_MASKZ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_INT_QI },
32493 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vternlogv4si_mask, "__builtin_ia32_pternlogd128_mask", IX86_BUILTIN_VTERNLOGD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_INT_QI },
32494 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vternlogv4si_maskz, "__builtin_ia32_pternlogd128_maskz", IX86_BUILTIN_VTERNLOGD128_MASKZ, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_INT_QI },
32495 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_scalefv4df_mask, "__builtin_ia32_scalefpd256_mask", IX86_BUILTIN_SCALEFPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32496 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_scalefv8sf_mask, "__builtin_ia32_scalefps256_mask", IX86_BUILTIN_SCALEFPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32497 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_scalefv2df_mask, "__builtin_ia32_scalefpd128_mask", IX86_BUILTIN_SCALEFPD128, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32498 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_scalefv4sf_mask, "__builtin_ia32_scalefps128_mask", IX86_BUILTIN_SCALEFPS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32499 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmadd_v4df_mask, "__builtin_ia32_vfmaddpd256_mask", IX86_BUILTIN_VFMADDPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32500 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmadd_v4df_mask3, "__builtin_ia32_vfmaddpd256_mask3", IX86_BUILTIN_VFMADDPD256_MASK3, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32501 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmadd_v4df_maskz, "__builtin_ia32_vfmaddpd256_maskz", IX86_BUILTIN_VFMADDPD256_MASKZ, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32502 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmadd_v2df_mask, "__builtin_ia32_vfmaddpd128_mask", IX86_BUILTIN_VFMADDPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32503 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmadd_v2df_mask3, "__builtin_ia32_vfmaddpd128_mask3", IX86_BUILTIN_VFMADDPD128_MASK3, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32504 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmadd_v2df_maskz, "__builtin_ia32_vfmaddpd128_maskz", IX86_BUILTIN_VFMADDPD128_MASKZ, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32505 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmadd_v8sf_mask, "__builtin_ia32_vfmaddps256_mask", IX86_BUILTIN_VFMADDPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32506 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmadd_v8sf_mask3, "__builtin_ia32_vfmaddps256_mask3", IX86_BUILTIN_VFMADDPS256_MASK3, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32507 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmadd_v8sf_maskz, "__builtin_ia32_vfmaddps256_maskz", IX86_BUILTIN_VFMADDPS256_MASKZ, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32508 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmadd_v4sf_mask, "__builtin_ia32_vfmaddps128_mask", IX86_BUILTIN_VFMADDPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32509 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmadd_v4sf_mask3, "__builtin_ia32_vfmaddps128_mask3", IX86_BUILTIN_VFMADDPS128_MASK3, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32510 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmadd_v4sf_maskz, "__builtin_ia32_vfmaddps128_maskz", IX86_BUILTIN_VFMADDPS128_MASKZ, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32511 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmsub_v4df_mask3, "__builtin_ia32_vfmsubpd256_mask3", IX86_BUILTIN_VFMSUBPD256_MASK3, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32512 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmsub_v2df_mask3, "__builtin_ia32_vfmsubpd128_mask3", IX86_BUILTIN_VFMSUBPD128_MASK3, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32513 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmsub_v8sf_mask3, "__builtin_ia32_vfmsubps256_mask3", IX86_BUILTIN_VFMSUBPS256_MASK3, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32514 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmsub_v4sf_mask3, "__builtin_ia32_vfmsubps128_mask3", IX86_BUILTIN_VFMSUBPS128_MASK3, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32515 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fnmadd_v4df_mask, "__builtin_ia32_vfnmaddpd256_mask", IX86_BUILTIN_VFNMADDPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32516 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fnmadd_v2df_mask, "__builtin_ia32_vfnmaddpd128_mask", IX86_BUILTIN_VFNMADDPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32517 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fnmadd_v8sf_mask, "__builtin_ia32_vfnmaddps256_mask", IX86_BUILTIN_VFNMADDPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32518 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fnmadd_v4sf_mask, "__builtin_ia32_vfnmaddps128_mask", IX86_BUILTIN_VFNMADDPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32519 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fnmsub_v4df_mask, "__builtin_ia32_vfnmsubpd256_mask", IX86_BUILTIN_VFNMSUBPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32520 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fnmsub_v4df_mask3, "__builtin_ia32_vfnmsubpd256_mask3", IX86_BUILTIN_VFNMSUBPD256_MASK3, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32521 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fnmsub_v2df_mask, "__builtin_ia32_vfnmsubpd128_mask", IX86_BUILTIN_VFNMSUBPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32522 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fnmsub_v2df_mask3, "__builtin_ia32_vfnmsubpd128_mask3", IX86_BUILTIN_VFNMSUBPD128_MASK3, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32523 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fnmsub_v8sf_mask, "__builtin_ia32_vfnmsubps256_mask", IX86_BUILTIN_VFNMSUBPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32524 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fnmsub_v8sf_mask3, "__builtin_ia32_vfnmsubps256_mask3", IX86_BUILTIN_VFNMSUBPS256_MASK3, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32525 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fnmsub_v4sf_mask, "__builtin_ia32_vfnmsubps128_mask", IX86_BUILTIN_VFNMSUBPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32526 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fnmsub_v4sf_mask3, "__builtin_ia32_vfnmsubps128_mask3", IX86_BUILTIN_VFNMSUBPS128_MASK3, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32527 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmaddsub_v4df_mask, "__builtin_ia32_vfmaddsubpd256_mask", IX86_BUILTIN_VFMADDSUBPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32528 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmaddsub_v4df_mask3, "__builtin_ia32_vfmaddsubpd256_mask3", IX86_BUILTIN_VFMADDSUBPD256_MASK3, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32529 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmaddsub_v4df_maskz, "__builtin_ia32_vfmaddsubpd256_maskz", IX86_BUILTIN_VFMADDSUBPD256_MASKZ, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32530 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmaddsub_v2df_mask, "__builtin_ia32_vfmaddsubpd128_mask", IX86_BUILTIN_VFMADDSUBPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32531 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmaddsub_v2df_mask3, "__builtin_ia32_vfmaddsubpd128_mask3", IX86_BUILTIN_VFMADDSUBPD128_MASK3, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32532 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmaddsub_v2df_maskz, "__builtin_ia32_vfmaddsubpd128_maskz", IX86_BUILTIN_VFMADDSUBPD128_MASKZ, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32533 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmaddsub_v8sf_mask, "__builtin_ia32_vfmaddsubps256_mask", IX86_BUILTIN_VFMADDSUBPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32534 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmaddsub_v8sf_mask3, "__builtin_ia32_vfmaddsubps256_mask3", IX86_BUILTIN_VFMADDSUBPS256_MASK3, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32535 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmaddsub_v8sf_maskz, "__builtin_ia32_vfmaddsubps256_maskz", IX86_BUILTIN_VFMADDSUBPS256_MASKZ, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32536 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmaddsub_v4sf_mask, "__builtin_ia32_vfmaddsubps128_mask", IX86_BUILTIN_VFMADDSUBPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32537 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmaddsub_v4sf_mask3, "__builtin_ia32_vfmaddsubps128_mask3", IX86_BUILTIN_VFMADDSUBPS128_MASK3, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32538 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmaddsub_v4sf_maskz, "__builtin_ia32_vfmaddsubps128_maskz", IX86_BUILTIN_VFMADDSUBPS128_MASKZ, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32539 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmsubadd_v4df_mask3, "__builtin_ia32_vfmsubaddpd256_mask3", IX86_BUILTIN_VFMSUBADDPD256_MASK3, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32540 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmsubadd_v2df_mask3, "__builtin_ia32_vfmsubaddpd128_mask3", IX86_BUILTIN_VFMSUBADDPD128_MASK3, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32541 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmsubadd_v8sf_mask3, "__builtin_ia32_vfmsubaddps256_mask3", IX86_BUILTIN_VFMSUBADDPS256_MASK3, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32542 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_fmsubadd_v4sf_mask3, "__builtin_ia32_vfmsubaddps128_mask3", IX86_BUILTIN_VFMSUBADDPS128_MASK3, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32543 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vinsertv4df, "__builtin_ia32_insertf64x2_256_mask", IX86_BUILTIN_INSERTF64X2_256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V2DF_INT_V4DF_QI },
32544 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vinsertv4di, "__builtin_ia32_inserti64x2_256_mask", IX86_BUILTIN_INSERTI64X2_256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_INT_V4DI_QI },
32545 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ashrvv16hi_mask, "__builtin_ia32_psrav16hi_mask", IX86_BUILTIN_PSRAVV16HI, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32546 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ashrvv8hi_mask, "__builtin_ia32_psrav8hi_mask", IX86_BUILTIN_PSRAVV8HI, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32547 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512bw_pmaddubsw512v16hi_mask, "__builtin_ia32_pmaddubsw256_mask", IX86_BUILTIN_PMADDUBSW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V32QI_V32QI_V16HI_HI },
32548 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512bw_pmaddubsw512v8hi_mask, "__builtin_ia32_pmaddubsw128_mask", IX86_BUILTIN_PMADDUBSW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V16QI_V16QI_V8HI_QI },
32549 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512bw_pmaddwd512v16hi_mask, "__builtin_ia32_pmaddwd256_mask", IX86_BUILTIN_PMADDWD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V16HI_V16HI_V8SI_QI },
32550 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512bw_pmaddwd512v8hi_mask, "__builtin_ia32_pmaddwd128_mask", IX86_BUILTIN_PMADDWD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V8HI_V8HI_V4SI_QI },
32551 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_lshrvv16hi_mask, "__builtin_ia32_psrlv16hi_mask", IX86_BUILTIN_PSRLVV16HI, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32552 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_lshrvv8hi_mask, "__builtin_ia32_psrlv8hi_mask", IX86_BUILTIN_PSRLVV8HI, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32553 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_fix_notruncv8sfv8si_mask, "__builtin_ia32_cvtps2dq256_mask", IX86_BUILTIN_CVTPS2DQ256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SF_V8SI_QI },
32554 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_fix_notruncv4sfv4si_mask, "__builtin_ia32_cvtps2dq128_mask", IX86_BUILTIN_CVTPS2DQ128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SF_V4SI_QI },
32555 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ufix_notruncv8sfv8si_mask, "__builtin_ia32_cvtps2udq256_mask", IX86_BUILTIN_CVTPS2UDQ256, UNKNOWN, (int) V8SI_FTYPE_V8SF_V8SI_QI },
32556 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ufix_notruncv4sfv4si_mask, "__builtin_ia32_cvtps2udq128_mask", IX86_BUILTIN_CVTPS2UDQ128, UNKNOWN, (int) V4SI_FTYPE_V4SF_V4SI_QI },
32557 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_cvtps2qqv4di_mask, "__builtin_ia32_cvtps2qq256_mask", IX86_BUILTIN_CVTPS2QQ256, UNKNOWN, (int) V4DI_FTYPE_V4SF_V4DI_QI },
32558 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_cvtps2qqv2di_mask, "__builtin_ia32_cvtps2qq128_mask", IX86_BUILTIN_CVTPS2QQ128, UNKNOWN, (int) V2DI_FTYPE_V4SF_V2DI_QI },
32559 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_cvtps2uqqv4di_mask, "__builtin_ia32_cvtps2uqq256_mask", IX86_BUILTIN_CVTPS2UQQ256, UNKNOWN, (int) V4DI_FTYPE_V4SF_V4DI_QI },
32560 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_cvtps2uqqv2di_mask, "__builtin_ia32_cvtps2uqq128_mask", IX86_BUILTIN_CVTPS2UQQ128, UNKNOWN, (int) V2DI_FTYPE_V4SF_V2DI_QI },
32561 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_getmantv8sf_mask, "__builtin_ia32_getmantps256_mask", IX86_BUILTIN_GETMANTPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_INT_V8SF_QI },
32562 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_getmantv4sf_mask, "__builtin_ia32_getmantps128_mask", IX86_BUILTIN_GETMANTPS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_INT_V4SF_QI },
32563 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_getmantv4df_mask, "__builtin_ia32_getmantpd256_mask", IX86_BUILTIN_GETMANTPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT_V4DF_QI },
32564 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_getmantv2df_mask, "__builtin_ia32_getmantpd128_mask", IX86_BUILTIN_GETMANTPD128, UNKNOWN, (int) V2DF_FTYPE_V2DF_INT_V2DF_QI },
32565 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_movddup256_mask, "__builtin_ia32_movddup256_mask", IX86_BUILTIN_MOVDDUP256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_QI },
32566 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_vec_dupv2df_mask, "__builtin_ia32_movddup128_mask", IX86_BUILTIN_MOVDDUP128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_QI },
32567 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_movshdup256_mask, "__builtin_ia32_movshdup256_mask", IX86_BUILTIN_MOVSHDUP256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_QI },
32568 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse3_movshdup_mask, "__builtin_ia32_movshdup128_mask", IX86_BUILTIN_MOVSHDUP128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_QI },
32569 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_movsldup256_mask, "__builtin_ia32_movsldup256_mask", IX86_BUILTIN_MOVSLDUP256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_QI },
32570 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse3_movsldup_mask, "__builtin_ia32_movsldup128_mask", IX86_BUILTIN_MOVSLDUP128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_QI },
32571 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_floatv4div4sf2_mask, "__builtin_ia32_cvtqq2ps256_mask", IX86_BUILTIN_CVTQQ2PS256, UNKNOWN, (int) V4SF_FTYPE_V4DI_V4SF_QI },
32572 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_floatv2div2sf2_mask, "__builtin_ia32_cvtqq2ps128_mask", IX86_BUILTIN_CVTQQ2PS128, UNKNOWN, (int) V4SF_FTYPE_V2DI_V4SF_QI },
32573 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufloatv4div4sf2_mask, "__builtin_ia32_cvtuqq2ps256_mask", IX86_BUILTIN_CVTUQQ2PS256, UNKNOWN, (int) V4SF_FTYPE_V4DI_V4SF_QI },
32574 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufloatv2div2sf2_mask, "__builtin_ia32_cvtuqq2ps128_mask", IX86_BUILTIN_CVTUQQ2PS128, UNKNOWN, (int) V4SF_FTYPE_V2DI_V4SF_QI },
32575 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_floatv4div4df2_mask, "__builtin_ia32_cvtqq2pd256_mask", IX86_BUILTIN_CVTQQ2PD256, UNKNOWN, (int) V4DF_FTYPE_V4DI_V4DF_QI },
32576 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_floatv2div2df2_mask, "__builtin_ia32_cvtqq2pd128_mask", IX86_BUILTIN_CVTQQ2PD128, UNKNOWN, (int) V2DF_FTYPE_V2DI_V2DF_QI },
32577 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufloatv4div4df2_mask, "__builtin_ia32_cvtuqq2pd256_mask", IX86_BUILTIN_CVTUQQ2PD256, UNKNOWN, (int) V4DF_FTYPE_V4DI_V4DF_QI },
32578 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ufloatv2div2df2_mask, "__builtin_ia32_cvtuqq2pd128_mask", IX86_BUILTIN_CVTUQQ2PD128, UNKNOWN, (int) V2DF_FTYPE_V2DI_V2DF_QI },
32579 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv4di3_mask, "__builtin_ia32_vpermt2varq256_mask", IX86_BUILTIN_VPERMT2VARQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32580 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv4di3_maskz, "__builtin_ia32_vpermt2varq256_maskz", IX86_BUILTIN_VPERMT2VARQ256_MASKZ, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32581 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv8si3_mask, "__builtin_ia32_vpermt2vard256_mask", IX86_BUILTIN_VPERMT2VARD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32582 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv8si3_maskz, "__builtin_ia32_vpermt2vard256_maskz", IX86_BUILTIN_VPERMT2VARD256_MASKZ, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32583 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermi2varv4di3_mask, "__builtin_ia32_vpermi2varq256_mask", IX86_BUILTIN_VPERMI2VARQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32584 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermi2varv8si3_mask, "__builtin_ia32_vpermi2vard256_mask", IX86_BUILTIN_VPERMI2VARD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32585 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv4df3_mask, "__builtin_ia32_vpermt2varpd256_mask", IX86_BUILTIN_VPERMT2VARPD256, UNKNOWN, (int) V4DF_FTYPE_V4DI_V4DF_V4DF_QI },
32586 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv4df3_maskz, "__builtin_ia32_vpermt2varpd256_maskz", IX86_BUILTIN_VPERMT2VARPD256_MASKZ, UNKNOWN, (int) V4DF_FTYPE_V4DI_V4DF_V4DF_QI },
32587 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv8sf3_mask, "__builtin_ia32_vpermt2varps256_mask", IX86_BUILTIN_VPERMT2VARPS256, UNKNOWN, (int) V8SF_FTYPE_V8SI_V8SF_V8SF_QI },
32588 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv8sf3_maskz, "__builtin_ia32_vpermt2varps256_maskz", IX86_BUILTIN_VPERMT2VARPS256_MASKZ, UNKNOWN, (int) V8SF_FTYPE_V8SI_V8SF_V8SF_QI },
32589 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermi2varv4df3_mask, "__builtin_ia32_vpermi2varpd256_mask", IX86_BUILTIN_VPERMI2VARPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DI_V4DF_QI },
32590 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermi2varv8sf3_mask, "__builtin_ia32_vpermi2varps256_mask", IX86_BUILTIN_VPERMI2VARPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SI_V8SF_QI },
32591 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv2di3_mask, "__builtin_ia32_vpermt2varq128_mask", IX86_BUILTIN_VPERMT2VARQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32592 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv2di3_maskz, "__builtin_ia32_vpermt2varq128_maskz", IX86_BUILTIN_VPERMT2VARQ128_MASKZ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32593 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv4si3_mask, "__builtin_ia32_vpermt2vard128_mask", IX86_BUILTIN_VPERMT2VARD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32594 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv4si3_maskz, "__builtin_ia32_vpermt2vard128_maskz", IX86_BUILTIN_VPERMT2VARD128_MASKZ, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32595 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermi2varv2di3_mask, "__builtin_ia32_vpermi2varq128_mask", IX86_BUILTIN_VPERMI2VARQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32596 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermi2varv4si3_mask, "__builtin_ia32_vpermi2vard128_mask", IX86_BUILTIN_VPERMI2VARD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32597 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv2df3_mask, "__builtin_ia32_vpermt2varpd128_mask", IX86_BUILTIN_VPERMT2VARPD128, UNKNOWN, (int) V2DF_FTYPE_V2DI_V2DF_V2DF_QI },
32598 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv2df3_maskz, "__builtin_ia32_vpermt2varpd128_maskz", IX86_BUILTIN_VPERMT2VARPD128_MASKZ, UNKNOWN, (int) V2DF_FTYPE_V2DI_V2DF_V2DF_QI },
32599 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv4sf3_mask, "__builtin_ia32_vpermt2varps128_mask", IX86_BUILTIN_VPERMT2VARPS128, UNKNOWN, (int) V4SF_FTYPE_V4SI_V4SF_V4SF_QI },
32600 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv4sf3_maskz, "__builtin_ia32_vpermt2varps128_maskz", IX86_BUILTIN_VPERMT2VARPS128_MASKZ, UNKNOWN, (int) V4SF_FTYPE_V4SI_V4SF_V4SF_QI },
32601 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermi2varv2df3_mask, "__builtin_ia32_vpermi2varpd128_mask", IX86_BUILTIN_VPERMI2VARPD128, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DI_V2DF_QI },
32602 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermi2varv4sf3_mask, "__builtin_ia32_vpermi2varps128_mask", IX86_BUILTIN_VPERMI2VARPS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SI_V4SF_QI },
32603 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_pshufbv32qi3_mask, "__builtin_ia32_pshufb256_mask", IX86_BUILTIN_PSHUFB256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
32604 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ssse3_pshufbv16qi3_mask, "__builtin_ia32_pshufb128_mask", IX86_BUILTIN_PSHUFB128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
32605 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_pshufhwv3_mask, "__builtin_ia32_pshufhw256_mask", IX86_BUILTIN_PSHUFHW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_INT_V16HI_HI },
32606 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_pshufhw_mask, "__builtin_ia32_pshufhw128_mask", IX86_BUILTIN_PSHUFHW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_INT_V8HI_QI },
32607 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_pshuflwv3_mask, "__builtin_ia32_pshuflw256_mask", IX86_BUILTIN_PSHUFLW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_INT_V16HI_HI },
32608 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_pshuflw_mask, "__builtin_ia32_pshuflw128_mask", IX86_BUILTIN_PSHUFLW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_INT_V8HI_QI },
32609 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_pshufdv3_mask, "__builtin_ia32_pshufd256_mask", IX86_BUILTIN_PSHUFD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_INT_V8SI_QI },
32610 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_pshufd_mask, "__builtin_ia32_pshufd128_mask", IX86_BUILTIN_PSHUFD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_INT_V4SI_QI },
32611 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_shufpd256_mask, "__builtin_ia32_shufpd256_mask", IX86_BUILTIN_SHUFPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT_V4DF_QI },
32612 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_shufpd_mask, "__builtin_ia32_shufpd128_mask", IX86_BUILTIN_SHUFPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT_V2DF_QI },
32613 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_shufps256_mask, "__builtin_ia32_shufps256_mask", IX86_BUILTIN_SHUFPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT_V8SF_QI },
32614 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse_shufps_mask, "__builtin_ia32_shufps128_mask", IX86_BUILTIN_SHUFPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT_V4SF_QI },
32615 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rolvv4di_mask, "__builtin_ia32_prolvq256_mask", IX86_BUILTIN_PROLVQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32616 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rolvv2di_mask, "__builtin_ia32_prolvq128_mask", IX86_BUILTIN_PROLVQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32617 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rolv4di_mask, "__builtin_ia32_prolq256_mask", IX86_BUILTIN_PROLQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_V4DI_QI },
32618 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rolv2di_mask, "__builtin_ia32_prolq128_mask", IX86_BUILTIN_PROLQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT_V2DI_QI },
32619 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rorvv4di_mask, "__builtin_ia32_prorvq256_mask", IX86_BUILTIN_PRORVQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32620 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rorvv2di_mask, "__builtin_ia32_prorvq128_mask", IX86_BUILTIN_PRORVQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32621 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rorv4di_mask, "__builtin_ia32_prorq256_mask", IX86_BUILTIN_PRORQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_V4DI_QI },
32622 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rorv2di_mask, "__builtin_ia32_prorq128_mask", IX86_BUILTIN_PRORQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT_V2DI_QI },
32623 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_ashrvv2di_mask, "__builtin_ia32_psravq128_mask", IX86_BUILTIN_PSRAVQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32624 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_ashrvv4di_mask, "__builtin_ia32_psravq256_mask", IX86_BUILTIN_PSRAVQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32625 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_ashlvv4di_mask, "__builtin_ia32_psllv4di_mask", IX86_BUILTIN_PSLLVV4DI_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32626 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_ashlvv2di_mask, "__builtin_ia32_psllv2di_mask", IX86_BUILTIN_PSLLVV2DI_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32627 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_ashlvv8si_mask, "__builtin_ia32_psllv8si_mask", IX86_BUILTIN_PSLLVV8SI_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32628 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_ashlvv4si_mask, "__builtin_ia32_psllv4si_mask", IX86_BUILTIN_PSLLVV4SI_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32629 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_ashrvv8si_mask, "__builtin_ia32_psrav8si_mask", IX86_BUILTIN_PSRAVV8SI_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32630 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_ashrvv4si_mask, "__builtin_ia32_psrav4si_mask", IX86_BUILTIN_PSRAVV4SI_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32631 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_lshrvv4di_mask, "__builtin_ia32_psrlv4di_mask", IX86_BUILTIN_PSRLVV4DI_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32632 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_lshrvv2di_mask, "__builtin_ia32_psrlv2di_mask", IX86_BUILTIN_PSRLVV2DI_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32633 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_lshrvv8si_mask, "__builtin_ia32_psrlv8si_mask", IX86_BUILTIN_PSRLVV8SI_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32634 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_lshrvv4si_mask, "__builtin_ia32_psrlv4si_mask", IX86_BUILTIN_PSRLVV4SI_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32635 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashrv16hi3_mask, "__builtin_ia32_psrawi256_mask", IX86_BUILTIN_PSRAWI256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_INT_V16HI_HI },
32636 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashrv16hi3_mask, "__builtin_ia32_psraw256_mask", IX86_BUILTIN_PSRAW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_V16HI_HI },
32637 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashrv8hi3_mask, "__builtin_ia32_psrawi128_mask", IX86_BUILTIN_PSRAWI128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_INT_V8HI_QI },
32638 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_ashrv8hi3_mask, "__builtin_ia32_psraw128_mask", IX86_BUILTIN_PSRAW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32639 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_lshrv16hi3_mask, "__builtin_ia32_psrlwi256_mask", IX86_BUILTIN_PSRLWI256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_INT_V16HI_HI },
32640 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_lshrv16hi3_mask, "__builtin_ia32_psrlw256_mask", IX86_BUILTIN_PSRLW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_V16HI_HI },
32641 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_lshrv8hi3_mask, "__builtin_ia32_psrlwi128_mask", IX86_BUILTIN_PSRLWI128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_INT_V8HI_QI },
32642 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_lshrv8hi3_mask, "__builtin_ia32_psrlw128_mask", IX86_BUILTIN_PSRLW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32643 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rorvv8si_mask, "__builtin_ia32_prorvd256_mask", IX86_BUILTIN_PRORVD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32644 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rolvv8si_mask, "__builtin_ia32_prolvd256_mask", IX86_BUILTIN_PROLVD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32645 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rorv8si_mask, "__builtin_ia32_prord256_mask", IX86_BUILTIN_PRORD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_INT_V8SI_QI },
32646 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rolv8si_mask, "__builtin_ia32_prold256_mask", IX86_BUILTIN_PROLD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_INT_V8SI_QI },
32647 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rorvv4si_mask, "__builtin_ia32_prorvd128_mask", IX86_BUILTIN_PRORVD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32648 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rolvv4si_mask, "__builtin_ia32_prolvd128_mask", IX86_BUILTIN_PROLVD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32649 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rorv4si_mask, "__builtin_ia32_prord128_mask", IX86_BUILTIN_PRORD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_INT_V4SI_QI },
32650 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_rolv4si_mask, "__builtin_ia32_prold128_mask", IX86_BUILTIN_PROLD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_INT_V4SI_QI },
32651 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_fpclassv4df_mask, "__builtin_ia32_fpclasspd256_mask", IX86_BUILTIN_FPCLASSPD256, UNKNOWN, (int) QI_FTYPE_V4DF_INT_QI },
32652 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_fpclassv2df_mask, "__builtin_ia32_fpclasspd128_mask", IX86_BUILTIN_FPCLASSPD128, UNKNOWN, (int) QI_FTYPE_V2DF_INT_QI },
32653 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_vmfpclassv2df, "__builtin_ia32_fpclasssd", IX86_BUILTIN_FPCLASSSD, UNKNOWN, (int) QI_FTYPE_V2DF_INT },
32654 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_fpclassv8sf_mask, "__builtin_ia32_fpclassps256_mask", IX86_BUILTIN_FPCLASSPS256, UNKNOWN, (int) QI_FTYPE_V8SF_INT_QI },
32655 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512dq_fpclassv4sf_mask, "__builtin_ia32_fpclassps128_mask", IX86_BUILTIN_FPCLASSPS128, UNKNOWN, (int) QI_FTYPE_V4SF_INT_QI },
32656 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_vmfpclassv4sf, "__builtin_ia32_fpclassss", IX86_BUILTIN_FPCLASSSS, UNKNOWN, (int) QI_FTYPE_V4SF_INT },
32657 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cvtb2maskv16qi, "__builtin_ia32_cvtb2mask128", IX86_BUILTIN_CVTB2MASK128, UNKNOWN, (int) HI_FTYPE_V16QI },
32658 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cvtb2maskv32qi, "__builtin_ia32_cvtb2mask256", IX86_BUILTIN_CVTB2MASK256, UNKNOWN, (int) SI_FTYPE_V32QI },
32659 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cvtw2maskv8hi, "__builtin_ia32_cvtw2mask128", IX86_BUILTIN_CVTW2MASK128, UNKNOWN, (int) QI_FTYPE_V8HI },
32660 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cvtw2maskv16hi, "__builtin_ia32_cvtw2mask256", IX86_BUILTIN_CVTW2MASK256, UNKNOWN, (int) HI_FTYPE_V16HI },
32661 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cvtd2maskv4si, "__builtin_ia32_cvtd2mask128", IX86_BUILTIN_CVTD2MASK128, UNKNOWN, (int) QI_FTYPE_V4SI },
32662 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cvtd2maskv8si, "__builtin_ia32_cvtd2mask256", IX86_BUILTIN_CVTD2MASK256, UNKNOWN, (int) QI_FTYPE_V8SI },
32663 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cvtq2maskv2di, "__builtin_ia32_cvtq2mask128", IX86_BUILTIN_CVTQ2MASK128, UNKNOWN, (int) QI_FTYPE_V2DI },
32664 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cvtq2maskv4di, "__builtin_ia32_cvtq2mask256", IX86_BUILTIN_CVTQ2MASK256, UNKNOWN, (int) QI_FTYPE_V4DI },
32665 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cvtmask2bv16qi, "__builtin_ia32_cvtmask2b128", IX86_BUILTIN_CVTMASK2B128, UNKNOWN, (int) V16QI_FTYPE_HI },
32666 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cvtmask2bv32qi, "__builtin_ia32_cvtmask2b256", IX86_BUILTIN_CVTMASK2B256, UNKNOWN, (int) V32QI_FTYPE_SI },
32667 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cvtmask2wv8hi, "__builtin_ia32_cvtmask2w128", IX86_BUILTIN_CVTMASK2W128, UNKNOWN, (int) V8HI_FTYPE_QI },
32668 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cvtmask2wv16hi, "__builtin_ia32_cvtmask2w256", IX86_BUILTIN_CVTMASK2W256, UNKNOWN, (int) V16HI_FTYPE_HI },
32669 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cvtmask2dv4si, "__builtin_ia32_cvtmask2d128", IX86_BUILTIN_CVTMASK2D128, UNKNOWN, (int) V4SI_FTYPE_QI },
32670 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cvtmask2dv8si, "__builtin_ia32_cvtmask2d256", IX86_BUILTIN_CVTMASK2D256, UNKNOWN, (int) V8SI_FTYPE_QI },
32671 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cvtmask2qv2di, "__builtin_ia32_cvtmask2q128", IX86_BUILTIN_CVTMASK2Q128, UNKNOWN, (int) V2DI_FTYPE_QI },
32672 { OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cvtmask2qv4di, "__builtin_ia32_cvtmask2q256", IX86_BUILTIN_CVTMASK2Q256, UNKNOWN, (int) V4DI_FTYPE_QI },
32673 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_eqv16qi3_mask, "__builtin_ia32_pcmpeqb128_mask", IX86_BUILTIN_PCMPEQB128_MASK, UNKNOWN, (int) HI_FTYPE_V16QI_V16QI_HI },
32674 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_eqv32qi3_mask, "__builtin_ia32_pcmpeqb256_mask", IX86_BUILTIN_PCMPEQB256_MASK, UNKNOWN, (int) SI_FTYPE_V32QI_V32QI_SI },
32675 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_eqv8hi3_mask, "__builtin_ia32_pcmpeqw128_mask", IX86_BUILTIN_PCMPEQW128_MASK, UNKNOWN, (int) QI_FTYPE_V8HI_V8HI_QI },
32676 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_eqv16hi3_mask, "__builtin_ia32_pcmpeqw256_mask", IX86_BUILTIN_PCMPEQW256_MASK, UNKNOWN, (int) HI_FTYPE_V16HI_V16HI_HI },
32677 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_eqv4si3_mask, "__builtin_ia32_pcmpeqd128_mask", IX86_BUILTIN_PCMPEQD128_MASK, UNKNOWN, (int) QI_FTYPE_V4SI_V4SI_QI },
32678 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_eqv8si3_mask, "__builtin_ia32_pcmpeqd256_mask", IX86_BUILTIN_PCMPEQD256_MASK, UNKNOWN, (int) QI_FTYPE_V8SI_V8SI_QI },
32679 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_eqv2di3_mask, "__builtin_ia32_pcmpeqq128_mask", IX86_BUILTIN_PCMPEQQ128_MASK, UNKNOWN, (int) QI_FTYPE_V2DI_V2DI_QI },
32680 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_eqv4di3_mask, "__builtin_ia32_pcmpeqq256_mask", IX86_BUILTIN_PCMPEQQ256_MASK, UNKNOWN, (int) QI_FTYPE_V4DI_V4DI_QI },
32681 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_gtv16qi3_mask, "__builtin_ia32_pcmpgtb128_mask", IX86_BUILTIN_PCMPGTB128_MASK, UNKNOWN, (int) HI_FTYPE_V16QI_V16QI_HI },
32682 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_gtv32qi3_mask, "__builtin_ia32_pcmpgtb256_mask", IX86_BUILTIN_PCMPGTB256_MASK, UNKNOWN, (int) SI_FTYPE_V32QI_V32QI_SI },
32683 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_gtv8hi3_mask, "__builtin_ia32_pcmpgtw128_mask", IX86_BUILTIN_PCMPGTW128_MASK, UNKNOWN, (int) QI_FTYPE_V8HI_V8HI_QI },
32684 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_gtv16hi3_mask, "__builtin_ia32_pcmpgtw256_mask", IX86_BUILTIN_PCMPGTW256_MASK, UNKNOWN, (int) HI_FTYPE_V16HI_V16HI_HI },
32685 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_gtv4si3_mask, "__builtin_ia32_pcmpgtd128_mask", IX86_BUILTIN_PCMPGTD128_MASK, UNKNOWN, (int) QI_FTYPE_V4SI_V4SI_QI },
32686 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_gtv8si3_mask, "__builtin_ia32_pcmpgtd256_mask", IX86_BUILTIN_PCMPGTD256_MASK, UNKNOWN, (int) QI_FTYPE_V8SI_V8SI_QI },
32687 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_gtv2di3_mask, "__builtin_ia32_pcmpgtq128_mask", IX86_BUILTIN_PCMPGTQ128_MASK, UNKNOWN, (int) QI_FTYPE_V2DI_V2DI_QI },
32688 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_gtv4di3_mask, "__builtin_ia32_pcmpgtq256_mask", IX86_BUILTIN_PCMPGTQ256_MASK, UNKNOWN, (int) QI_FTYPE_V4DI_V4DI_QI },
32689 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_testmv16qi3_mask, "__builtin_ia32_ptestmb128", IX86_BUILTIN_PTESTMB128, UNKNOWN, (int) HI_FTYPE_V16QI_V16QI_HI },
32690 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_testmv32qi3_mask, "__builtin_ia32_ptestmb256", IX86_BUILTIN_PTESTMB256, UNKNOWN, (int) SI_FTYPE_V32QI_V32QI_SI },
32691 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_testmv8hi3_mask, "__builtin_ia32_ptestmw128", IX86_BUILTIN_PTESTMW128, UNKNOWN, (int) QI_FTYPE_V8HI_V8HI_QI },
32692 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_testmv16hi3_mask, "__builtin_ia32_ptestmw256", IX86_BUILTIN_PTESTMW256, UNKNOWN, (int) HI_FTYPE_V16HI_V16HI_HI },
32693 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_testmv4si3_mask, "__builtin_ia32_ptestmd128", IX86_BUILTIN_PTESTMD128, UNKNOWN, (int) QI_FTYPE_V4SI_V4SI_QI },
32694 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_testmv8si3_mask, "__builtin_ia32_ptestmd256", IX86_BUILTIN_PTESTMD256, UNKNOWN, (int) QI_FTYPE_V8SI_V8SI_QI },
32695 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_testmv2di3_mask, "__builtin_ia32_ptestmq128", IX86_BUILTIN_PTESTMQ128, UNKNOWN, (int) QI_FTYPE_V2DI_V2DI_QI },
32696 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_testmv4di3_mask, "__builtin_ia32_ptestmq256", IX86_BUILTIN_PTESTMQ256, UNKNOWN, (int) QI_FTYPE_V4DI_V4DI_QI },
32697 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_testnmv16qi3_mask, "__builtin_ia32_ptestnmb128", IX86_BUILTIN_PTESTNMB128, UNKNOWN, (int) HI_FTYPE_V16QI_V16QI_HI },
32698 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_testnmv32qi3_mask, "__builtin_ia32_ptestnmb256", IX86_BUILTIN_PTESTNMB256, UNKNOWN, (int) SI_FTYPE_V32QI_V32QI_SI },
32699 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_testnmv8hi3_mask, "__builtin_ia32_ptestnmw128", IX86_BUILTIN_PTESTNMW128, UNKNOWN, (int) QI_FTYPE_V8HI_V8HI_QI },
32700 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_testnmv16hi3_mask, "__builtin_ia32_ptestnmw256", IX86_BUILTIN_PTESTNMW256, UNKNOWN, (int) HI_FTYPE_V16HI_V16HI_HI },
32701 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_testnmv4si3_mask, "__builtin_ia32_ptestnmd128", IX86_BUILTIN_PTESTNMD128, UNKNOWN, (int) QI_FTYPE_V4SI_V4SI_QI },
32702 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_testnmv8si3_mask, "__builtin_ia32_ptestnmd256", IX86_BUILTIN_PTESTNMD256, UNKNOWN, (int) QI_FTYPE_V8SI_V8SI_QI },
32703 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_testnmv2di3_mask, "__builtin_ia32_ptestnmq128", IX86_BUILTIN_PTESTNMQ128, UNKNOWN, (int) QI_FTYPE_V2DI_V2DI_QI },
32704 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_testnmv4di3_mask, "__builtin_ia32_ptestnmq256", IX86_BUILTIN_PTESTNMQ256, UNKNOWN, (int) QI_FTYPE_V4DI_V4DI_QI },
32705 { OPTION_MASK_ISA_AVX512VL | OPTION_MASK_ISA_AVX512CD, CODE_FOR_avx512cd_maskb_vec_dupv2di, "__builtin_ia32_broadcastmb128", IX86_BUILTIN_PBROADCASTMB128, UNKNOWN, (int) V2DI_FTYPE_QI },
32706 { OPTION_MASK_ISA_AVX512VL | OPTION_MASK_ISA_AVX512CD, CODE_FOR_avx512cd_maskb_vec_dupv4di, "__builtin_ia32_broadcastmb256", IX86_BUILTIN_PBROADCASTMB256, UNKNOWN, (int) V4DI_FTYPE_QI },
32707 { OPTION_MASK_ISA_AVX512VL | OPTION_MASK_ISA_AVX512CD, CODE_FOR_avx512cd_maskw_vec_dupv4si, "__builtin_ia32_broadcastmw128", IX86_BUILTIN_PBROADCASTMW128, UNKNOWN, (int) V4SI_FTYPE_HI },
32708 { OPTION_MASK_ISA_AVX512VL | OPTION_MASK_ISA_AVX512CD, CODE_FOR_avx512cd_maskw_vec_dupv8si, "__builtin_ia32_broadcastmw256", IX86_BUILTIN_PBROADCASTMW256, UNKNOWN, (int) V8SI_FTYPE_HI },
32709 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_compressv4df_mask, "__builtin_ia32_compressdf256_mask", IX86_BUILTIN_COMPRESSPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_QI },
32710 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_compressv2df_mask, "__builtin_ia32_compressdf128_mask", IX86_BUILTIN_COMPRESSPD128, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_QI },
32711 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_compressv8sf_mask, "__builtin_ia32_compresssf256_mask", IX86_BUILTIN_COMPRESSPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_QI },
32712 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_compressv4sf_mask, "__builtin_ia32_compresssf128_mask", IX86_BUILTIN_COMPRESSPS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_QI },
32713 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_compressv4di_mask, "__builtin_ia32_compressdi256_mask", IX86_BUILTIN_PCOMPRESSQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_QI },
32714 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_compressv2di_mask, "__builtin_ia32_compressdi128_mask", IX86_BUILTIN_PCOMPRESSQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_QI },
32715 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_compressv8si_mask, "__builtin_ia32_compresssi256_mask", IX86_BUILTIN_PCOMPRESSD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_QI },
32716 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_compressv4si_mask, "__builtin_ia32_compresssi128_mask", IX86_BUILTIN_PCOMPRESSD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_QI },
32717 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv4df_mask, "__builtin_ia32_expanddf256_mask", IX86_BUILTIN_EXPANDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_QI },
32718 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv2df_mask, "__builtin_ia32_expanddf128_mask", IX86_BUILTIN_EXPANDPD128, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_QI },
32719 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv8sf_mask, "__builtin_ia32_expandsf256_mask", IX86_BUILTIN_EXPANDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_QI },
32720 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv4sf_mask, "__builtin_ia32_expandsf128_mask", IX86_BUILTIN_EXPANDPS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_QI },
32721 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv4di_mask, "__builtin_ia32_expanddi256_mask", IX86_BUILTIN_PEXPANDQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_QI },
32722 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv2di_mask, "__builtin_ia32_expanddi128_mask", IX86_BUILTIN_PEXPANDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_QI },
32723 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv8si_mask, "__builtin_ia32_expandsi256_mask", IX86_BUILTIN_PEXPANDD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_QI },
32724 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv4si_mask, "__builtin_ia32_expandsi128_mask", IX86_BUILTIN_PEXPANDD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_QI },
32725 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv4df_maskz, "__builtin_ia32_expanddf256_maskz", IX86_BUILTIN_EXPANDPD256Z, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_QI },
32726 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv2df_maskz, "__builtin_ia32_expanddf128_maskz", IX86_BUILTIN_EXPANDPD128Z, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_QI },
32727 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv8sf_maskz, "__builtin_ia32_expandsf256_maskz", IX86_BUILTIN_EXPANDPS256Z, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_QI },
32728 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv4sf_maskz, "__builtin_ia32_expandsf128_maskz", IX86_BUILTIN_EXPANDPS128Z, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_QI },
32729 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv4di_maskz, "__builtin_ia32_expanddi256_maskz", IX86_BUILTIN_PEXPANDQ256Z, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_QI },
32730 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv2di_maskz, "__builtin_ia32_expanddi128_maskz", IX86_BUILTIN_PEXPANDQ128Z, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_QI },
32731 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv8si_maskz, "__builtin_ia32_expandsi256_maskz", IX86_BUILTIN_PEXPANDD256Z, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_QI },
32732 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_expandv4si_maskz, "__builtin_ia32_expandsi128_maskz", IX86_BUILTIN_PEXPANDD128Z, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_QI },
32733 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_smaxv8si3_mask, "__builtin_ia32_pmaxsd256_mask", IX86_BUILTIN_PMAXSD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32734 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sminv8si3_mask, "__builtin_ia32_pminsd256_mask", IX86_BUILTIN_PMINSD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32735 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_umaxv8si3_mask, "__builtin_ia32_pmaxud256_mask", IX86_BUILTIN_PMAXUD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32736 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_uminv8si3_mask, "__builtin_ia32_pminud256_mask", IX86_BUILTIN_PMINUD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32737 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_smaxv4si3_mask, "__builtin_ia32_pmaxsd128_mask", IX86_BUILTIN_PMAXSD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32738 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sminv4si3_mask, "__builtin_ia32_pminsd128_mask", IX86_BUILTIN_PMINSD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32739 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_umaxv4si3_mask, "__builtin_ia32_pmaxud128_mask", IX86_BUILTIN_PMAXUD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32740 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_uminv4si3_mask, "__builtin_ia32_pminud128_mask", IX86_BUILTIN_PMINUD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32741 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_smaxv4di3_mask, "__builtin_ia32_pmaxsq256_mask", IX86_BUILTIN_PMAXSQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32742 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sminv4di3_mask, "__builtin_ia32_pminsq256_mask", IX86_BUILTIN_PMINSQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32743 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_umaxv4di3_mask, "__builtin_ia32_pmaxuq256_mask", IX86_BUILTIN_PMAXUQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32744 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_uminv4di3_mask, "__builtin_ia32_pminuq256_mask", IX86_BUILTIN_PMINUQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32745 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_smaxv2di3_mask, "__builtin_ia32_pmaxsq128_mask", IX86_BUILTIN_PMAXSQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32746 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sminv2di3_mask, "__builtin_ia32_pminsq128_mask", IX86_BUILTIN_PMINSQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32747 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_umaxv2di3_mask, "__builtin_ia32_pmaxuq128_mask", IX86_BUILTIN_PMAXUQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32748 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_uminv2di3_mask, "__builtin_ia32_pminuq128_mask", IX86_BUILTIN_PMINUQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32749 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sminv32qi3_mask, "__builtin_ia32_pminsb256_mask", IX86_BUILTIN_PMINSB256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
32750 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_uminv32qi3_mask, "__builtin_ia32_pminub256_mask", IX86_BUILTIN_PMINUB256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
32751 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_smaxv32qi3_mask, "__builtin_ia32_pmaxsb256_mask", IX86_BUILTIN_PMAXSB256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
32752 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_umaxv32qi3_mask, "__builtin_ia32_pmaxub256_mask", IX86_BUILTIN_PMAXUB256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
32753 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sminv16qi3_mask, "__builtin_ia32_pminsb128_mask", IX86_BUILTIN_PMINSB128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
32754 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_uminv16qi3_mask, "__builtin_ia32_pminub128_mask", IX86_BUILTIN_PMINUB128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
32755 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_smaxv16qi3_mask, "__builtin_ia32_pmaxsb128_mask", IX86_BUILTIN_PMAXSB128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
32756 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_umaxv16qi3_mask, "__builtin_ia32_pmaxub128_mask", IX86_BUILTIN_PMAXUB128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
32757 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sminv16hi3_mask, "__builtin_ia32_pminsw256_mask", IX86_BUILTIN_PMINSW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32758 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_uminv16hi3_mask, "__builtin_ia32_pminuw256_mask", IX86_BUILTIN_PMINUW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32759 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_smaxv16hi3_mask, "__builtin_ia32_pmaxsw256_mask", IX86_BUILTIN_PMAXSW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32760 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_umaxv16hi3_mask, "__builtin_ia32_pmaxuw256_mask", IX86_BUILTIN_PMAXUW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32761 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sminv8hi3_mask, "__builtin_ia32_pminsw128_mask", IX86_BUILTIN_PMINSW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32762 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_uminv8hi3_mask, "__builtin_ia32_pminuw128_mask", IX86_BUILTIN_PMINUW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32763 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_smaxv8hi3_mask, "__builtin_ia32_pmaxsw128_mask", IX86_BUILTIN_PMAXSW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32764 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_umaxv8hi3_mask, "__builtin_ia32_pmaxuw128_mask", IX86_BUILTIN_PMAXUW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32765 { OPTION_MASK_ISA_AVX512CD | OPTION_MASK_ISA_AVX512VL, CODE_FOR_conflictv4di_mask, "__builtin_ia32_vpconflictdi_256_mask", IX86_BUILTIN_VPCONFLICTQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_QI },
32766 { OPTION_MASK_ISA_AVX512CD | OPTION_MASK_ISA_AVX512VL, CODE_FOR_conflictv8si_mask, "__builtin_ia32_vpconflictsi_256_mask", IX86_BUILTIN_VPCONFLICTD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_QI },
32767 { OPTION_MASK_ISA_AVX512CD | OPTION_MASK_ISA_AVX512VL, CODE_FOR_clzv4di2_mask, "__builtin_ia32_vplzcntq_256_mask", IX86_BUILTIN_VPCLZCNTQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_QI },
32768 { OPTION_MASK_ISA_AVX512CD | OPTION_MASK_ISA_AVX512VL, CODE_FOR_clzv8si2_mask, "__builtin_ia32_vplzcntd_256_mask", IX86_BUILTIN_VPCLZCNTD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_QI },
32769 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_unpckhpd256_mask, "__builtin_ia32_unpckhpd256_mask", IX86_BUILTIN_UNPCKHPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32770 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_unpckhpd128_mask, "__builtin_ia32_unpckhpd128_mask", IX86_BUILTIN_UNPCKHPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32771 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_unpckhps256_mask, "__builtin_ia32_unpckhps256_mask", IX86_BUILTIN_UNPCKHPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32772 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_vec_interleave_highv4sf_mask, "__builtin_ia32_unpckhps128_mask", IX86_BUILTIN_UNPCKHPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32773 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_unpcklpd256_mask, "__builtin_ia32_unpcklpd256_mask", IX86_BUILTIN_UNPCKLPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF_QI },
32774 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_unpcklpd128_mask, "__builtin_ia32_unpcklpd128_mask", IX86_BUILTIN_UNPCKLPD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_QI },
32775 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_unpcklps256_mask, "__builtin_ia32_unpcklps256_mask", IX86_BUILTIN_UNPCKLPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF_QI },
32776 { OPTION_MASK_ISA_AVX512CD | OPTION_MASK_ISA_AVX512VL, CODE_FOR_conflictv2di_mask, "__builtin_ia32_vpconflictdi_128_mask", IX86_BUILTIN_VPCONFLICTQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_QI },
32777 { OPTION_MASK_ISA_AVX512CD | OPTION_MASK_ISA_AVX512VL, CODE_FOR_conflictv4si_mask, "__builtin_ia32_vpconflictsi_128_mask", IX86_BUILTIN_VPCONFLICTD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_QI },
32778 { OPTION_MASK_ISA_AVX512CD | OPTION_MASK_ISA_AVX512VL, CODE_FOR_clzv2di2_mask, "__builtin_ia32_vplzcntq_128_mask", IX86_BUILTIN_VPCLZCNTQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_QI },
32779 { OPTION_MASK_ISA_AVX512CD | OPTION_MASK_ISA_AVX512VL, CODE_FOR_clzv4si2_mask, "__builtin_ia32_vplzcntd_128_mask", IX86_BUILTIN_VPCLZCNTD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_QI },
32780 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_unpcklps128_mask, "__builtin_ia32_unpcklps128_mask", IX86_BUILTIN_UNPCKLPS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_QI },
32781 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_alignv8si_mask, "__builtin_ia32_alignd256_mask", IX86_BUILTIN_ALIGND256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_INT_V8SI_QI },
32782 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_alignv4di_mask, "__builtin_ia32_alignq256_mask", IX86_BUILTIN_ALIGNQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_INT_V4DI_QI },
32783 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_alignv4si_mask, "__builtin_ia32_alignd128_mask", IX86_BUILTIN_ALIGND128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_INT_V4SI_QI },
32784 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_alignv2di_mask, "__builtin_ia32_alignq128_mask", IX86_BUILTIN_ALIGNQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_INT_V2DI_QI },
32785 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_vcvtps2ph256_mask, "__builtin_ia32_vcvtps2ph256_mask", IX86_BUILTIN_CVTPS2PH256_MASK, UNKNOWN, (int) V8HI_FTYPE_V8SF_INT_V8HI_QI },
32786 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_vcvtps2ph_mask, "__builtin_ia32_vcvtps2ph_mask", IX86_BUILTIN_CVTPS2PH_MASK, UNKNOWN, (int) V8HI_FTYPE_V4SF_INT_V8HI_QI },
32787 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_vcvtph2ps_mask, "__builtin_ia32_vcvtph2ps_mask", IX86_BUILTIN_CVTPH2PS_MASK, UNKNOWN, (int) V4SF_FTYPE_V8HI_V4SF_QI },
32788 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_vcvtph2ps256_mask, "__builtin_ia32_vcvtph2ps256_mask", IX86_BUILTIN_CVTPH2PS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8HI_V8SF_QI },
32789 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_vec_interleave_highv4si_mask, "__builtin_ia32_punpckhdq128_mask", IX86_BUILTIN_PUNPCKHDQ128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32790 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_interleave_highv8si_mask, "__builtin_ia32_punpckhdq256_mask", IX86_BUILTIN_PUNPCKHDQ256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32791 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_vec_interleave_highv2di_mask, "__builtin_ia32_punpckhqdq128_mask", IX86_BUILTIN_PUNPCKHQDQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32792 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_interleave_highv4di_mask, "__builtin_ia32_punpckhqdq256_mask", IX86_BUILTIN_PUNPCKHQDQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32793 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_vec_interleave_lowv4si_mask, "__builtin_ia32_punpckldq128_mask", IX86_BUILTIN_PUNPCKLDQ128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32794 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_interleave_lowv8si_mask, "__builtin_ia32_punpckldq256_mask", IX86_BUILTIN_PUNPCKLDQ256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32795 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_vec_interleave_lowv2di_mask, "__builtin_ia32_punpcklqdq128_mask", IX86_BUILTIN_PUNPCKLQDQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
32796 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_interleave_lowv4di_mask, "__builtin_ia32_punpcklqdq256_mask", IX86_BUILTIN_PUNPCKLQDQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32797 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_vec_interleave_highv16qi_mask, "__builtin_ia32_punpckhbw128_mask", IX86_BUILTIN_PUNPCKHBW128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
32798 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_interleave_highv32qi_mask, "__builtin_ia32_punpckhbw256_mask", IX86_BUILTIN_PUNPCKHBW256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
32799 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_vec_interleave_highv8hi_mask, "__builtin_ia32_punpckhwd128_mask", IX86_BUILTIN_PUNPCKHWD128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32800 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_interleave_highv16hi_mask, "__builtin_ia32_punpckhwd256_mask", IX86_BUILTIN_PUNPCKHWD256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32801 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_vec_interleave_lowv16qi_mask, "__builtin_ia32_punpcklbw128_mask", IX86_BUILTIN_PUNPCKLBW128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
32802 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_interleave_lowv32qi_mask, "__builtin_ia32_punpcklbw256_mask", IX86_BUILTIN_PUNPCKLBW256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
32803 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_vec_interleave_lowv8hi_mask, "__builtin_ia32_punpcklwd128_mask", IX86_BUILTIN_PUNPCKLWD128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32804 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_interleave_lowv16hi_mask, "__builtin_ia32_punpcklwd256_mask", IX86_BUILTIN_PUNPCKLWD256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32805 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ashlvv16hi_mask, "__builtin_ia32_psllv16hi_mask", IX86_BUILTIN_PSLLVV16HI, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32806 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ashlvv8hi_mask, "__builtin_ia32_psllv8hi_mask", IX86_BUILTIN_PSLLVV8HI, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32807 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_packssdw_mask, "__builtin_ia32_packssdw256_mask", IX86_BUILTIN_PACKSSDW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V8SI_V8SI_V16HI_HI },
32808 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_packssdw_mask, "__builtin_ia32_packssdw128_mask", IX86_BUILTIN_PACKSSDW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V4SI_V4SI_V8HI_QI },
32809 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_packusdw_mask, "__builtin_ia32_packusdw256_mask", IX86_BUILTIN_PACKUSDW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V8SI_V8SI_V16HI_HI },
32810 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse4_1_packusdw_mask, "__builtin_ia32_packusdw128_mask", IX86_BUILTIN_PACKUSDW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V4SI_V4SI_V8HI_QI },
32811 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_uavgv32qi3_mask, "__builtin_ia32_pavgb256_mask", IX86_BUILTIN_PAVGB256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
32812 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_uavgv16hi3_mask, "__builtin_ia32_pavgw256_mask", IX86_BUILTIN_PAVGW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_V16HI_HI },
32813 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_uavgv16qi3_mask, "__builtin_ia32_pavgb128_mask", IX86_BUILTIN_PAVGB128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
32814 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_uavgv8hi3_mask, "__builtin_ia32_pavgw128_mask", IX86_BUILTIN_PAVGW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI_QI },
32815 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_permvarv8sf_mask, "__builtin_ia32_permvarsf256_mask", IX86_BUILTIN_VPERMVARSF256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SI_V8SF_QI },
32816 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_permvarv4df_mask, "__builtin_ia32_permvardf256_mask", IX86_BUILTIN_VPERMVARDF256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DI_V4DF_QI },
32817 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_permv4df_mask, "__builtin_ia32_permdf256_mask", IX86_BUILTIN_VPERMDF256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT_V4DF_QI },
32818 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_absv32qi2_mask, "__builtin_ia32_pabsb256_mask", IX86_BUILTIN_PABSB256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_SI },
32819 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_absv16qi2_mask, "__builtin_ia32_pabsb128_mask", IX86_BUILTIN_PABSB128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_HI },
32820 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_absv16hi2_mask, "__builtin_ia32_pabsw256_mask", IX86_BUILTIN_PABSW256_MASK, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_HI },
32821 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_absv8hi2_mask, "__builtin_ia32_pabsw128_mask", IX86_BUILTIN_PABSW128_MASK, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_QI },
32822 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_vpermilvarv2df3_mask, "__builtin_ia32_vpermilvarpd_mask", IX86_BUILTIN_VPERMILVARPD_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DI_V2DF_QI },
32823 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_vpermilvarv4sf3_mask, "__builtin_ia32_vpermilvarps_mask", IX86_BUILTIN_VPERMILVARPS_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SI_V4SF_QI },
32824 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_vpermilvarv4df3_mask, "__builtin_ia32_vpermilvarpd256_mask", IX86_BUILTIN_VPERMILVARPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DI_V4DF_QI },
32825 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_vpermilvarv8sf3_mask, "__builtin_ia32_vpermilvarps256_mask", IX86_BUILTIN_VPERMILVARPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SI_V8SF_QI },
32826 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_vpermilv2df_mask, "__builtin_ia32_vpermilpd_mask", IX86_BUILTIN_VPERMILPD_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_INT_V2DF_QI },
32827 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_vpermilv4sf_mask, "__builtin_ia32_vpermilps_mask", IX86_BUILTIN_VPERMILPS_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_INT_V4SF_QI },
32828 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_vpermilv4df_mask, "__builtin_ia32_vpermilpd256_mask", IX86_BUILTIN_VPERMILPD256_MASK, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT_V4DF_QI },
32829 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_vpermilv8sf_mask, "__builtin_ia32_vpermilps256_mask", IX86_BUILTIN_VPERMILPS256_MASK, UNKNOWN, (int) V8SF_FTYPE_V8SF_INT_V8SF_QI },
32830 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_blendmv4di, "__builtin_ia32_blendmq_256_mask", IX86_BUILTIN_BLENDMQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_QI },
32831 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_blendmv8si, "__builtin_ia32_blendmd_256_mask", IX86_BUILTIN_BLENDMD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_QI },
32832 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_blendmv4df, "__builtin_ia32_blendmpd_256_mask", IX86_BUILTIN_BLENDMPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_QI },
32833 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_blendmv8sf, "__builtin_ia32_blendmps_256_mask", IX86_BUILTIN_BLENDMPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_QI },
32834 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_blendmv2di, "__builtin_ia32_blendmq_128_mask", IX86_BUILTIN_BLENDMQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_QI },
32835 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_blendmv4si, "__builtin_ia32_blendmd_128_mask", IX86_BUILTIN_BLENDMD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_QI },
32836 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_blendmv2df, "__builtin_ia32_blendmpd_128_mask", IX86_BUILTIN_BLENDMPD128, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_QI },
32837 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_blendmv4sf, "__builtin_ia32_blendmps_128_mask", IX86_BUILTIN_BLENDMPS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_QI },
32838 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_blendmv16hi, "__builtin_ia32_blendmw_256_mask", IX86_BUILTIN_BLENDMW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_HI },
32839 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_blendmv32qi, "__builtin_ia32_blendmb_256_mask", IX86_BUILTIN_BLENDMB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_SI },
32840 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_blendmv8hi, "__builtin_ia32_blendmw_128_mask", IX86_BUILTIN_BLENDMW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_QI },
32841 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_blendmv16qi, "__builtin_ia32_blendmb_128_mask", IX86_BUILTIN_BLENDMB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_HI },
32842 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_mulv8si3_mask, "__builtin_ia32_pmulld256_mask", IX86_BUILTIN_PMULLD256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32843 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_mulv4si3_mask, "__builtin_ia32_pmulld128_mask", IX86_BUILTIN_PMULLD128_MASK, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI_QI },
32844 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_vec_widen_umult_even_v8si_mask, "__builtin_ia32_pmuludq256_mask", IX86_BUILTIN_PMULUDQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V8SI_V8SI_V4DI_QI },
32845 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_vec_widen_smult_even_v8si_mask, "__builtin_ia32_pmuldq256_mask", IX86_BUILTIN_PMULDQ256_MASK, UNKNOWN, (int) V4DI_FTYPE_V8SI_V8SI_V4DI_QI },
32846 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse4_1_mulv2siv2di3_mask, "__builtin_ia32_pmuldq128_mask", IX86_BUILTIN_PMULDQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V4SI_V4SI_V2DI_QI },
32847 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_vec_widen_umult_even_v4si_mask, "__builtin_ia32_pmuludq128_mask", IX86_BUILTIN_PMULUDQ128_MASK, UNKNOWN, (int) V2DI_FTYPE_V4SI_V4SI_V2DI_QI },
32848 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx_cvtpd2ps256_mask, "__builtin_ia32_cvtpd2ps256_mask", IX86_BUILTIN_CVTPD2PS256_MASK, UNKNOWN, (int) V4SF_FTYPE_V4DF_V4SF_QI },
32849 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_sse2_cvtpd2ps_mask, "__builtin_ia32_cvtpd2ps_mask", IX86_BUILTIN_CVTPD2PS_MASK, UNKNOWN, (int) V4SF_FTYPE_V2DF_V4SF_QI },
32850 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_permvarv8si_mask, "__builtin_ia32_permvarsi256_mask", IX86_BUILTIN_VPERMVARSI256_MASK, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_V8SI_QI },
32851 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx2_permvarv4di_mask, "__builtin_ia32_permvardi256_mask", IX86_BUILTIN_VPERMVARDI256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
32852 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_permv4di_mask, "__builtin_ia32_permdi256_mask", IX86_BUILTIN_VPERMDI256_MASK, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_V4DI_QI },
32853 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cmpv4di3_mask, "__builtin_ia32_cmpq256_mask", IX86_BUILTIN_CMPQ256, UNKNOWN, (int) QI_FTYPE_V4DI_V4DI_INT_QI },
32854 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cmpv8si3_mask, "__builtin_ia32_cmpd256_mask", IX86_BUILTIN_CMPD256, UNKNOWN, (int) QI_FTYPE_V8SI_V8SI_INT_QI },
32855 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ucmpv4di3_mask, "__builtin_ia32_ucmpq256_mask", IX86_BUILTIN_UCMPQ256, UNKNOWN, (int) QI_FTYPE_V4DI_V4DI_INT_QI },
32856 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ucmpv8si3_mask, "__builtin_ia32_ucmpd256_mask", IX86_BUILTIN_UCMPD256, UNKNOWN, (int) QI_FTYPE_V8SI_V8SI_INT_QI },
32857 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cmpv32qi3_mask, "__builtin_ia32_cmpb256_mask", IX86_BUILTIN_CMPB256, UNKNOWN, (int) SI_FTYPE_V32QI_V32QI_INT_SI },
32858 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cmpv16hi3_mask, "__builtin_ia32_cmpw256_mask", IX86_BUILTIN_CMPW256, UNKNOWN, (int) HI_FTYPE_V16HI_V16HI_INT_HI },
32859 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ucmpv32qi3_mask, "__builtin_ia32_ucmpb256_mask", IX86_BUILTIN_UCMPB256, UNKNOWN, (int) SI_FTYPE_V32QI_V32QI_INT_SI },
32860 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ucmpv16hi3_mask, "__builtin_ia32_ucmpw256_mask", IX86_BUILTIN_UCMPW256, UNKNOWN, (int) HI_FTYPE_V16HI_V16HI_INT_HI },
32861 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cmpv4df3_mask, "__builtin_ia32_cmppd256_mask", IX86_BUILTIN_CMPPD256_MASK, UNKNOWN, (int) QI_FTYPE_V4DF_V4DF_INT_QI },
32862 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cmpv8sf3_mask, "__builtin_ia32_cmpps256_mask", IX86_BUILTIN_CMPPS256_MASK, UNKNOWN, (int) QI_FTYPE_V8SF_V8SF_INT_QI },
32863 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cmpv2di3_mask, "__builtin_ia32_cmpq128_mask", IX86_BUILTIN_CMPQ128, UNKNOWN, (int) QI_FTYPE_V2DI_V2DI_INT_QI },
32864 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cmpv4si3_mask, "__builtin_ia32_cmpd128_mask", IX86_BUILTIN_CMPD128, UNKNOWN, (int) QI_FTYPE_V4SI_V4SI_INT_QI },
32865 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ucmpv2di3_mask, "__builtin_ia32_ucmpq128_mask", IX86_BUILTIN_UCMPQ128, UNKNOWN, (int) QI_FTYPE_V2DI_V2DI_INT_QI },
32866 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ucmpv4si3_mask, "__builtin_ia32_ucmpd128_mask", IX86_BUILTIN_UCMPD128, UNKNOWN, (int) QI_FTYPE_V4SI_V4SI_INT_QI },
32867 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cmpv16qi3_mask, "__builtin_ia32_cmpb128_mask", IX86_BUILTIN_CMPB128, UNKNOWN, (int) HI_FTYPE_V16QI_V16QI_INT_HI },
32868 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cmpv8hi3_mask, "__builtin_ia32_cmpw128_mask", IX86_BUILTIN_CMPW128, UNKNOWN, (int) QI_FTYPE_V8HI_V8HI_INT_QI },
32869 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ucmpv16qi3_mask, "__builtin_ia32_ucmpb128_mask", IX86_BUILTIN_UCMPB128, UNKNOWN, (int) HI_FTYPE_V16QI_V16QI_INT_HI },
32870 { OPTION_MASK_ISA_AVX512BW | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_ucmpv8hi3_mask, "__builtin_ia32_ucmpw128_mask", IX86_BUILTIN_UCMPW128, UNKNOWN, (int) QI_FTYPE_V8HI_V8HI_INT_QI },
32871 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cmpv2df3_mask, "__builtin_ia32_cmppd128_mask", IX86_BUILTIN_CMPPD128_MASK, UNKNOWN, (int) QI_FTYPE_V2DF_V2DF_INT_QI },
32872 { OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_cmpv4sf3_mask, "__builtin_ia32_cmpps128_mask", IX86_BUILTIN_CMPPS128_MASK, UNKNOWN, (int) QI_FTYPE_V4SF_V4SF_INT_QI },
32875 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_broadcastv16sf_mask, "__builtin_ia32_broadcastf32x2_512_mask", IX86_BUILTIN_BROADCASTF32x2_512, UNKNOWN, (int) V16SF_FTYPE_V4SF_V16SF_HI },
32876 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_broadcastv16si_mask, "__builtin_ia32_broadcasti32x2_512_mask", IX86_BUILTIN_BROADCASTI32x2_512, UNKNOWN, (int) V16SI_FTYPE_V4SI_V16SI_HI },
32877 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_broadcastv8df_mask_1, "__builtin_ia32_broadcastf64x2_512_mask", IX86_BUILTIN_BROADCASTF64X2_512, UNKNOWN, (int) V8DF_FTYPE_V2DF_V8DF_QI },
32878 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_broadcastv8di_mask_1, "__builtin_ia32_broadcasti64x2_512_mask", IX86_BUILTIN_BROADCASTI64X2_512, UNKNOWN, (int) V8DI_FTYPE_V2DI_V8DI_QI },
32879 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_broadcastv16sf_mask_1, "__builtin_ia32_broadcastf32x8_512_mask", IX86_BUILTIN_BROADCASTF32X8_512, UNKNOWN, (int) V16SF_FTYPE_V8SF_V16SF_HI },
32880 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_broadcastv16si_mask_1, "__builtin_ia32_broadcasti32x8_512_mask", IX86_BUILTIN_BROADCASTI32X8_512, UNKNOWN, (int) V16SI_FTYPE_V8SI_V16SI_HI },
32881 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_vextractf64x2_mask, "__builtin_ia32_extractf64x2_512_mask", IX86_BUILTIN_EXTRACTF64X2_512, UNKNOWN, (int) V2DF_FTYPE_V8DF_INT_V2DF_QI },
32882 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_vextractf32x8_mask, "__builtin_ia32_extractf32x8_mask", IX86_BUILTIN_EXTRACTF32X8, UNKNOWN, (int) V8SF_FTYPE_V16SF_INT_V8SF_QI },
32883 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_vextracti64x2_mask, "__builtin_ia32_extracti64x2_512_mask", IX86_BUILTIN_EXTRACTI64X2_512, UNKNOWN, (int) V2DI_FTYPE_V8DI_INT_V2DI_QI },
32884 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_vextracti32x8_mask, "__builtin_ia32_extracti32x8_mask", IX86_BUILTIN_EXTRACTI32X8, UNKNOWN, (int) V8SI_FTYPE_V16SI_INT_V8SI_QI },
32885 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_reducepv8df_mask, "__builtin_ia32_reducepd512_mask", IX86_BUILTIN_REDUCEPD512_MASK, UNKNOWN, (int) V8DF_FTYPE_V8DF_INT_V8DF_QI },
32886 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_reducepv16sf_mask, "__builtin_ia32_reduceps512_mask", IX86_BUILTIN_REDUCEPS512_MASK, UNKNOWN, (int) V16SF_FTYPE_V16SF_INT_V16SF_HI },
32887 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_mulv8di3_mask, "__builtin_ia32_pmullq512_mask", IX86_BUILTIN_PMULLQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
32888 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_xorv8df3_mask, "__builtin_ia32_xorpd512_mask", IX86_BUILTIN_XORPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI },
32889 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_xorv16sf3_mask, "__builtin_ia32_xorps512_mask", IX86_BUILTIN_XORPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI },
32890 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_iorv8df3_mask, "__builtin_ia32_orpd512_mask", IX86_BUILTIN_ORPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI },
32891 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_iorv16sf3_mask, "__builtin_ia32_orps512_mask", IX86_BUILTIN_ORPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI },
32892 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_andv8df3_mask, "__builtin_ia32_andpd512_mask", IX86_BUILTIN_ANDPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI },
32893 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_andv16sf3_mask, "__builtin_ia32_andps512_mask", IX86_BUILTIN_ANDPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI },
32894 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512f_andnotv8df3_mask, "__builtin_ia32_andnpd512_mask", IX86_BUILTIN_ANDNPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI},
32895 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512f_andnotv16sf3_mask, "__builtin_ia32_andnps512_mask", IX86_BUILTIN_ANDNPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI },
32896 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_vinsertf32x8_mask, "__builtin_ia32_insertf32x8_mask", IX86_BUILTIN_INSERTF32X8, UNKNOWN, (int) V16SF_FTYPE_V16SF_V8SF_INT_V16SF_HI },
32897 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_vinserti32x8_mask, "__builtin_ia32_inserti32x8_mask", IX86_BUILTIN_INSERTI32X8, UNKNOWN, (int) V16SI_FTYPE_V16SI_V8SI_INT_V16SI_HI },
32898 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_vinsertf64x2_mask, "__builtin_ia32_insertf64x2_512_mask", IX86_BUILTIN_INSERTF64X2_512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V2DF_INT_V8DF_QI },
32899 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_vinserti64x2_mask, "__builtin_ia32_inserti64x2_512_mask", IX86_BUILTIN_INSERTI64X2_512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V2DI_INT_V8DI_QI },
32900 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_fpclassv8df_mask, "__builtin_ia32_fpclasspd512_mask", IX86_BUILTIN_FPCLASSPD512, UNKNOWN, (int) QI_FTYPE_V8DF_INT_QI },
32901 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_fpclassv16sf_mask, "__builtin_ia32_fpclassps512_mask", IX86_BUILTIN_FPCLASSPS512, UNKNOWN, (int) HI_FTYPE_V16SF_INT_HI },
32902 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512f_cvtd2maskv16si, "__builtin_ia32_cvtd2mask512", IX86_BUILTIN_CVTD2MASK512, UNKNOWN, (int) HI_FTYPE_V16SI },
32903 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512f_cvtq2maskv8di, "__builtin_ia32_cvtq2mask512", IX86_BUILTIN_CVTQ2MASK512, UNKNOWN, (int) QI_FTYPE_V8DI },
32904 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512f_cvtmask2dv16si, "__builtin_ia32_cvtmask2d512", IX86_BUILTIN_CVTMASK2D512, UNKNOWN, (int) V16SI_FTYPE_HI },
32905 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512f_cvtmask2qv8di, "__builtin_ia32_cvtmask2q512", IX86_BUILTIN_CVTMASK2Q512, UNKNOWN, (int) V8DI_FTYPE_QI },
32908 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_kunpcksi, "__builtin_ia32_kunpcksi", IX86_BUILTIN_KUNPCKWD, UNKNOWN, (int) SI_FTYPE_SI_SI },
32909 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_kunpckdi, "__builtin_ia32_kunpckdi", IX86_BUILTIN_KUNPCKDQ, UNKNOWN, (int) DI_FTYPE_DI_DI },
32910 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_packusdw_mask, "__builtin_ia32_packusdw512_mask", IX86_BUILTIN_PACKUSDW512, UNKNOWN, (int) V32HI_FTYPE_V16SI_V16SI_V32HI_SI },
32911 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_ashlv4ti3, "__builtin_ia32_pslldq512", IX86_BUILTIN_PSLLDQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_INT_CONVERT },
32912 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_lshrv4ti3, "__builtin_ia32_psrldq512", IX86_BUILTIN_PSRLDQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_INT_CONVERT },
32913 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_packssdw_mask, "__builtin_ia32_packssdw512_mask", IX86_BUILTIN_PACKSSDW512, UNKNOWN, (int) V32HI_FTYPE_V16SI_V16SI_V32HI_SI },
32914 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_palignrv4ti, "__builtin_ia32_palignr512", IX86_BUILTIN_PALIGNR512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_INT_CONVERT },
32915 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_palignrv64qi_mask, "__builtin_ia32_palignr512_mask", IX86_BUILTIN_PALIGNR512_MASK, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_INT_V8DI_DI_CONVERT },
32916 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_loaddquv32hi_mask, "__builtin_ia32_movdquhi512_mask", IX86_BUILTIN_MOVDQUHI512_MASK, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_SI },
32917 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512f_loaddquv64qi_mask, "__builtin_ia32_movdquqi512_mask", IX86_BUILTIN_MOVDQUQI512_MASK, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_DI },
32918 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512f_psadbw, "__builtin_ia32_psadbw512", IX86_BUILTIN_PSADBW512, UNKNOWN, (int) V8DI_FTYPE_V64QI_V64QI },
32919 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_dbpsadbwv32hi_mask, "__builtin_ia32_dbpsadbw512_mask", IX86_BUILTIN_DBPSADBW512, UNKNOWN, (int) V32HI_FTYPE_V64QI_V64QI_INT_V32HI_SI },
32920 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_vec_dupv64qi_mask, "__builtin_ia32_pbroadcastb512_mask", IX86_BUILTIN_PBROADCASTB512, UNKNOWN, (int) V64QI_FTYPE_V16QI_V64QI_DI },
32921 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_vec_dup_gprv64qi_mask, "__builtin_ia32_pbroadcastb512_gpr_mask", IX86_BUILTIN_PBROADCASTB512_GPR, UNKNOWN, (int) V64QI_FTYPE_QI_V64QI_DI },
32922 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_vec_dupv32hi_mask, "__builtin_ia32_pbroadcastw512_mask", IX86_BUILTIN_PBROADCASTW512, UNKNOWN, (int) V32HI_FTYPE_V8HI_V32HI_SI },
32923 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_vec_dup_gprv32hi_mask, "__builtin_ia32_pbroadcastw512_gpr_mask", IX86_BUILTIN_PBROADCASTW512_GPR, UNKNOWN, (int) V32HI_FTYPE_HI_V32HI_SI },
32924 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_sign_extendv32qiv32hi2_mask, "__builtin_ia32_pmovsxbw512_mask", IX86_BUILTIN_PMOVSXBW512_MASK, UNKNOWN, (int) V32HI_FTYPE_V32QI_V32HI_SI },
32925 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_zero_extendv32qiv32hi2_mask, "__builtin_ia32_pmovzxbw512_mask", IX86_BUILTIN_PMOVZXBW512_MASK, UNKNOWN, (int) V32HI_FTYPE_V32QI_V32HI_SI },
32926 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_permvarv32hi_mask, "__builtin_ia32_permvarhi512_mask", IX86_BUILTIN_VPERMVARHI512_MASK, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
32927 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_vpermt2varv32hi3_mask, "__builtin_ia32_vpermt2varhi512_mask", IX86_BUILTIN_VPERMT2VARHI512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
32928 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_vpermt2varv32hi3_maskz, "__builtin_ia32_vpermt2varhi512_maskz", IX86_BUILTIN_VPERMT2VARHI512_MASKZ, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
32929 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_vpermi2varv32hi3_mask, "__builtin_ia32_vpermi2varhi512_mask", IX86_BUILTIN_VPERMI2VARHI512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
32930 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_uavgv64qi3_mask, "__builtin_ia32_pavgb512_mask", IX86_BUILTIN_PAVGB512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
32931 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_uavgv32hi3_mask, "__builtin_ia32_pavgw512_mask", IX86_BUILTIN_PAVGW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
32932 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_addv64qi3_mask, "__builtin_ia32_paddb512_mask", IX86_BUILTIN_PADDB512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
32933 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_subv64qi3_mask, "__builtin_ia32_psubb512_mask", IX86_BUILTIN_PSUBB512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
32934 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_sssubv64qi3_mask, "__builtin_ia32_psubsb512_mask", IX86_BUILTIN_PSUBSB512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
32935 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_ssaddv64qi3_mask, "__builtin_ia32_paddsb512_mask", IX86_BUILTIN_PADDSB512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
32936 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_ussubv64qi3_mask, "__builtin_ia32_psubusb512_mask", IX86_BUILTIN_PSUBUSB512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
32937 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_usaddv64qi3_mask, "__builtin_ia32_paddusb512_mask", IX86_BUILTIN_PADDUSB512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
32938 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_subv32hi3_mask, "__builtin_ia32_psubw512_mask", IX86_BUILTIN_PSUBW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
32939 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_addv32hi3_mask, "__builtin_ia32_paddw512_mask", IX86_BUILTIN_PADDW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
32940 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_sssubv32hi3_mask, "__builtin_ia32_psubsw512_mask", IX86_BUILTIN_PSUBSW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
32941 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_ssaddv32hi3_mask, "__builtin_ia32_paddsw512_mask", IX86_BUILTIN_PADDSW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
32942 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_ussubv32hi3_mask, "__builtin_ia32_psubusw512_mask", IX86_BUILTIN_PSUBUSW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
32943 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_usaddv32hi3_mask, "__builtin_ia32_paddusw512_mask", IX86_BUILTIN_PADDUSW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
32944 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_umaxv32hi3_mask, "__builtin_ia32_pmaxuw512_mask", IX86_BUILTIN_PMAXUW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
32945 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_smaxv32hi3_mask, "__builtin_ia32_pmaxsw512_mask", IX86_BUILTIN_PMAXSW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
32946 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_uminv32hi3_mask, "__builtin_ia32_pminuw512_mask", IX86_BUILTIN_PMINUW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
32947 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_sminv32hi3_mask, "__builtin_ia32_pminsw512_mask", IX86_BUILTIN_PMINSW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
32948 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_umaxv64qi3_mask, "__builtin_ia32_pmaxub512_mask", IX86_BUILTIN_PMAXUB512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
32949 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_smaxv64qi3_mask, "__builtin_ia32_pmaxsb512_mask", IX86_BUILTIN_PMAXSB512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
32950 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_uminv64qi3_mask, "__builtin_ia32_pminub512_mask", IX86_BUILTIN_PMINUB512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
32951 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_sminv64qi3_mask, "__builtin_ia32_pminsb512_mask", IX86_BUILTIN_PMINSB512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
32952 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_truncatev32hiv32qi2_mask, "__builtin_ia32_pmovwb512_mask", IX86_BUILTIN_PMOVWB512, UNKNOWN, (int) V32QI_FTYPE_V32HI_V32QI_SI },
32953 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_ss_truncatev32hiv32qi2_mask, "__builtin_ia32_pmovswb512_mask", IX86_BUILTIN_PMOVSWB512, UNKNOWN, (int) V32QI_FTYPE_V32HI_V32QI_SI },
32954 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_us_truncatev32hiv32qi2_mask, "__builtin_ia32_pmovuswb512_mask", IX86_BUILTIN_PMOVUSWB512, UNKNOWN, (int) V32QI_FTYPE_V32HI_V32QI_SI },
32955 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_umulhrswv32hi3_mask, "__builtin_ia32_pmulhrsw512_mask", IX86_BUILTIN_PMULHRSW512_MASK, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
32956 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_umulv32hi3_highpart_mask, "__builtin_ia32_pmulhuw512_mask" , IX86_BUILTIN_PMULHUW512_MASK, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
32957 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_smulv32hi3_highpart_mask, "__builtin_ia32_pmulhw512_mask" , IX86_BUILTIN_PMULHW512_MASK, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
32958 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_mulv32hi3_mask, "__builtin_ia32_pmullw512_mask", IX86_BUILTIN_PMULLW512_MASK, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
32959 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_ashlv32hi3_mask, "__builtin_ia32_psllwi512_mask", IX86_BUILTIN_PSLLWI512_MASK, UNKNOWN, (int) V32HI_FTYPE_V32HI_INT_V32HI_SI },
32960 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_ashlv32hi3_mask, "__builtin_ia32_psllw512_mask", IX86_BUILTIN_PSLLW512_MASK, UNKNOWN, (int) V32HI_FTYPE_V32HI_V8HI_V32HI_SI },
32961 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_packsswb_mask, "__builtin_ia32_packsswb512_mask", IX86_BUILTIN_PACKSSWB512, UNKNOWN, (int) V64QI_FTYPE_V32HI_V32HI_V64QI_DI },
32962 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_packuswb_mask, "__builtin_ia32_packuswb512_mask", IX86_BUILTIN_PACKUSWB512, UNKNOWN, (int) V64QI_FTYPE_V32HI_V32HI_V64QI_DI },
32963 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_ashrvv32hi_mask, "__builtin_ia32_psrav32hi_mask", IX86_BUILTIN_PSRAVV32HI, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
32964 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_pmaddubsw512v32hi_mask, "__builtin_ia32_pmaddubsw512_mask", IX86_BUILTIN_PMADDUBSW512_MASK, UNKNOWN, (int) V32HI_FTYPE_V64QI_V64QI_V32HI_SI },
32965 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_pmaddwd512v32hi_mask, "__builtin_ia32_pmaddwd512_mask", IX86_BUILTIN_PMADDWD512_MASK, UNKNOWN, (int) V16SI_FTYPE_V32HI_V32HI_V16SI_HI },
32966 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_lshrvv32hi_mask, "__builtin_ia32_psrlv32hi_mask", IX86_BUILTIN_PSRLVV32HI, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
32967 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_interleave_highv64qi_mask, "__builtin_ia32_punpckhbw512_mask", IX86_BUILTIN_PUNPCKHBW512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
32968 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_interleave_highv32hi_mask, "__builtin_ia32_punpckhwd512_mask", IX86_BUILTIN_PUNPCKHWD512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
32969 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_interleave_lowv64qi_mask, "__builtin_ia32_punpcklbw512_mask", IX86_BUILTIN_PUNPCKLBW512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
32970 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_interleave_lowv32hi_mask, "__builtin_ia32_punpcklwd512_mask", IX86_BUILTIN_PUNPCKLWD512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
32971 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_pshufbv64qi3_mask, "__builtin_ia32_pshufb512_mask", IX86_BUILTIN_PSHUFB512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
32972 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_pshufhwv32hi_mask, "__builtin_ia32_pshufhw512_mask", IX86_BUILTIN_PSHUFHW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_INT_V32HI_SI },
32973 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_pshuflwv32hi_mask, "__builtin_ia32_pshuflw512_mask", IX86_BUILTIN_PSHUFLW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_INT_V32HI_SI },
32974 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_ashrv32hi3_mask, "__builtin_ia32_psrawi512_mask", IX86_BUILTIN_PSRAWI512, UNKNOWN, (int) V32HI_FTYPE_V32HI_INT_V32HI_SI },
32975 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_ashrv32hi3_mask, "__builtin_ia32_psraw512_mask", IX86_BUILTIN_PSRAW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V8HI_V32HI_SI },
32976 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_lshrv32hi3_mask, "__builtin_ia32_psrlwi512_mask", IX86_BUILTIN_PSRLWI512, UNKNOWN, (int) V32HI_FTYPE_V32HI_INT_V32HI_SI },
32977 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_lshrv32hi3_mask, "__builtin_ia32_psrlw512_mask", IX86_BUILTIN_PSRLW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V8HI_V32HI_SI },
32978 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_cvtb2maskv64qi, "__builtin_ia32_cvtb2mask512", IX86_BUILTIN_CVTB2MASK512, UNKNOWN, (int) DI_FTYPE_V64QI },
32979 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_cvtw2maskv32hi, "__builtin_ia32_cvtw2mask512", IX86_BUILTIN_CVTW2MASK512, UNKNOWN, (int) SI_FTYPE_V32HI },
32980 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_cvtmask2bv64qi, "__builtin_ia32_cvtmask2b512", IX86_BUILTIN_CVTMASK2B512, UNKNOWN, (int) V64QI_FTYPE_DI },
32981 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_cvtmask2wv32hi, "__builtin_ia32_cvtmask2w512", IX86_BUILTIN_CVTMASK2W512, UNKNOWN, (int) V32HI_FTYPE_SI },
32982 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_eqv64qi3_mask, "__builtin_ia32_pcmpeqb512_mask", IX86_BUILTIN_PCMPEQB512_MASK, UNKNOWN, (int) DI_FTYPE_V64QI_V64QI_DI },
32983 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_eqv32hi3_mask, "__builtin_ia32_pcmpeqw512_mask", IX86_BUILTIN_PCMPEQW512_MASK, UNKNOWN, (int) SI_FTYPE_V32HI_V32HI_SI },
32984 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_gtv64qi3_mask, "__builtin_ia32_pcmpgtb512_mask", IX86_BUILTIN_PCMPGTB512_MASK, UNKNOWN, (int) DI_FTYPE_V64QI_V64QI_DI },
32985 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_gtv32hi3_mask, "__builtin_ia32_pcmpgtw512_mask", IX86_BUILTIN_PCMPGTW512_MASK, UNKNOWN, (int) SI_FTYPE_V32HI_V32HI_SI },
32986 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_testmv64qi3_mask, "__builtin_ia32_ptestmb512", IX86_BUILTIN_PTESTMB512, UNKNOWN, (int) DI_FTYPE_V64QI_V64QI_DI },
32987 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_testmv32hi3_mask, "__builtin_ia32_ptestmw512", IX86_BUILTIN_PTESTMW512, UNKNOWN, (int) SI_FTYPE_V32HI_V32HI_SI },
32988 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_testnmv64qi3_mask, "__builtin_ia32_ptestnmb512", IX86_BUILTIN_PTESTNMB512, UNKNOWN, (int) DI_FTYPE_V64QI_V64QI_DI },
32989 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_testnmv32hi3_mask, "__builtin_ia32_ptestnmw512", IX86_BUILTIN_PTESTNMW512, UNKNOWN, (int) SI_FTYPE_V32HI_V32HI_SI },
32990 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_ashlvv32hi_mask, "__builtin_ia32_psllv32hi_mask", IX86_BUILTIN_PSLLVV32HI, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_V32HI_SI },
32991 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_absv64qi2_mask, "__builtin_ia32_pabsb512_mask", IX86_BUILTIN_PABSB512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_DI },
32992 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_absv32hi2_mask, "__builtin_ia32_pabsw512_mask", IX86_BUILTIN_PABSW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_SI },
32993 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_blendmv32hi, "__builtin_ia32_blendmw_512_mask", IX86_BUILTIN_BLENDMW512, UNKNOWN, (int) V32HI_FTYPE_V32HI_V32HI_SI },
32994 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_blendmv64qi, "__builtin_ia32_blendmb_512_mask", IX86_BUILTIN_BLENDMB512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_DI },
32995 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_cmpv64qi3_mask, "__builtin_ia32_cmpb512_mask", IX86_BUILTIN_CMPB512, UNKNOWN, (int) DI_FTYPE_V64QI_V64QI_INT_DI },
32996 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_cmpv32hi3_mask, "__builtin_ia32_cmpw512_mask", IX86_BUILTIN_CMPW512, UNKNOWN, (int) SI_FTYPE_V32HI_V32HI_INT_SI },
32997 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_ucmpv64qi3_mask, "__builtin_ia32_ucmpb512_mask", IX86_BUILTIN_UCMPB512, UNKNOWN, (int) DI_FTYPE_V64QI_V64QI_INT_DI },
32998 { OPTION_MASK_ISA_AVX512BW, CODE_FOR_avx512bw_ucmpv32hi3_mask, "__builtin_ia32_ucmpw512_mask", IX86_BUILTIN_UCMPW512, UNKNOWN, (int) SI_FTYPE_V32HI_V32HI_INT_SI },
33001 { OPTION_MASK_ISA_AVX512IFMA, CODE_FOR_vpamdd52luqv8di_mask, "__builtin_ia32_vpmadd52luq512_mask", IX86_BUILTIN_VPMADD52LUQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
33002 { OPTION_MASK_ISA_AVX512IFMA, CODE_FOR_vpamdd52luqv8di_maskz, "__builtin_ia32_vpmadd52luq512_maskz", IX86_BUILTIN_VPMADD52LUQ512_MASKZ, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
33003 { OPTION_MASK_ISA_AVX512IFMA, CODE_FOR_vpamdd52huqv8di_mask, "__builtin_ia32_vpmadd52huq512_mask", IX86_BUILTIN_VPMADD52HUQ512, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
33004 { OPTION_MASK_ISA_AVX512IFMA, CODE_FOR_vpamdd52huqv8di_maskz, "__builtin_ia32_vpmadd52huq512_maskz", IX86_BUILTIN_VPMADD52HUQ512_MASKZ, UNKNOWN, (int) V8DI_FTYPE_V8DI_V8DI_V8DI_QI },
33005 { OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL, CODE_FOR_vpamdd52luqv4di_mask, "__builtin_ia32_vpmadd52luq256_mask", IX86_BUILTIN_VPMADD52LUQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
33006 { OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL, CODE_FOR_vpamdd52luqv4di_maskz, "__builtin_ia32_vpmadd52luq256_maskz", IX86_BUILTIN_VPMADD52LUQ256_MASKZ, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
33007 { OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL, CODE_FOR_vpamdd52huqv4di_mask, "__builtin_ia32_vpmadd52huq256_mask", IX86_BUILTIN_VPMADD52HUQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
33008 { OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL, CODE_FOR_vpamdd52huqv4di_maskz, "__builtin_ia32_vpmadd52huq256_maskz", IX86_BUILTIN_VPMADD52HUQ256_MASKZ, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_V4DI_QI },
33009 { OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL, CODE_FOR_vpamdd52luqv2di_mask, "__builtin_ia32_vpmadd52luq128_mask", IX86_BUILTIN_VPMADD52LUQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
33010 { OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL, CODE_FOR_vpamdd52luqv2di_maskz, "__builtin_ia32_vpmadd52luq128_maskz", IX86_BUILTIN_VPMADD52LUQ128_MASKZ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
33011 { OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL, CODE_FOR_vpamdd52huqv2di_mask, "__builtin_ia32_vpmadd52huq128_mask", IX86_BUILTIN_VPMADD52HUQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
33012 { OPTION_MASK_ISA_AVX512IFMA | OPTION_MASK_ISA_AVX512VL, CODE_FOR_vpamdd52huqv2di_maskz, "__builtin_ia32_vpmadd52huq128_maskz", IX86_BUILTIN_VPMADD52HUQ128_MASKZ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI_QI },
33015 { OPTION_MASK_ISA_AVX512VBMI, CODE_FOR_vpmultishiftqbv64qi_mask, "__builtin_ia32_vpmultishiftqb512_mask", IX86_BUILTIN_VPMULTISHIFTQB512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
33016 { OPTION_MASK_ISA_AVX512VBMI | OPTION_MASK_ISA_AVX512VL, CODE_FOR_vpmultishiftqbv32qi_mask, "__builtin_ia32_vpmultishiftqb256_mask", IX86_BUILTIN_VPMULTISHIFTQB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
33017 { OPTION_MASK_ISA_AVX512VBMI | OPTION_MASK_ISA_AVX512VL, CODE_FOR_vpmultishiftqbv16qi_mask, "__builtin_ia32_vpmultishiftqb128_mask", IX86_BUILTIN_VPMULTISHIFTQB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
33018 { OPTION_MASK_ISA_AVX512VBMI, CODE_FOR_avx512bw_permvarv64qi_mask, "__builtin_ia32_permvarqi512_mask", IX86_BUILTIN_VPERMVARQI512_MASK, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
33019 { OPTION_MASK_ISA_AVX512VBMI, CODE_FOR_avx512bw_vpermt2varv64qi3_mask, "__builtin_ia32_vpermt2varqi512_mask", IX86_BUILTIN_VPERMT2VARQI512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
33020 { OPTION_MASK_ISA_AVX512VBMI, CODE_FOR_avx512bw_vpermt2varv64qi3_maskz, "__builtin_ia32_vpermt2varqi512_maskz", IX86_BUILTIN_VPERMT2VARQI512_MASKZ, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
33021 { OPTION_MASK_ISA_AVX512VBMI, CODE_FOR_avx512bw_vpermi2varv64qi3_mask, "__builtin_ia32_vpermi2varqi512_mask", IX86_BUILTIN_VPERMI2VARQI512, UNKNOWN, (int) V64QI_FTYPE_V64QI_V64QI_V64QI_DI },
33022 { OPTION_MASK_ISA_AVX512VBMI | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_permvarv32qi_mask, "__builtin_ia32_permvarqi256_mask", IX86_BUILTIN_VPERMVARQI256_MASK, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
33023 { OPTION_MASK_ISA_AVX512VBMI | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_permvarv16qi_mask, "__builtin_ia32_permvarqi128_mask", IX86_BUILTIN_VPERMVARQI128_MASK, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
33024 { OPTION_MASK_ISA_AVX512VBMI | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv32qi3_mask, "__builtin_ia32_vpermt2varqi256_mask", IX86_BUILTIN_VPERMT2VARQI256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
33025 { OPTION_MASK_ISA_AVX512VBMI | OPTION_MASK_ISA_AVX512VBMI | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv32qi3_maskz, "__builtin_ia32_vpermt2varqi256_maskz", IX86_BUILTIN_VPERMT2VARQI256_MASKZ, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
33026 { OPTION_MASK_ISA_AVX512VBMI | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv16qi3_mask, "__builtin_ia32_vpermt2varqi128_mask", IX86_BUILTIN_VPERMT2VARQI128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
33027 { OPTION_MASK_ISA_AVX512VBMI | OPTION_MASK_ISA_AVX512VBMI | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermt2varv16qi3_maskz, "__builtin_ia32_vpermt2varqi128_maskz", IX86_BUILTIN_VPERMT2VARQI128_MASKZ, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
33028 { OPTION_MASK_ISA_AVX512VBMI | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermi2varv32qi3_mask, "__builtin_ia32_vpermi2varqi256_mask", IX86_BUILTIN_VPERMI2VARQI256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI_SI },
33029 { OPTION_MASK_ISA_AVX512VBMI | OPTION_MASK_ISA_AVX512VL, CODE_FOR_avx512vl_vpermi2varv16qi3_mask, "__builtin_ia32_vpermi2varqi128_mask", IX86_BUILTIN_VPERMI2VARQI128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI_HI },
33032 /* Builtins with rounding support. */
33033 static const struct builtin_description bdesc_round_args[] =
33036 { OPTION_MASK_ISA_AVX512F, CODE_FOR_addv8df3_mask_round, "__builtin_ia32_addpd512_mask", IX86_BUILTIN_ADDPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33037 { OPTION_MASK_ISA_AVX512F, CODE_FOR_addv16sf3_mask_round, "__builtin_ia32_addps512_mask", IX86_BUILTIN_ADDPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33038 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse2_vmaddv2df3_round, "__builtin_ia32_addsd_round", IX86_BUILTIN_ADDSD_ROUND, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
33039 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse_vmaddv4sf3_round, "__builtin_ia32_addss_round", IX86_BUILTIN_ADDSS_ROUND, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
33040 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_cmpv8df3_mask_round, "__builtin_ia32_cmppd512_mask", IX86_BUILTIN_CMPPD512, UNKNOWN, (int) QI_FTYPE_V8DF_V8DF_INT_QI_INT },
33041 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_cmpv16sf3_mask_round, "__builtin_ia32_cmpps512_mask", IX86_BUILTIN_CMPPS512, UNKNOWN, (int) HI_FTYPE_V16SF_V16SF_INT_HI_INT },
33042 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vmcmpv2df3_mask_round, "__builtin_ia32_cmpsd_mask", IX86_BUILTIN_CMPSD_MASK, UNKNOWN, (int) QI_FTYPE_V2DF_V2DF_INT_QI_INT },
33043 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vmcmpv4sf3_mask_round, "__builtin_ia32_cmpss_mask", IX86_BUILTIN_CMPSS_MASK, UNKNOWN, (int) QI_FTYPE_V4SF_V4SF_INT_QI_INT },
33044 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse2_comi_round, "__builtin_ia32_vcomisd", IX86_BUILTIN_COMIDF, UNKNOWN, (int) INT_FTYPE_V2DF_V2DF_INT_INT },
33045 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse_comi_round, "__builtin_ia32_vcomiss", IX86_BUILTIN_COMISF, UNKNOWN, (int) INT_FTYPE_V4SF_V4SF_INT_INT },
33046 { OPTION_MASK_ISA_AVX512F, CODE_FOR_floatv16siv16sf2_mask_round, "__builtin_ia32_cvtdq2ps512_mask", IX86_BUILTIN_CVTDQ2PS512, UNKNOWN, (int) V16SF_FTYPE_V16SI_V16SF_HI_INT },
33047 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_cvtpd2dq512_mask_round, "__builtin_ia32_cvtpd2dq512_mask", IX86_BUILTIN_CVTPD2DQ512, UNKNOWN, (int) V8SI_FTYPE_V8DF_V8SI_QI_INT },
33048 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_cvtpd2ps512_mask_round, "__builtin_ia32_cvtpd2ps512_mask", IX86_BUILTIN_CVTPD2PS512, UNKNOWN, (int) V8SF_FTYPE_V8DF_V8SF_QI_INT },
33049 { OPTION_MASK_ISA_AVX512F, CODE_FOR_ufix_notruncv8dfv8si2_mask_round, "__builtin_ia32_cvtpd2udq512_mask", IX86_BUILTIN_CVTPD2UDQ512, UNKNOWN, (int) V8SI_FTYPE_V8DF_V8SI_QI_INT },
33050 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vcvtph2ps512_mask_round, "__builtin_ia32_vcvtph2ps512_mask", IX86_BUILTIN_CVTPH2PS512, UNKNOWN, (int) V16SF_FTYPE_V16HI_V16SF_HI_INT },
33051 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fix_notruncv16sfv16si_mask_round, "__builtin_ia32_cvtps2dq512_mask", IX86_BUILTIN_CVTPS2DQ512, UNKNOWN, (int) V16SI_FTYPE_V16SF_V16SI_HI_INT },
33052 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_cvtps2pd512_mask_round, "__builtin_ia32_cvtps2pd512_mask", IX86_BUILTIN_CVTPS2PD512, UNKNOWN, (int) V8DF_FTYPE_V8SF_V8DF_QI_INT },
33053 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_ufix_notruncv16sfv16si_mask_round, "__builtin_ia32_cvtps2udq512_mask", IX86_BUILTIN_CVTPS2UDQ512, UNKNOWN, (int) V16SI_FTYPE_V16SF_V16SI_HI_INT },
33054 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse2_cvtsd2ss_round, "__builtin_ia32_cvtsd2ss_round", IX86_BUILTIN_CVTSD2SS_ROUND, UNKNOWN, (int) V4SF_FTYPE_V4SF_V2DF_INT },
33055 { OPTION_MASK_ISA_AVX512F | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvtsi2sdq_round, "__builtin_ia32_cvtsi2sd64", IX86_BUILTIN_CVTSI2SD64, UNKNOWN, (int) V2DF_FTYPE_V2DF_INT64_INT },
33056 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse_cvtsi2ss_round, "__builtin_ia32_cvtsi2ss32", IX86_BUILTIN_CVTSI2SS32, UNKNOWN, (int) V4SF_FTYPE_V4SF_INT_INT },
33057 { OPTION_MASK_ISA_AVX512F | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvtsi2ssq_round, "__builtin_ia32_cvtsi2ss64", IX86_BUILTIN_CVTSI2SS64, UNKNOWN, (int) V4SF_FTYPE_V4SF_INT64_INT },
33058 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse2_cvtss2sd_round, "__builtin_ia32_cvtss2sd_round", IX86_BUILTIN_CVTSS2SD_ROUND, UNKNOWN, (int) V2DF_FTYPE_V2DF_V4SF_INT },
33059 { OPTION_MASK_ISA_AVX512F, CODE_FOR_fix_truncv8dfv8si2_mask_round, "__builtin_ia32_cvttpd2dq512_mask", IX86_BUILTIN_CVTTPD2DQ512, UNKNOWN, (int) V8SI_FTYPE_V8DF_V8SI_QI_INT },
33060 { OPTION_MASK_ISA_AVX512F, CODE_FOR_ufix_truncv8dfv8si2_mask_round, "__builtin_ia32_cvttpd2udq512_mask", IX86_BUILTIN_CVTTPD2UDQ512, UNKNOWN, (int) V8SI_FTYPE_V8DF_V8SI_QI_INT },
33061 { OPTION_MASK_ISA_AVX512F, CODE_FOR_fix_truncv16sfv16si2_mask_round, "__builtin_ia32_cvttps2dq512_mask", IX86_BUILTIN_CVTTPS2DQ512, UNKNOWN, (int) V16SI_FTYPE_V16SF_V16SI_HI_INT },
33062 { OPTION_MASK_ISA_AVX512F, CODE_FOR_ufix_truncv16sfv16si2_mask_round, "__builtin_ia32_cvttps2udq512_mask", IX86_BUILTIN_CVTTPS2UDQ512, UNKNOWN, (int) V16SI_FTYPE_V16SF_V16SI_HI_INT },
33063 { OPTION_MASK_ISA_AVX512F, CODE_FOR_ufloatv16siv16sf2_mask_round, "__builtin_ia32_cvtudq2ps512_mask", IX86_BUILTIN_CVTUDQ2PS512, UNKNOWN, (int) V16SF_FTYPE_V16SI_V16SF_HI_INT },
33064 { OPTION_MASK_ISA_AVX512F | OPTION_MASK_ISA_64BIT, CODE_FOR_cvtusi2sd64_round, "__builtin_ia32_cvtusi2sd64", IX86_BUILTIN_CVTUSI2SD64, UNKNOWN, (int) V2DF_FTYPE_V2DF_UINT64_INT },
33065 { OPTION_MASK_ISA_AVX512F, CODE_FOR_cvtusi2ss32_round, "__builtin_ia32_cvtusi2ss32", IX86_BUILTIN_CVTUSI2SS32, UNKNOWN, (int) V4SF_FTYPE_V4SF_UINT_INT },
33066 { OPTION_MASK_ISA_AVX512F | OPTION_MASK_ISA_64BIT, CODE_FOR_cvtusi2ss64_round, "__builtin_ia32_cvtusi2ss64", IX86_BUILTIN_CVTUSI2SS64, UNKNOWN, (int) V4SF_FTYPE_V4SF_UINT64_INT },
33067 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_divv8df3_mask_round, "__builtin_ia32_divpd512_mask", IX86_BUILTIN_DIVPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33068 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_divv16sf3_mask_round, "__builtin_ia32_divps512_mask", IX86_BUILTIN_DIVPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33069 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse2_vmdivv2df3_round, "__builtin_ia32_divsd_round", IX86_BUILTIN_DIVSD_ROUND, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
33070 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse_vmdivv4sf3_round, "__builtin_ia32_divss_round", IX86_BUILTIN_DIVSS_ROUND, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
33071 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fixupimmv8df_mask_round, "__builtin_ia32_fixupimmpd512_mask", IX86_BUILTIN_FIXUPIMMPD512_MASK, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DI_INT_QI_INT },
33072 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fixupimmv8df_maskz_round, "__builtin_ia32_fixupimmpd512_maskz", IX86_BUILTIN_FIXUPIMMPD512_MASKZ, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DI_INT_QI_INT },
33073 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fixupimmv16sf_mask_round, "__builtin_ia32_fixupimmps512_mask", IX86_BUILTIN_FIXUPIMMPS512_MASK, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SI_INT_HI_INT },
33074 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fixupimmv16sf_maskz_round, "__builtin_ia32_fixupimmps512_maskz", IX86_BUILTIN_FIXUPIMMPS512_MASKZ, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SI_INT_HI_INT },
33075 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_sfixupimmv2df_mask_round, "__builtin_ia32_fixupimmsd_mask", IX86_BUILTIN_FIXUPIMMSD128_MASK, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DI_INT_QI_INT },
33076 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_sfixupimmv2df_maskz_round, "__builtin_ia32_fixupimmsd_maskz", IX86_BUILTIN_FIXUPIMMSD128_MASKZ, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DI_INT_QI_INT },
33077 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_sfixupimmv4sf_mask_round, "__builtin_ia32_fixupimmss_mask", IX86_BUILTIN_FIXUPIMMSS128_MASK, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SI_INT_QI_INT },
33078 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_sfixupimmv4sf_maskz_round, "__builtin_ia32_fixupimmss_maskz", IX86_BUILTIN_FIXUPIMMSS128_MASKZ, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SI_INT_QI_INT },
33079 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_getexpv8df_mask_round, "__builtin_ia32_getexppd512_mask", IX86_BUILTIN_GETEXPPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_QI_INT },
33080 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_getexpv16sf_mask_round, "__builtin_ia32_getexpps512_mask", IX86_BUILTIN_GETEXPPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_HI_INT },
33081 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_sgetexpv2df_round, "__builtin_ia32_getexpsd128_round", IX86_BUILTIN_GETEXPSD128, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
33082 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_sgetexpv4sf_round, "__builtin_ia32_getexpss128_round", IX86_BUILTIN_GETEXPSS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
33083 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_getmantv8df_mask_round, "__builtin_ia32_getmantpd512_mask", IX86_BUILTIN_GETMANTPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_INT_V8DF_QI_INT },
33084 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_getmantv16sf_mask_round, "__builtin_ia32_getmantps512_mask", IX86_BUILTIN_GETMANTPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_INT_V16SF_HI_INT },
33085 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vgetmantv2df_round, "__builtin_ia32_getmantsd_round", IX86_BUILTIN_GETMANTSD128, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT_INT },
33086 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vgetmantv4sf_round, "__builtin_ia32_getmantss_round", IX86_BUILTIN_GETMANTSS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT_INT },
33087 { OPTION_MASK_ISA_AVX512F, CODE_FOR_smaxv8df3_mask_round, "__builtin_ia32_maxpd512_mask", IX86_BUILTIN_MAXPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33088 { OPTION_MASK_ISA_AVX512F, CODE_FOR_smaxv16sf3_mask_round, "__builtin_ia32_maxps512_mask", IX86_BUILTIN_MAXPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33089 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse2_vmsmaxv2df3_round, "__builtin_ia32_maxsd_round", IX86_BUILTIN_MAXSD_ROUND, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
33090 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse_vmsmaxv4sf3_round, "__builtin_ia32_maxss_round", IX86_BUILTIN_MAXSS_ROUND, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
33091 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sminv8df3_mask_round, "__builtin_ia32_minpd512_mask", IX86_BUILTIN_MINPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33092 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sminv16sf3_mask_round, "__builtin_ia32_minps512_mask", IX86_BUILTIN_MINPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33093 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse2_vmsminv2df3_round, "__builtin_ia32_minsd_round", IX86_BUILTIN_MINSD_ROUND, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
33094 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse_vmsminv4sf3_round, "__builtin_ia32_minss_round", IX86_BUILTIN_MINSS_ROUND, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
33095 { OPTION_MASK_ISA_AVX512F, CODE_FOR_mulv8df3_mask_round, "__builtin_ia32_mulpd512_mask", IX86_BUILTIN_MULPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33096 { OPTION_MASK_ISA_AVX512F, CODE_FOR_mulv16sf3_mask_round, "__builtin_ia32_mulps512_mask", IX86_BUILTIN_MULPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33097 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse2_vmmulv2df3_round, "__builtin_ia32_mulsd_round", IX86_BUILTIN_MULSD_ROUND, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
33098 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse_vmmulv4sf3_round, "__builtin_ia32_mulss_round", IX86_BUILTIN_MULSS_ROUND, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
33099 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_rndscalev8df_mask_round, "__builtin_ia32_rndscalepd_mask", IX86_BUILTIN_RNDSCALEPD, UNKNOWN, (int) V8DF_FTYPE_V8DF_INT_V8DF_QI_INT },
33100 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_rndscalev16sf_mask_round, "__builtin_ia32_rndscaleps_mask", IX86_BUILTIN_RNDSCALEPS, UNKNOWN, (int) V16SF_FTYPE_V16SF_INT_V16SF_HI_INT },
33101 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_rndscalev2df_round, "__builtin_ia32_rndscalesd_round", IX86_BUILTIN_RNDSCALESD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT_INT },
33102 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_rndscalev4sf_round, "__builtin_ia32_rndscaless_round", IX86_BUILTIN_RNDSCALESS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT_INT },
33103 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_scalefv8df_mask_round, "__builtin_ia32_scalefpd512_mask", IX86_BUILTIN_SCALEFPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33104 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_scalefv16sf_mask_round, "__builtin_ia32_scalefps512_mask", IX86_BUILTIN_SCALEFPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33105 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vmscalefv2df_round, "__builtin_ia32_scalefsd_round", IX86_BUILTIN_SCALEFSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
33106 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vmscalefv4sf_round, "__builtin_ia32_scalefss_round", IX86_BUILTIN_SCALEFSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
33107 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_sqrtv8df2_mask_round, "__builtin_ia32_sqrtpd512_mask", IX86_BUILTIN_SQRTPD512_MASK, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_QI_INT },
33108 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_sqrtv16sf2_mask_round, "__builtin_ia32_sqrtps512_mask", IX86_BUILTIN_SQRTPS512_MASK, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_HI_INT },
33109 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse2_vmsqrtv2df2_round, "__builtin_ia32_sqrtsd_round", IX86_BUILTIN_SQRTSD_ROUND, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
33110 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse_vmsqrtv4sf2_round, "__builtin_ia32_sqrtss_round", IX86_BUILTIN_SQRTSS_ROUND, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
33111 { OPTION_MASK_ISA_AVX512F, CODE_FOR_subv8df3_mask_round, "__builtin_ia32_subpd512_mask", IX86_BUILTIN_SUBPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33112 { OPTION_MASK_ISA_AVX512F, CODE_FOR_subv16sf3_mask_round, "__builtin_ia32_subps512_mask", IX86_BUILTIN_SUBPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33113 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse2_vmsubv2df3_round, "__builtin_ia32_subsd_round", IX86_BUILTIN_SUBSD_ROUND, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
33114 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse_vmsubv4sf3_round, "__builtin_ia32_subss_round", IX86_BUILTIN_SUBSS_ROUND, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
33115 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse2_cvtsd2si_round, "__builtin_ia32_vcvtsd2si32", IX86_BUILTIN_VCVTSD2SI32, UNKNOWN, (int) INT_FTYPE_V2DF_INT },
33116 { OPTION_MASK_ISA_AVX512F | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvtsd2siq_round, "__builtin_ia32_vcvtsd2si64", IX86_BUILTIN_VCVTSD2SI64, UNKNOWN, (int) INT64_FTYPE_V2DF_INT },
33117 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vcvtsd2usi_round, "__builtin_ia32_vcvtsd2usi32", IX86_BUILTIN_VCVTSD2USI32, UNKNOWN, (int) UINT_FTYPE_V2DF_INT },
33118 { OPTION_MASK_ISA_AVX512F | OPTION_MASK_ISA_64BIT, CODE_FOR_avx512f_vcvtsd2usiq_round, "__builtin_ia32_vcvtsd2usi64", IX86_BUILTIN_VCVTSD2USI64, UNKNOWN, (int) UINT64_FTYPE_V2DF_INT },
33119 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse_cvtss2si_round, "__builtin_ia32_vcvtss2si32", IX86_BUILTIN_VCVTSS2SI32, UNKNOWN, (int) INT_FTYPE_V4SF_INT },
33120 { OPTION_MASK_ISA_AVX512F | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvtss2siq_round, "__builtin_ia32_vcvtss2si64", IX86_BUILTIN_VCVTSS2SI64, UNKNOWN, (int) INT64_FTYPE_V4SF_INT },
33121 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vcvtss2usi_round, "__builtin_ia32_vcvtss2usi32", IX86_BUILTIN_VCVTSS2USI32, UNKNOWN, (int) UINT_FTYPE_V4SF_INT },
33122 { OPTION_MASK_ISA_AVX512F | OPTION_MASK_ISA_64BIT, CODE_FOR_avx512f_vcvtss2usiq_round, "__builtin_ia32_vcvtss2usi64", IX86_BUILTIN_VCVTSS2USI64, UNKNOWN, (int) UINT64_FTYPE_V4SF_INT },
33123 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse2_cvttsd2si_round, "__builtin_ia32_vcvttsd2si32", IX86_BUILTIN_VCVTTSD2SI32, UNKNOWN, (int) INT_FTYPE_V2DF_INT },
33124 { OPTION_MASK_ISA_AVX512F | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvttsd2siq_round, "__builtin_ia32_vcvttsd2si64", IX86_BUILTIN_VCVTTSD2SI64, UNKNOWN, (int) INT64_FTYPE_V2DF_INT },
33125 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vcvttsd2usi_round, "__builtin_ia32_vcvttsd2usi32", IX86_BUILTIN_VCVTTSD2USI32, UNKNOWN, (int) UINT_FTYPE_V2DF_INT },
33126 { OPTION_MASK_ISA_AVX512F | OPTION_MASK_ISA_64BIT, CODE_FOR_avx512f_vcvttsd2usiq_round, "__builtin_ia32_vcvttsd2usi64", IX86_BUILTIN_VCVTTSD2USI64, UNKNOWN, (int) UINT64_FTYPE_V2DF_INT },
33127 { OPTION_MASK_ISA_AVX512F, CODE_FOR_sse_cvttss2si_round, "__builtin_ia32_vcvttss2si32", IX86_BUILTIN_VCVTTSS2SI32, UNKNOWN, (int) INT_FTYPE_V4SF_INT },
33128 { OPTION_MASK_ISA_AVX512F | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvttss2siq_round, "__builtin_ia32_vcvttss2si64", IX86_BUILTIN_VCVTTSS2SI64, UNKNOWN, (int) INT64_FTYPE_V4SF_INT },
33129 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_vcvttss2usi_round, "__builtin_ia32_vcvttss2usi32", IX86_BUILTIN_VCVTTSS2USI32, UNKNOWN, (int) UINT_FTYPE_V4SF_INT },
33130 { OPTION_MASK_ISA_AVX512F | OPTION_MASK_ISA_64BIT, CODE_FOR_avx512f_vcvttss2usiq_round, "__builtin_ia32_vcvttss2usi64", IX86_BUILTIN_VCVTTSS2USI64, UNKNOWN, (int) UINT64_FTYPE_V4SF_INT },
33131 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fmadd_v8df_mask_round, "__builtin_ia32_vfmaddpd512_mask", IX86_BUILTIN_VFMADDPD512_MASK, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33132 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fmadd_v8df_mask3_round, "__builtin_ia32_vfmaddpd512_mask3", IX86_BUILTIN_VFMADDPD512_MASK3, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33133 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fmadd_v8df_maskz_round, "__builtin_ia32_vfmaddpd512_maskz", IX86_BUILTIN_VFMADDPD512_MASKZ, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33134 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fmadd_v16sf_mask_round, "__builtin_ia32_vfmaddps512_mask", IX86_BUILTIN_VFMADDPS512_MASK, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33135 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fmadd_v16sf_mask3_round, "__builtin_ia32_vfmaddps512_mask3", IX86_BUILTIN_VFMADDPS512_MASK3, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33136 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fmadd_v16sf_maskz_round, "__builtin_ia32_vfmaddps512_maskz", IX86_BUILTIN_VFMADDPS512_MASKZ, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33137 { OPTION_MASK_ISA_AVX512F, CODE_FOR_fmai_vmfmadd_v2df_round, "__builtin_ia32_vfmaddsd3_round", IX86_BUILTIN_VFMADDSD3_ROUND, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF_INT },
33138 { OPTION_MASK_ISA_AVX512F, CODE_FOR_fmai_vmfmadd_v4sf_round, "__builtin_ia32_vfmaddss3_round", IX86_BUILTIN_VFMADDSS3_ROUND, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF_INT },
33139 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fmaddsub_v8df_mask_round, "__builtin_ia32_vfmaddsubpd512_mask", IX86_BUILTIN_VFMADDSUBPD512_MASK, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33140 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fmaddsub_v8df_mask3_round, "__builtin_ia32_vfmaddsubpd512_mask3", IX86_BUILTIN_VFMADDSUBPD512_MASK3, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33141 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fmaddsub_v8df_maskz_round, "__builtin_ia32_vfmaddsubpd512_maskz", IX86_BUILTIN_VFMADDSUBPD512_MASKZ, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33142 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fmaddsub_v16sf_mask_round, "__builtin_ia32_vfmaddsubps512_mask", IX86_BUILTIN_VFMADDSUBPS512_MASK, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33143 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fmaddsub_v16sf_mask3_round, "__builtin_ia32_vfmaddsubps512_mask3", IX86_BUILTIN_VFMADDSUBPS512_MASK3, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33144 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fmaddsub_v16sf_maskz_round, "__builtin_ia32_vfmaddsubps512_maskz", IX86_BUILTIN_VFMADDSUBPS512_MASKZ, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33145 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fmsubadd_v8df_mask3_round, "__builtin_ia32_vfmsubaddpd512_mask3", IX86_BUILTIN_VFMSUBADDPD512_MASK3, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33146 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fmsubadd_v16sf_mask3_round, "__builtin_ia32_vfmsubaddps512_mask3", IX86_BUILTIN_VFMSUBADDPS512_MASK3, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33147 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fmsub_v8df_mask3_round, "__builtin_ia32_vfmsubpd512_mask3", IX86_BUILTIN_VFMSUBPD512_MASK3, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33148 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fmsub_v16sf_mask3_round, "__builtin_ia32_vfmsubps512_mask3", IX86_BUILTIN_VFMSUBPS512_MASK3, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33149 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fnmadd_v8df_mask_round, "__builtin_ia32_vfnmaddpd512_mask", IX86_BUILTIN_VFNMADDPD512_MASK, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33150 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fnmadd_v16sf_mask_round, "__builtin_ia32_vfnmaddps512_mask", IX86_BUILTIN_VFNMADDPS512_MASK, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33151 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fnmsub_v8df_mask_round, "__builtin_ia32_vfnmsubpd512_mask", IX86_BUILTIN_VFNMSUBPD512_MASK, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33152 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fnmsub_v8df_mask3_round, "__builtin_ia32_vfnmsubpd512_mask3", IX86_BUILTIN_VFNMSUBPD512_MASK3, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT },
33153 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fnmsub_v16sf_mask_round, "__builtin_ia32_vfnmsubps512_mask", IX86_BUILTIN_VFNMSUBPS512_MASK, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33154 { OPTION_MASK_ISA_AVX512F, CODE_FOR_avx512f_fnmsub_v16sf_mask3_round, "__builtin_ia32_vfnmsubps512_mask3", IX86_BUILTIN_VFNMSUBPS512_MASK3, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT },
33157 { OPTION_MASK_ISA_AVX512ER, CODE_FOR_avx512er_exp2v8df_mask_round, "__builtin_ia32_exp2pd_mask", IX86_BUILTIN_EXP2PD_MASK, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_QI_INT },
33158 { OPTION_MASK_ISA_AVX512ER, CODE_FOR_avx512er_exp2v16sf_mask_round, "__builtin_ia32_exp2ps_mask", IX86_BUILTIN_EXP2PS_MASK, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_HI_INT },
33159 { OPTION_MASK_ISA_AVX512ER, CODE_FOR_avx512er_rcp28v8df_mask_round, "__builtin_ia32_rcp28pd_mask", IX86_BUILTIN_RCP28PD, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_QI_INT },
33160 { OPTION_MASK_ISA_AVX512ER, CODE_FOR_avx512er_rcp28v16sf_mask_round, "__builtin_ia32_rcp28ps_mask", IX86_BUILTIN_RCP28PS, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_HI_INT },
33161 { OPTION_MASK_ISA_AVX512ER, CODE_FOR_avx512er_vmrcp28v2df_round, "__builtin_ia32_rcp28sd_round", IX86_BUILTIN_RCP28SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
33162 { OPTION_MASK_ISA_AVX512ER, CODE_FOR_avx512er_vmrcp28v4sf_round, "__builtin_ia32_rcp28ss_round", IX86_BUILTIN_RCP28SS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
33163 { OPTION_MASK_ISA_AVX512ER, CODE_FOR_avx512er_rsqrt28v8df_mask_round, "__builtin_ia32_rsqrt28pd_mask", IX86_BUILTIN_RSQRT28PD, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_QI_INT },
33164 { OPTION_MASK_ISA_AVX512ER, CODE_FOR_avx512er_rsqrt28v16sf_mask_round, "__builtin_ia32_rsqrt28ps_mask", IX86_BUILTIN_RSQRT28PS, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_HI_INT },
33165 { OPTION_MASK_ISA_AVX512ER, CODE_FOR_avx512er_vmrsqrt28v2df_round, "__builtin_ia32_rsqrt28sd_round", IX86_BUILTIN_RSQRT28SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
33166 { OPTION_MASK_ISA_AVX512ER, CODE_FOR_avx512er_vmrsqrt28v4sf_round, "__builtin_ia32_rsqrt28ss_round", IX86_BUILTIN_RSQRT28SS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
33169 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_rangesv2df_round, "__builtin_ia32_rangesd128_round", IX86_BUILTIN_RANGESD128, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT_INT },
33170 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_rangesv4sf_round, "__builtin_ia32_rangess128_round", IX86_BUILTIN_RANGESS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT_INT },
33171 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_fix_notruncv8dfv8di2_mask_round, "__builtin_ia32_cvtpd2qq512_mask", IX86_BUILTIN_CVTPD2QQ512, UNKNOWN, (int) V8DI_FTYPE_V8DF_V8DI_QI_INT },
33172 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_cvtps2qqv8di_mask_round, "__builtin_ia32_cvtps2qq512_mask", IX86_BUILTIN_CVTPS2QQ512, UNKNOWN, (int) V8DI_FTYPE_V8SF_V8DI_QI_INT },
33173 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_ufix_notruncv8dfv8di2_mask_round, "__builtin_ia32_cvtpd2uqq512_mask", IX86_BUILTIN_CVTPD2UQQ512, UNKNOWN, (int) V8DI_FTYPE_V8DF_V8DI_QI_INT },
33174 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_cvtps2uqqv8di_mask_round, "__builtin_ia32_cvtps2uqq512_mask", IX86_BUILTIN_CVTPS2UQQ512, UNKNOWN, (int) V8DI_FTYPE_V8SF_V8DI_QI_INT },
33175 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_floatv8div8sf2_mask_round, "__builtin_ia32_cvtqq2ps512_mask", IX86_BUILTIN_CVTQQ2PS512, UNKNOWN, (int) V8SF_FTYPE_V8DI_V8SF_QI_INT },
33176 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_ufloatv8div8sf2_mask_round, "__builtin_ia32_cvtuqq2ps512_mask", IX86_BUILTIN_CVTUQQ2PS512, UNKNOWN, (int) V8SF_FTYPE_V8DI_V8SF_QI_INT },
33177 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_floatv8div8df2_mask_round, "__builtin_ia32_cvtqq2pd512_mask", IX86_BUILTIN_CVTQQ2PD512, UNKNOWN, (int) V8DF_FTYPE_V8DI_V8DF_QI_INT },
33178 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_ufloatv8div8df2_mask_round, "__builtin_ia32_cvtuqq2pd512_mask", IX86_BUILTIN_CVTUQQ2PD512, UNKNOWN, (int) V8DF_FTYPE_V8DI_V8DF_QI_INT },
33179 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_fix_truncv8sfv8di2_mask_round, "__builtin_ia32_cvttps2qq512_mask", IX86_BUILTIN_CVTTPS2QQ512, UNKNOWN, (int) V8DI_FTYPE_V8SF_V8DI_QI_INT },
33180 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_ufix_truncv8sfv8di2_mask_round, "__builtin_ia32_cvttps2uqq512_mask", IX86_BUILTIN_CVTTPS2UQQ512, UNKNOWN, (int) V8DI_FTYPE_V8SF_V8DI_QI_INT },
33181 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_fix_truncv8dfv8di2_mask_round, "__builtin_ia32_cvttpd2qq512_mask", IX86_BUILTIN_CVTTPD2QQ512, UNKNOWN, (int) V8DI_FTYPE_V8DF_V8DI_QI_INT },
33182 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_ufix_truncv8dfv8di2_mask_round, "__builtin_ia32_cvttpd2uqq512_mask", IX86_BUILTIN_CVTTPD2UQQ512, UNKNOWN, (int) V8DI_FTYPE_V8DF_V8DI_QI_INT },
33183 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_rangepv16sf_mask_round, "__builtin_ia32_rangeps512_mask", IX86_BUILTIN_RANGEPS512, UNKNOWN, (int) V16SF_FTYPE_V16SF_V16SF_INT_V16SF_HI_INT },
33184 { OPTION_MASK_ISA_AVX512DQ, CODE_FOR_avx512dq_rangepv8df_mask_round, "__builtin_ia32_rangepd512_mask", IX86_BUILTIN_RANGEPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF_V8DF_INT_V8DF_QI_INT },
33187 /* Bultins for MPX. */
33188 static const struct builtin_description bdesc_mpx[] =
33190 { OPTION_MASK_ISA_MPX, (enum insn_code)0, "__builtin_ia32_bndstx", IX86_BUILTIN_BNDSTX, UNKNOWN, (int) VOID_FTYPE_PCVOID_BND_PCVOID },
33191 { OPTION_MASK_ISA_MPX, (enum insn_code)0, "__builtin_ia32_bndcl", IX86_BUILTIN_BNDCL, UNKNOWN, (int) VOID_FTYPE_PCVOID_BND },
33192 { OPTION_MASK_ISA_MPX, (enum insn_code)0, "__builtin_ia32_bndcu", IX86_BUILTIN_BNDCU, UNKNOWN, (int) VOID_FTYPE_PCVOID_BND },
33195 /* Const builtins for MPX. */
33196 static const struct builtin_description bdesc_mpx_const[] =
33198 { OPTION_MASK_ISA_MPX, (enum insn_code)0, "__builtin_ia32_bndmk", IX86_BUILTIN_BNDMK, UNKNOWN, (int) BND_FTYPE_PCVOID_ULONG },
33199 { OPTION_MASK_ISA_MPX, (enum insn_code)0, "__builtin_ia32_bndldx", IX86_BUILTIN_BNDLDX, UNKNOWN, (int) BND_FTYPE_PCVOID_PCVOID },
33200 { OPTION_MASK_ISA_MPX, (enum insn_code)0, "__builtin_ia32_narrow_bounds", IX86_BUILTIN_BNDNARROW, UNKNOWN, (int) PVOID_FTYPE_PCVOID_BND_ULONG },
33201 { OPTION_MASK_ISA_MPX, (enum insn_code)0, "__builtin_ia32_bndint", IX86_BUILTIN_BNDINT, UNKNOWN, (int) BND_FTYPE_BND_BND },
33202 { OPTION_MASK_ISA_MPX, (enum insn_code)0, "__builtin_ia32_sizeof", IX86_BUILTIN_SIZEOF, UNKNOWN, (int) ULONG_FTYPE_VOID },
33203 { OPTION_MASK_ISA_MPX, (enum insn_code)0, "__builtin_ia32_bndlower", IX86_BUILTIN_BNDLOWER, UNKNOWN, (int) PVOID_FTYPE_BND },
33204 { OPTION_MASK_ISA_MPX, (enum insn_code)0, "__builtin_ia32_bndupper", IX86_BUILTIN_BNDUPPER, UNKNOWN, (int) PVOID_FTYPE_BND },
33205 { OPTION_MASK_ISA_MPX, (enum insn_code)0, "__builtin_ia32_bndret", IX86_BUILTIN_BNDRET, UNKNOWN, (int) BND_FTYPE_PCVOID },
33208 /* FMA4 and XOP. */
33209 #define MULTI_ARG_4_DF2_DI_I V2DF_FTYPE_V2DF_V2DF_V2DI_INT
33210 #define MULTI_ARG_4_DF2_DI_I1 V4DF_FTYPE_V4DF_V4DF_V4DI_INT
33211 #define MULTI_ARG_4_SF2_SI_I V4SF_FTYPE_V4SF_V4SF_V4SI_INT
33212 #define MULTI_ARG_4_SF2_SI_I1 V8SF_FTYPE_V8SF_V8SF_V8SI_INT
33213 #define MULTI_ARG_3_SF V4SF_FTYPE_V4SF_V4SF_V4SF
33214 #define MULTI_ARG_3_DF V2DF_FTYPE_V2DF_V2DF_V2DF
33215 #define MULTI_ARG_3_SF2 V8SF_FTYPE_V8SF_V8SF_V8SF
33216 #define MULTI_ARG_3_DF2 V4DF_FTYPE_V4DF_V4DF_V4DF
33217 #define MULTI_ARG_3_DI V2DI_FTYPE_V2DI_V2DI_V2DI
33218 #define MULTI_ARG_3_SI V4SI_FTYPE_V4SI_V4SI_V4SI
33219 #define MULTI_ARG_3_SI_DI V4SI_FTYPE_V4SI_V4SI_V2DI
33220 #define MULTI_ARG_3_HI V8HI_FTYPE_V8HI_V8HI_V8HI
33221 #define MULTI_ARG_3_HI_SI V8HI_FTYPE_V8HI_V8HI_V4SI
33222 #define MULTI_ARG_3_QI V16QI_FTYPE_V16QI_V16QI_V16QI
33223 #define MULTI_ARG_3_DI2 V4DI_FTYPE_V4DI_V4DI_V4DI
33224 #define MULTI_ARG_3_SI2 V8SI_FTYPE_V8SI_V8SI_V8SI
33225 #define MULTI_ARG_3_HI2 V16HI_FTYPE_V16HI_V16HI_V16HI
33226 #define MULTI_ARG_3_QI2 V32QI_FTYPE_V32QI_V32QI_V32QI
33227 #define MULTI_ARG_2_SF V4SF_FTYPE_V4SF_V4SF
33228 #define MULTI_ARG_2_DF V2DF_FTYPE_V2DF_V2DF
33229 #define MULTI_ARG_2_DI V2DI_FTYPE_V2DI_V2DI
33230 #define MULTI_ARG_2_SI V4SI_FTYPE_V4SI_V4SI
33231 #define MULTI_ARG_2_HI V8HI_FTYPE_V8HI_V8HI
33232 #define MULTI_ARG_2_QI V16QI_FTYPE_V16QI_V16QI
33233 #define MULTI_ARG_2_DI_IMM V2DI_FTYPE_V2DI_SI
33234 #define MULTI_ARG_2_SI_IMM V4SI_FTYPE_V4SI_SI
33235 #define MULTI_ARG_2_HI_IMM V8HI_FTYPE_V8HI_SI
33236 #define MULTI_ARG_2_QI_IMM V16QI_FTYPE_V16QI_SI
33237 #define MULTI_ARG_2_DI_CMP V2DI_FTYPE_V2DI_V2DI_CMP
33238 #define MULTI_ARG_2_SI_CMP V4SI_FTYPE_V4SI_V4SI_CMP
33239 #define MULTI_ARG_2_HI_CMP V8HI_FTYPE_V8HI_V8HI_CMP
33240 #define MULTI_ARG_2_QI_CMP V16QI_FTYPE_V16QI_V16QI_CMP
33241 #define MULTI_ARG_2_SF_TF V4SF_FTYPE_V4SF_V4SF_TF
33242 #define MULTI_ARG_2_DF_TF V2DF_FTYPE_V2DF_V2DF_TF
33243 #define MULTI_ARG_2_DI_TF V2DI_FTYPE_V2DI_V2DI_TF
33244 #define MULTI_ARG_2_SI_TF V4SI_FTYPE_V4SI_V4SI_TF
33245 #define MULTI_ARG_2_HI_TF V8HI_FTYPE_V8HI_V8HI_TF
33246 #define MULTI_ARG_2_QI_TF V16QI_FTYPE_V16QI_V16QI_TF
33247 #define MULTI_ARG_1_SF V4SF_FTYPE_V4SF
33248 #define MULTI_ARG_1_DF V2DF_FTYPE_V2DF
33249 #define MULTI_ARG_1_SF2 V8SF_FTYPE_V8SF
33250 #define MULTI_ARG_1_DF2 V4DF_FTYPE_V4DF
33251 #define MULTI_ARG_1_DI V2DI_FTYPE_V2DI
33252 #define MULTI_ARG_1_SI V4SI_FTYPE_V4SI
33253 #define MULTI_ARG_1_HI V8HI_FTYPE_V8HI
33254 #define MULTI_ARG_1_QI V16QI_FTYPE_V16QI
33255 #define MULTI_ARG_1_SI_DI V2DI_FTYPE_V4SI
33256 #define MULTI_ARG_1_HI_DI V2DI_FTYPE_V8HI
33257 #define MULTI_ARG_1_HI_SI V4SI_FTYPE_V8HI
33258 #define MULTI_ARG_1_QI_DI V2DI_FTYPE_V16QI
33259 #define MULTI_ARG_1_QI_SI V4SI_FTYPE_V16QI
33260 #define MULTI_ARG_1_QI_HI V8HI_FTYPE_V16QI
33262 static const struct builtin_description bdesc_multi_arg[] =
33264 { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_vmfmadd_v4sf,
33265 "__builtin_ia32_vfmaddss", IX86_BUILTIN_VFMADDSS,
33266 UNKNOWN, (int)MULTI_ARG_3_SF },
33267 { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_vmfmadd_v2df,
33268 "__builtin_ia32_vfmaddsd", IX86_BUILTIN_VFMADDSD,
33269 UNKNOWN, (int)MULTI_ARG_3_DF },
33271 { OPTION_MASK_ISA_FMA, CODE_FOR_fmai_vmfmadd_v4sf,
33272 "__builtin_ia32_vfmaddss3", IX86_BUILTIN_VFMADDSS3,
33273 UNKNOWN, (int)MULTI_ARG_3_SF },
33274 { OPTION_MASK_ISA_FMA, CODE_FOR_fmai_vmfmadd_v2df,
33275 "__builtin_ia32_vfmaddsd3", IX86_BUILTIN_VFMADDSD3,
33276 UNKNOWN, (int)MULTI_ARG_3_DF },
33278 { OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmadd_v4sf,
33279 "__builtin_ia32_vfmaddps", IX86_BUILTIN_VFMADDPS,
33280 UNKNOWN, (int)MULTI_ARG_3_SF },
33281 { OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmadd_v2df,
33282 "__builtin_ia32_vfmaddpd", IX86_BUILTIN_VFMADDPD,
33283 UNKNOWN, (int)MULTI_ARG_3_DF },
33284 { OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmadd_v8sf,
33285 "__builtin_ia32_vfmaddps256", IX86_BUILTIN_VFMADDPS256,
33286 UNKNOWN, (int)MULTI_ARG_3_SF2 },
33287 { OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmadd_v4df,
33288 "__builtin_ia32_vfmaddpd256", IX86_BUILTIN_VFMADDPD256,
33289 UNKNOWN, (int)MULTI_ARG_3_DF2 },
33291 { OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4, CODE_FOR_fmaddsub_v4sf,
33292 "__builtin_ia32_vfmaddsubps", IX86_BUILTIN_VFMADDSUBPS,
33293 UNKNOWN, (int)MULTI_ARG_3_SF },
33294 { OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4, CODE_FOR_fmaddsub_v2df,
33295 "__builtin_ia32_vfmaddsubpd", IX86_BUILTIN_VFMADDSUBPD,
33296 UNKNOWN, (int)MULTI_ARG_3_DF },
33297 { OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4, CODE_FOR_fmaddsub_v8sf,
33298 "__builtin_ia32_vfmaddsubps256", IX86_BUILTIN_VFMADDSUBPS256,
33299 UNKNOWN, (int)MULTI_ARG_3_SF2 },
33300 { OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4, CODE_FOR_fmaddsub_v4df,
33301 "__builtin_ia32_vfmaddsubpd256", IX86_BUILTIN_VFMADDSUBPD256,
33302 UNKNOWN, (int)MULTI_ARG_3_DF2 },
33304 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v2di, "__builtin_ia32_vpcmov", IX86_BUILTIN_VPCMOV, UNKNOWN, (int)MULTI_ARG_3_DI },
33305 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v2di, "__builtin_ia32_vpcmov_v2di", IX86_BUILTIN_VPCMOV_V2DI, UNKNOWN, (int)MULTI_ARG_3_DI },
33306 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4si, "__builtin_ia32_vpcmov_v4si", IX86_BUILTIN_VPCMOV_V4SI, UNKNOWN, (int)MULTI_ARG_3_SI },
33307 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v8hi, "__builtin_ia32_vpcmov_v8hi", IX86_BUILTIN_VPCMOV_V8HI, UNKNOWN, (int)MULTI_ARG_3_HI },
33308 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v16qi, "__builtin_ia32_vpcmov_v16qi",IX86_BUILTIN_VPCMOV_V16QI,UNKNOWN, (int)MULTI_ARG_3_QI },
33309 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v2df, "__builtin_ia32_vpcmov_v2df", IX86_BUILTIN_VPCMOV_V2DF, UNKNOWN, (int)MULTI_ARG_3_DF },
33310 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4sf, "__builtin_ia32_vpcmov_v4sf", IX86_BUILTIN_VPCMOV_V4SF, UNKNOWN, (int)MULTI_ARG_3_SF },
33312 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4di256, "__builtin_ia32_vpcmov256", IX86_BUILTIN_VPCMOV256, UNKNOWN, (int)MULTI_ARG_3_DI2 },
33313 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4di256, "__builtin_ia32_vpcmov_v4di256", IX86_BUILTIN_VPCMOV_V4DI256, UNKNOWN, (int)MULTI_ARG_3_DI2 },
33314 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v8si256, "__builtin_ia32_vpcmov_v8si256", IX86_BUILTIN_VPCMOV_V8SI256, UNKNOWN, (int)MULTI_ARG_3_SI2 },
33315 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v16hi256, "__builtin_ia32_vpcmov_v16hi256", IX86_BUILTIN_VPCMOV_V16HI256, UNKNOWN, (int)MULTI_ARG_3_HI2 },
33316 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v32qi256, "__builtin_ia32_vpcmov_v32qi256", IX86_BUILTIN_VPCMOV_V32QI256, UNKNOWN, (int)MULTI_ARG_3_QI2 },
33317 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4df256, "__builtin_ia32_vpcmov_v4df256", IX86_BUILTIN_VPCMOV_V4DF256, UNKNOWN, (int)MULTI_ARG_3_DF2 },
33318 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v8sf256, "__builtin_ia32_vpcmov_v8sf256", IX86_BUILTIN_VPCMOV_V8SF256, UNKNOWN, (int)MULTI_ARG_3_SF2 },
33320 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pperm, "__builtin_ia32_vpperm", IX86_BUILTIN_VPPERM, UNKNOWN, (int)MULTI_ARG_3_QI },
33322 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacssww, "__builtin_ia32_vpmacssww", IX86_BUILTIN_VPMACSSWW, UNKNOWN, (int)MULTI_ARG_3_HI },
33323 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsww, "__builtin_ia32_vpmacsww", IX86_BUILTIN_VPMACSWW, UNKNOWN, (int)MULTI_ARG_3_HI },
33324 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsswd, "__builtin_ia32_vpmacsswd", IX86_BUILTIN_VPMACSSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
33325 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacswd, "__builtin_ia32_vpmacswd", IX86_BUILTIN_VPMACSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
33326 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacssdd, "__builtin_ia32_vpmacssdd", IX86_BUILTIN_VPMACSSDD, UNKNOWN, (int)MULTI_ARG_3_SI },
33327 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsdd, "__builtin_ia32_vpmacsdd", IX86_BUILTIN_VPMACSDD, UNKNOWN, (int)MULTI_ARG_3_SI },
33328 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacssdql, "__builtin_ia32_vpmacssdql", IX86_BUILTIN_VPMACSSDQL, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
33329 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacssdqh, "__builtin_ia32_vpmacssdqh", IX86_BUILTIN_VPMACSSDQH, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
33330 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsdql, "__builtin_ia32_vpmacsdql", IX86_BUILTIN_VPMACSDQL, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
33331 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsdqh, "__builtin_ia32_vpmacsdqh", IX86_BUILTIN_VPMACSDQH, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
33332 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmadcsswd, "__builtin_ia32_vpmadcsswd", IX86_BUILTIN_VPMADCSSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
33333 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmadcswd, "__builtin_ia32_vpmadcswd", IX86_BUILTIN_VPMADCSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
33335 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vrotlv2di3, "__builtin_ia32_vprotq", IX86_BUILTIN_VPROTQ, UNKNOWN, (int)MULTI_ARG_2_DI },
33336 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vrotlv4si3, "__builtin_ia32_vprotd", IX86_BUILTIN_VPROTD, UNKNOWN, (int)MULTI_ARG_2_SI },
33337 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vrotlv8hi3, "__builtin_ia32_vprotw", IX86_BUILTIN_VPROTW, UNKNOWN, (int)MULTI_ARG_2_HI },
33338 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vrotlv16qi3, "__builtin_ia32_vprotb", IX86_BUILTIN_VPROTB, UNKNOWN, (int)MULTI_ARG_2_QI },
33339 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv2di3, "__builtin_ia32_vprotqi", IX86_BUILTIN_VPROTQ_IMM, UNKNOWN, (int)MULTI_ARG_2_DI_IMM },
33340 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv4si3, "__builtin_ia32_vprotdi", IX86_BUILTIN_VPROTD_IMM, UNKNOWN, (int)MULTI_ARG_2_SI_IMM },
33341 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv8hi3, "__builtin_ia32_vprotwi", IX86_BUILTIN_VPROTW_IMM, UNKNOWN, (int)MULTI_ARG_2_HI_IMM },
33342 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv16qi3, "__builtin_ia32_vprotbi", IX86_BUILTIN_VPROTB_IMM, UNKNOWN, (int)MULTI_ARG_2_QI_IMM },
33343 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav2di3, "__builtin_ia32_vpshaq", IX86_BUILTIN_VPSHAQ, UNKNOWN, (int)MULTI_ARG_2_DI },
33344 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav4si3, "__builtin_ia32_vpshad", IX86_BUILTIN_VPSHAD, UNKNOWN, (int)MULTI_ARG_2_SI },
33345 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav8hi3, "__builtin_ia32_vpshaw", IX86_BUILTIN_VPSHAW, UNKNOWN, (int)MULTI_ARG_2_HI },
33346 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav16qi3, "__builtin_ia32_vpshab", IX86_BUILTIN_VPSHAB, UNKNOWN, (int)MULTI_ARG_2_QI },
33347 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv2di3, "__builtin_ia32_vpshlq", IX86_BUILTIN_VPSHLQ, UNKNOWN, (int)MULTI_ARG_2_DI },
33348 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv4si3, "__builtin_ia32_vpshld", IX86_BUILTIN_VPSHLD, UNKNOWN, (int)MULTI_ARG_2_SI },
33349 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv8hi3, "__builtin_ia32_vpshlw", IX86_BUILTIN_VPSHLW, UNKNOWN, (int)MULTI_ARG_2_HI },
33350 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv16qi3, "__builtin_ia32_vpshlb", IX86_BUILTIN_VPSHLB, UNKNOWN, (int)MULTI_ARG_2_QI },
33352 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vmfrczv4sf2, "__builtin_ia32_vfrczss", IX86_BUILTIN_VFRCZSS, UNKNOWN, (int)MULTI_ARG_1_SF },
33353 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vmfrczv2df2, "__builtin_ia32_vfrczsd", IX86_BUILTIN_VFRCZSD, UNKNOWN, (int)MULTI_ARG_1_DF },
33354 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv4sf2, "__builtin_ia32_vfrczps", IX86_BUILTIN_VFRCZPS, UNKNOWN, (int)MULTI_ARG_1_SF },
33355 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv2df2, "__builtin_ia32_vfrczpd", IX86_BUILTIN_VFRCZPD, UNKNOWN, (int)MULTI_ARG_1_DF },
33356 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv8sf2, "__builtin_ia32_vfrczps256", IX86_BUILTIN_VFRCZPS256, UNKNOWN, (int)MULTI_ARG_1_SF2 },
33357 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv4df2, "__builtin_ia32_vfrczpd256", IX86_BUILTIN_VFRCZPD256, UNKNOWN, (int)MULTI_ARG_1_DF2 },
33359 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddbw, "__builtin_ia32_vphaddbw", IX86_BUILTIN_VPHADDBW, UNKNOWN, (int)MULTI_ARG_1_QI_HI },
33360 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddbd, "__builtin_ia32_vphaddbd", IX86_BUILTIN_VPHADDBD, UNKNOWN, (int)MULTI_ARG_1_QI_SI },
33361 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddbq, "__builtin_ia32_vphaddbq", IX86_BUILTIN_VPHADDBQ, UNKNOWN, (int)MULTI_ARG_1_QI_DI },
33362 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddwd, "__builtin_ia32_vphaddwd", IX86_BUILTIN_VPHADDWD, UNKNOWN, (int)MULTI_ARG_1_HI_SI },
33363 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddwq, "__builtin_ia32_vphaddwq", IX86_BUILTIN_VPHADDWQ, UNKNOWN, (int)MULTI_ARG_1_HI_DI },
33364 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phadddq, "__builtin_ia32_vphadddq", IX86_BUILTIN_VPHADDDQ, UNKNOWN, (int)MULTI_ARG_1_SI_DI },
33365 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddubw, "__builtin_ia32_vphaddubw", IX86_BUILTIN_VPHADDUBW, UNKNOWN, (int)MULTI_ARG_1_QI_HI },
33366 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddubd, "__builtin_ia32_vphaddubd", IX86_BUILTIN_VPHADDUBD, UNKNOWN, (int)MULTI_ARG_1_QI_SI },
33367 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddubq, "__builtin_ia32_vphaddubq", IX86_BUILTIN_VPHADDUBQ, UNKNOWN, (int)MULTI_ARG_1_QI_DI },
33368 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phadduwd, "__builtin_ia32_vphadduwd", IX86_BUILTIN_VPHADDUWD, UNKNOWN, (int)MULTI_ARG_1_HI_SI },
33369 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phadduwq, "__builtin_ia32_vphadduwq", IX86_BUILTIN_VPHADDUWQ, UNKNOWN, (int)MULTI_ARG_1_HI_DI },
33370 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddudq, "__builtin_ia32_vphaddudq", IX86_BUILTIN_VPHADDUDQ, UNKNOWN, (int)MULTI_ARG_1_SI_DI },
33371 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phsubbw, "__builtin_ia32_vphsubbw", IX86_BUILTIN_VPHSUBBW, UNKNOWN, (int)MULTI_ARG_1_QI_HI },
33372 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phsubwd, "__builtin_ia32_vphsubwd", IX86_BUILTIN_VPHSUBWD, UNKNOWN, (int)MULTI_ARG_1_HI_SI },
33373 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phsubdq, "__builtin_ia32_vphsubdq", IX86_BUILTIN_VPHSUBDQ, UNKNOWN, (int)MULTI_ARG_1_SI_DI },
33375 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomeqb", IX86_BUILTIN_VPCOMEQB, EQ, (int)MULTI_ARG_2_QI_CMP },
33376 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomneb", IX86_BUILTIN_VPCOMNEB, NE, (int)MULTI_ARG_2_QI_CMP },
33377 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomneqb", IX86_BUILTIN_VPCOMNEB, NE, (int)MULTI_ARG_2_QI_CMP },
33378 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomltb", IX86_BUILTIN_VPCOMLTB, LT, (int)MULTI_ARG_2_QI_CMP },
33379 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomleb", IX86_BUILTIN_VPCOMLEB, LE, (int)MULTI_ARG_2_QI_CMP },
33380 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomgtb", IX86_BUILTIN_VPCOMGTB, GT, (int)MULTI_ARG_2_QI_CMP },
33381 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomgeb", IX86_BUILTIN_VPCOMGEB, GE, (int)MULTI_ARG_2_QI_CMP },
33383 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomeqw", IX86_BUILTIN_VPCOMEQW, EQ, (int)MULTI_ARG_2_HI_CMP },
33384 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomnew", IX86_BUILTIN_VPCOMNEW, NE, (int)MULTI_ARG_2_HI_CMP },
33385 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomneqw", IX86_BUILTIN_VPCOMNEW, NE, (int)MULTI_ARG_2_HI_CMP },
33386 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomltw", IX86_BUILTIN_VPCOMLTW, LT, (int)MULTI_ARG_2_HI_CMP },
33387 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomlew", IX86_BUILTIN_VPCOMLEW, LE, (int)MULTI_ARG_2_HI_CMP },
33388 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomgtw", IX86_BUILTIN_VPCOMGTW, GT, (int)MULTI_ARG_2_HI_CMP },
33389 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomgew", IX86_BUILTIN_VPCOMGEW, GE, (int)MULTI_ARG_2_HI_CMP },
33391 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomeqd", IX86_BUILTIN_VPCOMEQD, EQ, (int)MULTI_ARG_2_SI_CMP },
33392 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomned", IX86_BUILTIN_VPCOMNED, NE, (int)MULTI_ARG_2_SI_CMP },
33393 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomneqd", IX86_BUILTIN_VPCOMNED, NE, (int)MULTI_ARG_2_SI_CMP },
33394 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomltd", IX86_BUILTIN_VPCOMLTD, LT, (int)MULTI_ARG_2_SI_CMP },
33395 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomled", IX86_BUILTIN_VPCOMLED, LE, (int)MULTI_ARG_2_SI_CMP },
33396 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomgtd", IX86_BUILTIN_VPCOMGTD, GT, (int)MULTI_ARG_2_SI_CMP },
33397 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomged", IX86_BUILTIN_VPCOMGED, GE, (int)MULTI_ARG_2_SI_CMP },
33399 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomeqq", IX86_BUILTIN_VPCOMEQQ, EQ, (int)MULTI_ARG_2_DI_CMP },
33400 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomneq", IX86_BUILTIN_VPCOMNEQ, NE, (int)MULTI_ARG_2_DI_CMP },
33401 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomneqq", IX86_BUILTIN_VPCOMNEQ, NE, (int)MULTI_ARG_2_DI_CMP },
33402 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomltq", IX86_BUILTIN_VPCOMLTQ, LT, (int)MULTI_ARG_2_DI_CMP },
33403 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomleq", IX86_BUILTIN_VPCOMLEQ, LE, (int)MULTI_ARG_2_DI_CMP },
33404 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomgtq", IX86_BUILTIN_VPCOMGTQ, GT, (int)MULTI_ARG_2_DI_CMP },
33405 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomgeq", IX86_BUILTIN_VPCOMGEQ, GE, (int)MULTI_ARG_2_DI_CMP },
33407 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v16qi3,"__builtin_ia32_vpcomequb", IX86_BUILTIN_VPCOMEQUB, EQ, (int)MULTI_ARG_2_QI_CMP },
33408 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v16qi3,"__builtin_ia32_vpcomneub", IX86_BUILTIN_VPCOMNEUB, NE, (int)MULTI_ARG_2_QI_CMP },
33409 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v16qi3,"__builtin_ia32_vpcomnequb", IX86_BUILTIN_VPCOMNEUB, NE, (int)MULTI_ARG_2_QI_CMP },
33410 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv16qi3, "__builtin_ia32_vpcomltub", IX86_BUILTIN_VPCOMLTUB, LTU, (int)MULTI_ARG_2_QI_CMP },
33411 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv16qi3, "__builtin_ia32_vpcomleub", IX86_BUILTIN_VPCOMLEUB, LEU, (int)MULTI_ARG_2_QI_CMP },
33412 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv16qi3, "__builtin_ia32_vpcomgtub", IX86_BUILTIN_VPCOMGTUB, GTU, (int)MULTI_ARG_2_QI_CMP },
33413 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv16qi3, "__builtin_ia32_vpcomgeub", IX86_BUILTIN_VPCOMGEUB, GEU, (int)MULTI_ARG_2_QI_CMP },
33415 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v8hi3, "__builtin_ia32_vpcomequw", IX86_BUILTIN_VPCOMEQUW, EQ, (int)MULTI_ARG_2_HI_CMP },
33416 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v8hi3, "__builtin_ia32_vpcomneuw", IX86_BUILTIN_VPCOMNEUW, NE, (int)MULTI_ARG_2_HI_CMP },
33417 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v8hi3, "__builtin_ia32_vpcomnequw", IX86_BUILTIN_VPCOMNEUW, NE, (int)MULTI_ARG_2_HI_CMP },
33418 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv8hi3, "__builtin_ia32_vpcomltuw", IX86_BUILTIN_VPCOMLTUW, LTU, (int)MULTI_ARG_2_HI_CMP },
33419 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv8hi3, "__builtin_ia32_vpcomleuw", IX86_BUILTIN_VPCOMLEUW, LEU, (int)MULTI_ARG_2_HI_CMP },
33420 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv8hi3, "__builtin_ia32_vpcomgtuw", IX86_BUILTIN_VPCOMGTUW, GTU, (int)MULTI_ARG_2_HI_CMP },
33421 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv8hi3, "__builtin_ia32_vpcomgeuw", IX86_BUILTIN_VPCOMGEUW, GEU, (int)MULTI_ARG_2_HI_CMP },
33423 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v4si3, "__builtin_ia32_vpcomequd", IX86_BUILTIN_VPCOMEQUD, EQ, (int)MULTI_ARG_2_SI_CMP },
33424 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v4si3, "__builtin_ia32_vpcomneud", IX86_BUILTIN_VPCOMNEUD, NE, (int)MULTI_ARG_2_SI_CMP },
33425 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v4si3, "__builtin_ia32_vpcomnequd", IX86_BUILTIN_VPCOMNEUD, NE, (int)MULTI_ARG_2_SI_CMP },
33426 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv4si3, "__builtin_ia32_vpcomltud", IX86_BUILTIN_VPCOMLTUD, LTU, (int)MULTI_ARG_2_SI_CMP },
33427 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv4si3, "__builtin_ia32_vpcomleud", IX86_BUILTIN_VPCOMLEUD, LEU, (int)MULTI_ARG_2_SI_CMP },
33428 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv4si3, "__builtin_ia32_vpcomgtud", IX86_BUILTIN_VPCOMGTUD, GTU, (int)MULTI_ARG_2_SI_CMP },
33429 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv4si3, "__builtin_ia32_vpcomgeud", IX86_BUILTIN_VPCOMGEUD, GEU, (int)MULTI_ARG_2_SI_CMP },
33431 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v2di3, "__builtin_ia32_vpcomequq", IX86_BUILTIN_VPCOMEQUQ, EQ, (int)MULTI_ARG_2_DI_CMP },
33432 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v2di3, "__builtin_ia32_vpcomneuq", IX86_BUILTIN_VPCOMNEUQ, NE, (int)MULTI_ARG_2_DI_CMP },
33433 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v2di3, "__builtin_ia32_vpcomnequq", IX86_BUILTIN_VPCOMNEUQ, NE, (int)MULTI_ARG_2_DI_CMP },
33434 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv2di3, "__builtin_ia32_vpcomltuq", IX86_BUILTIN_VPCOMLTUQ, LTU, (int)MULTI_ARG_2_DI_CMP },
33435 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv2di3, "__builtin_ia32_vpcomleuq", IX86_BUILTIN_VPCOMLEUQ, LEU, (int)MULTI_ARG_2_DI_CMP },
33436 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv2di3, "__builtin_ia32_vpcomgtuq", IX86_BUILTIN_VPCOMGTUQ, GTU, (int)MULTI_ARG_2_DI_CMP },
33437 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv2di3, "__builtin_ia32_vpcomgeuq", IX86_BUILTIN_VPCOMGEUQ, GEU, (int)MULTI_ARG_2_DI_CMP },
33439 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv16qi3, "__builtin_ia32_vpcomfalseb", IX86_BUILTIN_VPCOMFALSEB, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_QI_TF },
33440 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv8hi3, "__builtin_ia32_vpcomfalsew", IX86_BUILTIN_VPCOMFALSEW, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_HI_TF },
33441 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv4si3, "__builtin_ia32_vpcomfalsed", IX86_BUILTIN_VPCOMFALSED, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_SI_TF },
33442 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv2di3, "__builtin_ia32_vpcomfalseq", IX86_BUILTIN_VPCOMFALSEQ, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_DI_TF },
33443 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv16qi3, "__builtin_ia32_vpcomfalseub",IX86_BUILTIN_VPCOMFALSEUB,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_QI_TF },
33444 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv8hi3, "__builtin_ia32_vpcomfalseuw",IX86_BUILTIN_VPCOMFALSEUW,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_HI_TF },
33445 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv4si3, "__builtin_ia32_vpcomfalseud",IX86_BUILTIN_VPCOMFALSEUD,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_SI_TF },
33446 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv2di3, "__builtin_ia32_vpcomfalseuq",IX86_BUILTIN_VPCOMFALSEUQ,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_DI_TF },
33448 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv16qi3, "__builtin_ia32_vpcomtrueb", IX86_BUILTIN_VPCOMTRUEB, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_QI_TF },
33449 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv8hi3, "__builtin_ia32_vpcomtruew", IX86_BUILTIN_VPCOMTRUEW, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_HI_TF },
33450 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv4si3, "__builtin_ia32_vpcomtrued", IX86_BUILTIN_VPCOMTRUED, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_SI_TF },
33451 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv2di3, "__builtin_ia32_vpcomtrueq", IX86_BUILTIN_VPCOMTRUEQ, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_DI_TF },
33452 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv16qi3, "__builtin_ia32_vpcomtrueub", IX86_BUILTIN_VPCOMTRUEUB, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_QI_TF },
33453 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv8hi3, "__builtin_ia32_vpcomtrueuw", IX86_BUILTIN_VPCOMTRUEUW, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_HI_TF },
33454 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv4si3, "__builtin_ia32_vpcomtrueud", IX86_BUILTIN_VPCOMTRUEUD, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_SI_TF },
33455 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv2di3, "__builtin_ia32_vpcomtrueuq", IX86_BUILTIN_VPCOMTRUEUQ, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_DI_TF },
33457 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v2df3, "__builtin_ia32_vpermil2pd", IX86_BUILTIN_VPERMIL2PD, UNKNOWN, (int)MULTI_ARG_4_DF2_DI_I },
33458 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v4sf3, "__builtin_ia32_vpermil2ps", IX86_BUILTIN_VPERMIL2PS, UNKNOWN, (int)MULTI_ARG_4_SF2_SI_I },
33459 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v4df3, "__builtin_ia32_vpermil2pd256", IX86_BUILTIN_VPERMIL2PD256, UNKNOWN, (int)MULTI_ARG_4_DF2_DI_I1 },
33460 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v8sf3, "__builtin_ia32_vpermil2ps256", IX86_BUILTIN_VPERMIL2PS256, UNKNOWN, (int)MULTI_ARG_4_SF2_SI_I1 },
33464 /* TM vector builtins. */
33466 /* Reuse the existing x86-specific `struct builtin_description' cause
33467 we're lazy. Add casts to make them fit. */
33468 static const struct builtin_description bdesc_tm[] =
33470 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_WM64", (enum ix86_builtins) BUILT_IN_TM_STORE_M64, UNKNOWN, VOID_FTYPE_PV2SI_V2SI },
33471 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_WaRM64", (enum ix86_builtins) BUILT_IN_TM_STORE_WAR_M64, UNKNOWN, VOID_FTYPE_PV2SI_V2SI },
33472 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_WaWM64", (enum ix86_builtins) BUILT_IN_TM_STORE_WAW_M64, UNKNOWN, VOID_FTYPE_PV2SI_V2SI },
33473 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
33474 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RaRM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAR_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
33475 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RaWM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAW_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
33476 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RfWM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_RFW_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
33478 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_WM128", (enum ix86_builtins) BUILT_IN_TM_STORE_M128, UNKNOWN, VOID_FTYPE_PV4SF_V4SF },
33479 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_WaRM128", (enum ix86_builtins) BUILT_IN_TM_STORE_WAR_M128, UNKNOWN, VOID_FTYPE_PV4SF_V4SF },
33480 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_WaWM128", (enum ix86_builtins) BUILT_IN_TM_STORE_WAW_M128, UNKNOWN, VOID_FTYPE_PV4SF_V4SF },
33481 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
33482 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RaRM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAR_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
33483 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RaWM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAW_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
33484 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RfWM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_RFW_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
33486 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_WM256", (enum ix86_builtins) BUILT_IN_TM_STORE_M256, UNKNOWN, VOID_FTYPE_PV8SF_V8SF },
33487 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_WaRM256", (enum ix86_builtins) BUILT_IN_TM_STORE_WAR_M256, UNKNOWN, VOID_FTYPE_PV8SF_V8SF },
33488 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_WaWM256", (enum ix86_builtins) BUILT_IN_TM_STORE_WAW_M256, UNKNOWN, VOID_FTYPE_PV8SF_V8SF },
33489 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
33490 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RaRM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAR_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
33491 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RaWM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAW_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
33492 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RfWM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_RFW_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
33494 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_LM64", (enum ix86_builtins) BUILT_IN_TM_LOG_M64, UNKNOWN, VOID_FTYPE_PCVOID },
33495 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_LM128", (enum ix86_builtins) BUILT_IN_TM_LOG_M128, UNKNOWN, VOID_FTYPE_PCVOID },
33496 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_LM256", (enum ix86_builtins) BUILT_IN_TM_LOG_M256, UNKNOWN, VOID_FTYPE_PCVOID },
33499 /* TM callbacks. */
33501 /* Return the builtin decl needed to load a vector of TYPE. */
33504 ix86_builtin_tm_load (tree type)
33506 if (TREE_CODE (type) == VECTOR_TYPE)
33508 switch (tree_to_uhwi (TYPE_SIZE (type)))
33511 return builtin_decl_explicit (BUILT_IN_TM_LOAD_M64);
33513 return builtin_decl_explicit (BUILT_IN_TM_LOAD_M128);
33515 return builtin_decl_explicit (BUILT_IN_TM_LOAD_M256);
33521 /* Return the builtin decl needed to store a vector of TYPE. */
33524 ix86_builtin_tm_store (tree type)
33526 if (TREE_CODE (type) == VECTOR_TYPE)
33528 switch (tree_to_uhwi (TYPE_SIZE (type)))
33531 return builtin_decl_explicit (BUILT_IN_TM_STORE_M64);
33533 return builtin_decl_explicit (BUILT_IN_TM_STORE_M128);
33535 return builtin_decl_explicit (BUILT_IN_TM_STORE_M256);
33541 /* Initialize the transactional memory vector load/store builtins. */
33544 ix86_init_tm_builtins (void)
33546 enum ix86_builtin_func_type ftype;
33547 const struct builtin_description *d;
33550 tree attrs_load, attrs_type_load, attrs_store, attrs_type_store;
33551 tree attrs_log, attrs_type_log;
33556 /* If there are no builtins defined, we must be compiling in a
33557 language without trans-mem support. */
33558 if (!builtin_decl_explicit_p (BUILT_IN_TM_LOAD_1))
33561 /* Use whatever attributes a normal TM load has. */
33562 decl = builtin_decl_explicit (BUILT_IN_TM_LOAD_1);
33563 attrs_load = DECL_ATTRIBUTES (decl);
33564 attrs_type_load = TYPE_ATTRIBUTES (TREE_TYPE (decl));
33565 /* Use whatever attributes a normal TM store has. */
33566 decl = builtin_decl_explicit (BUILT_IN_TM_STORE_1);
33567 attrs_store = DECL_ATTRIBUTES (decl);
33568 attrs_type_store = TYPE_ATTRIBUTES (TREE_TYPE (decl));
33569 /* Use whatever attributes a normal TM log has. */
33570 decl = builtin_decl_explicit (BUILT_IN_TM_LOG);
33571 attrs_log = DECL_ATTRIBUTES (decl);
33572 attrs_type_log = TYPE_ATTRIBUTES (TREE_TYPE (decl));
33574 for (i = 0, d = bdesc_tm;
33575 i < ARRAY_SIZE (bdesc_tm);
33578 if ((d->mask & ix86_isa_flags) != 0
33579 || (lang_hooks.builtin_function
33580 == lang_hooks.builtin_function_ext_scope))
33582 tree type, attrs, attrs_type;
33583 enum built_in_function code = (enum built_in_function) d->code;
33585 ftype = (enum ix86_builtin_func_type) d->flag;
33586 type = ix86_get_builtin_func_type (ftype);
33588 if (BUILTIN_TM_LOAD_P (code))
33590 attrs = attrs_load;
33591 attrs_type = attrs_type_load;
33593 else if (BUILTIN_TM_STORE_P (code))
33595 attrs = attrs_store;
33596 attrs_type = attrs_type_store;
33601 attrs_type = attrs_type_log;
33603 decl = add_builtin_function (d->name, type, code, BUILT_IN_NORMAL,
33604 /* The builtin without the prefix for
33605 calling it directly. */
33606 d->name + strlen ("__builtin_"),
33608 /* add_builtin_function() will set the DECL_ATTRIBUTES, now
33609 set the TYPE_ATTRIBUTES. */
33610 decl_attributes (&TREE_TYPE (decl), attrs_type, ATTR_FLAG_BUILT_IN);
33612 set_builtin_decl (code, decl, false);
33617 /* Set up all the MMX/SSE builtins, even builtins for instructions that are not
33618 in the current target ISA to allow the user to compile particular modules
33619 with different target specific options that differ from the command line
33622 ix86_init_mmx_sse_builtins (void)
33624 const struct builtin_description * d;
33625 enum ix86_builtin_func_type ftype;
33628 /* Add all special builtins with variable number of operands. */
33629 for (i = 0, d = bdesc_special_args;
33630 i < ARRAY_SIZE (bdesc_special_args);
33636 ftype = (enum ix86_builtin_func_type) d->flag;
33637 def_builtin (d->mask, d->name, ftype, d->code);
33640 /* Add all builtins with variable number of operands. */
33641 for (i = 0, d = bdesc_args;
33642 i < ARRAY_SIZE (bdesc_args);
33648 ftype = (enum ix86_builtin_func_type) d->flag;
33649 def_builtin_const (d->mask, d->name, ftype, d->code);
33652 /* Add all builtins with rounding. */
33653 for (i = 0, d = bdesc_round_args;
33654 i < ARRAY_SIZE (bdesc_round_args);
33660 ftype = (enum ix86_builtin_func_type) d->flag;
33661 def_builtin_const (d->mask, d->name, ftype, d->code);
33664 /* pcmpestr[im] insns. */
33665 for (i = 0, d = bdesc_pcmpestr;
33666 i < ARRAY_SIZE (bdesc_pcmpestr);
33669 if (d->code == IX86_BUILTIN_PCMPESTRM128)
33670 ftype = V16QI_FTYPE_V16QI_INT_V16QI_INT_INT;
33672 ftype = INT_FTYPE_V16QI_INT_V16QI_INT_INT;
33673 def_builtin_const (d->mask, d->name, ftype, d->code);
33676 /* pcmpistr[im] insns. */
33677 for (i = 0, d = bdesc_pcmpistr;
33678 i < ARRAY_SIZE (bdesc_pcmpistr);
33681 if (d->code == IX86_BUILTIN_PCMPISTRM128)
33682 ftype = V16QI_FTYPE_V16QI_V16QI_INT;
33684 ftype = INT_FTYPE_V16QI_V16QI_INT;
33685 def_builtin_const (d->mask, d->name, ftype, d->code);
33688 /* comi/ucomi insns. */
33689 for (i = 0, d = bdesc_comi; i < ARRAY_SIZE (bdesc_comi); i++, d++)
33691 if (d->mask == OPTION_MASK_ISA_SSE2)
33692 ftype = INT_FTYPE_V2DF_V2DF;
33694 ftype = INT_FTYPE_V4SF_V4SF;
33695 def_builtin_const (d->mask, d->name, ftype, d->code);
33699 def_builtin (OPTION_MASK_ISA_SSE, "__builtin_ia32_ldmxcsr",
33700 VOID_FTYPE_UNSIGNED, IX86_BUILTIN_LDMXCSR);
33701 def_builtin (OPTION_MASK_ISA_SSE, "__builtin_ia32_stmxcsr",
33702 UNSIGNED_FTYPE_VOID, IX86_BUILTIN_STMXCSR);
33704 /* SSE or 3DNow!A */
33705 def_builtin (OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A,
33706 "__builtin_ia32_maskmovq", VOID_FTYPE_V8QI_V8QI_PCHAR,
33707 IX86_BUILTIN_MASKMOVQ);
33710 def_builtin (OPTION_MASK_ISA_SSE2, "__builtin_ia32_maskmovdqu",
33711 VOID_FTYPE_V16QI_V16QI_PCHAR, IX86_BUILTIN_MASKMOVDQU);
33713 def_builtin (OPTION_MASK_ISA_SSE2, "__builtin_ia32_clflush",
33714 VOID_FTYPE_PCVOID, IX86_BUILTIN_CLFLUSH);
33715 x86_mfence = def_builtin (OPTION_MASK_ISA_SSE2, "__builtin_ia32_mfence",
33716 VOID_FTYPE_VOID, IX86_BUILTIN_MFENCE);
33719 def_builtin (OPTION_MASK_ISA_SSE3, "__builtin_ia32_monitor",
33720 VOID_FTYPE_PCVOID_UNSIGNED_UNSIGNED, IX86_BUILTIN_MONITOR);
33721 def_builtin (OPTION_MASK_ISA_SSE3, "__builtin_ia32_mwait",
33722 VOID_FTYPE_UNSIGNED_UNSIGNED, IX86_BUILTIN_MWAIT);
33725 def_builtin_const (OPTION_MASK_ISA_AES, "__builtin_ia32_aesenc128",
33726 V2DI_FTYPE_V2DI_V2DI, IX86_BUILTIN_AESENC128);
33727 def_builtin_const (OPTION_MASK_ISA_AES, "__builtin_ia32_aesenclast128",
33728 V2DI_FTYPE_V2DI_V2DI, IX86_BUILTIN_AESENCLAST128);
33729 def_builtin_const (OPTION_MASK_ISA_AES, "__builtin_ia32_aesdec128",
33730 V2DI_FTYPE_V2DI_V2DI, IX86_BUILTIN_AESDEC128);
33731 def_builtin_const (OPTION_MASK_ISA_AES, "__builtin_ia32_aesdeclast128",
33732 V2DI_FTYPE_V2DI_V2DI, IX86_BUILTIN_AESDECLAST128);
33733 def_builtin_const (OPTION_MASK_ISA_AES, "__builtin_ia32_aesimc128",
33734 V2DI_FTYPE_V2DI, IX86_BUILTIN_AESIMC128);
33735 def_builtin_const (OPTION_MASK_ISA_AES, "__builtin_ia32_aeskeygenassist128",
33736 V2DI_FTYPE_V2DI_INT, IX86_BUILTIN_AESKEYGENASSIST128);
33739 def_builtin_const (OPTION_MASK_ISA_PCLMUL, "__builtin_ia32_pclmulqdq128",
33740 V2DI_FTYPE_V2DI_V2DI_INT, IX86_BUILTIN_PCLMULQDQ128);
33743 def_builtin (OPTION_MASK_ISA_RDRND, "__builtin_ia32_rdrand16_step",
33744 INT_FTYPE_PUSHORT, IX86_BUILTIN_RDRAND16_STEP);
33745 def_builtin (OPTION_MASK_ISA_RDRND, "__builtin_ia32_rdrand32_step",
33746 INT_FTYPE_PUNSIGNED, IX86_BUILTIN_RDRAND32_STEP);
33747 def_builtin (OPTION_MASK_ISA_RDRND | OPTION_MASK_ISA_64BIT,
33748 "__builtin_ia32_rdrand64_step", INT_FTYPE_PULONGLONG,
33749 IX86_BUILTIN_RDRAND64_STEP);
33752 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gathersiv2df",
33753 V2DF_FTYPE_V2DF_PCDOUBLE_V4SI_V2DF_INT,
33754 IX86_BUILTIN_GATHERSIV2DF);
33756 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gathersiv4df",
33757 V4DF_FTYPE_V4DF_PCDOUBLE_V4SI_V4DF_INT,
33758 IX86_BUILTIN_GATHERSIV4DF);
33760 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gatherdiv2df",
33761 V2DF_FTYPE_V2DF_PCDOUBLE_V2DI_V2DF_INT,
33762 IX86_BUILTIN_GATHERDIV2DF);
33764 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gatherdiv4df",
33765 V4DF_FTYPE_V4DF_PCDOUBLE_V4DI_V4DF_INT,
33766 IX86_BUILTIN_GATHERDIV4DF);
33768 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gathersiv4sf",
33769 V4SF_FTYPE_V4SF_PCFLOAT_V4SI_V4SF_INT,
33770 IX86_BUILTIN_GATHERSIV4SF);
33772 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gathersiv8sf",
33773 V8SF_FTYPE_V8SF_PCFLOAT_V8SI_V8SF_INT,
33774 IX86_BUILTIN_GATHERSIV8SF);
33776 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gatherdiv4sf",
33777 V4SF_FTYPE_V4SF_PCFLOAT_V2DI_V4SF_INT,
33778 IX86_BUILTIN_GATHERDIV4SF);
33780 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gatherdiv4sf256",
33781 V4SF_FTYPE_V4SF_PCFLOAT_V4DI_V4SF_INT,
33782 IX86_BUILTIN_GATHERDIV8SF);
33784 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gathersiv2di",
33785 V2DI_FTYPE_V2DI_PCINT64_V4SI_V2DI_INT,
33786 IX86_BUILTIN_GATHERSIV2DI);
33788 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gathersiv4di",
33789 V4DI_FTYPE_V4DI_PCINT64_V4SI_V4DI_INT,
33790 IX86_BUILTIN_GATHERSIV4DI);
33792 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gatherdiv2di",
33793 V2DI_FTYPE_V2DI_PCINT64_V2DI_V2DI_INT,
33794 IX86_BUILTIN_GATHERDIV2DI);
33796 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gatherdiv4di",
33797 V4DI_FTYPE_V4DI_PCINT64_V4DI_V4DI_INT,
33798 IX86_BUILTIN_GATHERDIV4DI);
33800 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gathersiv4si",
33801 V4SI_FTYPE_V4SI_PCINT_V4SI_V4SI_INT,
33802 IX86_BUILTIN_GATHERSIV4SI);
33804 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gathersiv8si",
33805 V8SI_FTYPE_V8SI_PCINT_V8SI_V8SI_INT,
33806 IX86_BUILTIN_GATHERSIV8SI);
33808 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gatherdiv4si",
33809 V4SI_FTYPE_V4SI_PCINT_V2DI_V4SI_INT,
33810 IX86_BUILTIN_GATHERDIV4SI);
33812 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gatherdiv4si256",
33813 V4SI_FTYPE_V4SI_PCINT_V4DI_V4SI_INT,
33814 IX86_BUILTIN_GATHERDIV8SI);
33816 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gatheraltsiv4df ",
33817 V4DF_FTYPE_V4DF_PCDOUBLE_V8SI_V4DF_INT,
33818 IX86_BUILTIN_GATHERALTSIV4DF);
33820 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gatheraltdiv4sf256 ",
33821 V8SF_FTYPE_V8SF_PCFLOAT_V4DI_V8SF_INT,
33822 IX86_BUILTIN_GATHERALTDIV8SF);
33824 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gatheraltsiv4di ",
33825 V4DI_FTYPE_V4DI_PCINT64_V8SI_V4DI_INT,
33826 IX86_BUILTIN_GATHERALTSIV4DI);
33828 def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gatheraltdiv4si256 ",
33829 V8SI_FTYPE_V8SI_PCINT_V4DI_V8SI_INT,
33830 IX86_BUILTIN_GATHERALTDIV8SI);
33833 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_gathersiv16sf",
33834 V16SF_FTYPE_V16SF_PCFLOAT_V16SI_HI_INT,
33835 IX86_BUILTIN_GATHER3SIV16SF);
33837 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_gathersiv8df",
33838 V8DF_FTYPE_V8DF_PCDOUBLE_V8SI_QI_INT,
33839 IX86_BUILTIN_GATHER3SIV8DF);
33841 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_gatherdiv16sf",
33842 V8SF_FTYPE_V8SF_PCFLOAT_V8DI_QI_INT,
33843 IX86_BUILTIN_GATHER3DIV16SF);
33845 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_gatherdiv8df",
33846 V8DF_FTYPE_V8DF_PCDOUBLE_V8DI_QI_INT,
33847 IX86_BUILTIN_GATHER3DIV8DF);
33849 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_gathersiv16si",
33850 V16SI_FTYPE_V16SI_PCINT_V16SI_HI_INT,
33851 IX86_BUILTIN_GATHER3SIV16SI);
33853 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_gathersiv8di",
33854 V8DI_FTYPE_V8DI_PCINT64_V8SI_QI_INT,
33855 IX86_BUILTIN_GATHER3SIV8DI);
33857 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_gatherdiv16si",
33858 V8SI_FTYPE_V8SI_PCINT_V8DI_QI_INT,
33859 IX86_BUILTIN_GATHER3DIV16SI);
33861 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_gatherdiv8di",
33862 V8DI_FTYPE_V8DI_PCINT64_V8DI_QI_INT,
33863 IX86_BUILTIN_GATHER3DIV8DI);
33865 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_gatheraltsiv8df ",
33866 V8DF_FTYPE_V8DF_PCDOUBLE_V16SI_QI_INT,
33867 IX86_BUILTIN_GATHER3ALTSIV8DF);
33869 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_gatheraltdiv8sf ",
33870 V16SF_FTYPE_V16SF_PCFLOAT_V8DI_HI_INT,
33871 IX86_BUILTIN_GATHER3ALTDIV16SF);
33873 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_gatheraltsiv8di ",
33874 V8DI_FTYPE_V8DI_PCINT64_V16SI_QI_INT,
33875 IX86_BUILTIN_GATHER3ALTSIV8DI);
33877 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_gatheraltdiv8si ",
33878 V16SI_FTYPE_V16SI_PCINT_V8DI_HI_INT,
33879 IX86_BUILTIN_GATHER3ALTDIV16SI);
33881 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_scattersiv16sf",
33882 VOID_FTYPE_PFLOAT_HI_V16SI_V16SF_INT,
33883 IX86_BUILTIN_SCATTERSIV16SF);
33885 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_scattersiv8df",
33886 VOID_FTYPE_PDOUBLE_QI_V8SI_V8DF_INT,
33887 IX86_BUILTIN_SCATTERSIV8DF);
33889 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_scatterdiv16sf",
33890 VOID_FTYPE_PFLOAT_QI_V8DI_V8SF_INT,
33891 IX86_BUILTIN_SCATTERDIV16SF);
33893 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_scatterdiv8df",
33894 VOID_FTYPE_PDOUBLE_QI_V8DI_V8DF_INT,
33895 IX86_BUILTIN_SCATTERDIV8DF);
33897 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_scattersiv16si",
33898 VOID_FTYPE_PINT_HI_V16SI_V16SI_INT,
33899 IX86_BUILTIN_SCATTERSIV16SI);
33901 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_scattersiv8di",
33902 VOID_FTYPE_PLONGLONG_QI_V8SI_V8DI_INT,
33903 IX86_BUILTIN_SCATTERSIV8DI);
33905 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_scatterdiv16si",
33906 VOID_FTYPE_PINT_QI_V8DI_V8SI_INT,
33907 IX86_BUILTIN_SCATTERDIV16SI);
33909 def_builtin (OPTION_MASK_ISA_AVX512F, "__builtin_ia32_scatterdiv8di",
33910 VOID_FTYPE_PLONGLONG_QI_V8DI_V8DI_INT,
33911 IX86_BUILTIN_SCATTERDIV8DI);
33914 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3siv2df",
33915 V2DF_FTYPE_V2DF_PCDOUBLE_V4SI_QI_INT,
33916 IX86_BUILTIN_GATHER3SIV2DF);
33918 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3siv4df",
33919 V4DF_FTYPE_V4DF_PCDOUBLE_V4SI_QI_INT,
33920 IX86_BUILTIN_GATHER3SIV4DF);
33922 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3div2df",
33923 V2DF_FTYPE_V2DF_PCDOUBLE_V2DI_QI_INT,
33924 IX86_BUILTIN_GATHER3DIV2DF);
33926 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3div4df",
33927 V4DF_FTYPE_V4DF_PCDOUBLE_V4DI_QI_INT,
33928 IX86_BUILTIN_GATHER3DIV4DF);
33930 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3siv4sf",
33931 V4SF_FTYPE_V4SF_PCFLOAT_V4SI_QI_INT,
33932 IX86_BUILTIN_GATHER3SIV4SF);
33934 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3siv8sf",
33935 V8SF_FTYPE_V8SF_PCFLOAT_V8SI_QI_INT,
33936 IX86_BUILTIN_GATHER3SIV8SF);
33938 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3div4sf",
33939 V4SF_FTYPE_V4SF_PCFLOAT_V2DI_QI_INT,
33940 IX86_BUILTIN_GATHER3DIV4SF);
33942 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3div8sf",
33943 V4SF_FTYPE_V4SF_PCFLOAT_V4DI_QI_INT,
33944 IX86_BUILTIN_GATHER3DIV8SF);
33946 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3siv2di",
33947 V2DI_FTYPE_V2DI_PCINT64_V4SI_QI_INT,
33948 IX86_BUILTIN_GATHER3SIV2DI);
33950 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3siv4di",
33951 V4DI_FTYPE_V4DI_PCINT64_V4SI_QI_INT,
33952 IX86_BUILTIN_GATHER3SIV4DI);
33954 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3div2di",
33955 V2DI_FTYPE_V2DI_PCINT64_V2DI_QI_INT,
33956 IX86_BUILTIN_GATHER3DIV2DI);
33958 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3div4di",
33959 V4DI_FTYPE_V4DI_PCINT64_V4DI_QI_INT,
33960 IX86_BUILTIN_GATHER3DIV4DI);
33962 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3siv4si",
33963 V4SI_FTYPE_V4SI_PCINT_V4SI_QI_INT,
33964 IX86_BUILTIN_GATHER3SIV4SI);
33966 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3siv8si",
33967 V8SI_FTYPE_V8SI_PCINT_V8SI_QI_INT,
33968 IX86_BUILTIN_GATHER3SIV8SI);
33970 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3div4si",
33971 V4SI_FTYPE_V4SI_PCINT_V2DI_QI_INT,
33972 IX86_BUILTIN_GATHER3DIV4SI);
33974 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3div8si",
33975 V4SI_FTYPE_V4SI_PCINT_V4DI_QI_INT,
33976 IX86_BUILTIN_GATHER3DIV8SI);
33978 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3altsiv4df ",
33979 V4DF_FTYPE_V4DF_PCDOUBLE_V8SI_QI_INT,
33980 IX86_BUILTIN_GATHER3ALTSIV4DF);
33982 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3altdiv8sf ",
33983 V8SF_FTYPE_V8SF_PCFLOAT_V4DI_QI_INT,
33984 IX86_BUILTIN_GATHER3ALTDIV8SF);
33986 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3altsiv4di ",
33987 V4DI_FTYPE_V4DI_PCINT64_V8SI_QI_INT,
33988 IX86_BUILTIN_GATHER3ALTSIV4DI);
33990 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_gather3altdiv8si ",
33991 V8SI_FTYPE_V8SI_PCINT_V4DI_QI_INT,
33992 IX86_BUILTIN_GATHER3ALTDIV8SI);
33994 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_scattersiv8sf",
33995 VOID_FTYPE_PFLOAT_QI_V8SI_V8SF_INT,
33996 IX86_BUILTIN_SCATTERSIV8SF);
33998 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_scattersiv4sf",
33999 VOID_FTYPE_PFLOAT_QI_V4SI_V4SF_INT,
34000 IX86_BUILTIN_SCATTERSIV4SF);
34002 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_scattersiv4df",
34003 VOID_FTYPE_PDOUBLE_QI_V4SI_V4DF_INT,
34004 IX86_BUILTIN_SCATTERSIV4DF);
34006 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_scattersiv2df",
34007 VOID_FTYPE_PDOUBLE_QI_V4SI_V2DF_INT,
34008 IX86_BUILTIN_SCATTERSIV2DF);
34010 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_scatterdiv8sf",
34011 VOID_FTYPE_PFLOAT_QI_V4DI_V4SF_INT,
34012 IX86_BUILTIN_SCATTERDIV8SF);
34014 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_scatterdiv4sf",
34015 VOID_FTYPE_PFLOAT_QI_V2DI_V4SF_INT,
34016 IX86_BUILTIN_SCATTERDIV4SF);
34018 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_scatterdiv4df",
34019 VOID_FTYPE_PDOUBLE_QI_V4DI_V4DF_INT,
34020 IX86_BUILTIN_SCATTERDIV4DF);
34022 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_scatterdiv2df",
34023 VOID_FTYPE_PDOUBLE_QI_V2DI_V2DF_INT,
34024 IX86_BUILTIN_SCATTERDIV2DF);
34026 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_scattersiv8si",
34027 VOID_FTYPE_PINT_QI_V8SI_V8SI_INT,
34028 IX86_BUILTIN_SCATTERSIV8SI);
34030 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_scattersiv4si",
34031 VOID_FTYPE_PINT_QI_V4SI_V4SI_INT,
34032 IX86_BUILTIN_SCATTERSIV4SI);
34034 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_scattersiv4di",
34035 VOID_FTYPE_PLONGLONG_QI_V4SI_V4DI_INT,
34036 IX86_BUILTIN_SCATTERSIV4DI);
34038 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_scattersiv2di",
34039 VOID_FTYPE_PLONGLONG_QI_V4SI_V2DI_INT,
34040 IX86_BUILTIN_SCATTERSIV2DI);
34042 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_scatterdiv8si",
34043 VOID_FTYPE_PINT_QI_V4DI_V4SI_INT,
34044 IX86_BUILTIN_SCATTERDIV8SI);
34046 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_scatterdiv4si",
34047 VOID_FTYPE_PINT_QI_V2DI_V4SI_INT,
34048 IX86_BUILTIN_SCATTERDIV4SI);
34050 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_scatterdiv4di",
34051 VOID_FTYPE_PLONGLONG_QI_V4DI_V4DI_INT,
34052 IX86_BUILTIN_SCATTERDIV4DI);
34054 def_builtin (OPTION_MASK_ISA_AVX512VL, "__builtin_ia32_scatterdiv2di",
34055 VOID_FTYPE_PLONGLONG_QI_V2DI_V2DI_INT,
34056 IX86_BUILTIN_SCATTERDIV2DI);
34059 def_builtin (OPTION_MASK_ISA_AVX512PF, "__builtin_ia32_gatherpfdpd",
34060 VOID_FTYPE_QI_V8SI_PCINT64_INT_INT,
34061 IX86_BUILTIN_GATHERPFDPD);
34062 def_builtin (OPTION_MASK_ISA_AVX512PF, "__builtin_ia32_gatherpfdps",
34063 VOID_FTYPE_HI_V16SI_PCINT_INT_INT,
34064 IX86_BUILTIN_GATHERPFDPS);
34065 def_builtin (OPTION_MASK_ISA_AVX512PF, "__builtin_ia32_gatherpfqpd",
34066 VOID_FTYPE_QI_V8DI_PCINT64_INT_INT,
34067 IX86_BUILTIN_GATHERPFQPD);
34068 def_builtin (OPTION_MASK_ISA_AVX512PF, "__builtin_ia32_gatherpfqps",
34069 VOID_FTYPE_QI_V8DI_PCINT_INT_INT,
34070 IX86_BUILTIN_GATHERPFQPS);
34071 def_builtin (OPTION_MASK_ISA_AVX512PF, "__builtin_ia32_scatterpfdpd",
34072 VOID_FTYPE_QI_V8SI_PCINT64_INT_INT,
34073 IX86_BUILTIN_SCATTERPFDPD);
34074 def_builtin (OPTION_MASK_ISA_AVX512PF, "__builtin_ia32_scatterpfdps",
34075 VOID_FTYPE_HI_V16SI_PCINT_INT_INT,
34076 IX86_BUILTIN_SCATTERPFDPS);
34077 def_builtin (OPTION_MASK_ISA_AVX512PF, "__builtin_ia32_scatterpfqpd",
34078 VOID_FTYPE_QI_V8DI_PCINT64_INT_INT,
34079 IX86_BUILTIN_SCATTERPFQPD);
34080 def_builtin (OPTION_MASK_ISA_AVX512PF, "__builtin_ia32_scatterpfqps",
34081 VOID_FTYPE_QI_V8DI_PCINT_INT_INT,
34082 IX86_BUILTIN_SCATTERPFQPS);
34085 def_builtin_const (OPTION_MASK_ISA_SHA, "__builtin_ia32_sha1msg1",
34086 V4SI_FTYPE_V4SI_V4SI, IX86_BUILTIN_SHA1MSG1);
34087 def_builtin_const (OPTION_MASK_ISA_SHA, "__builtin_ia32_sha1msg2",
34088 V4SI_FTYPE_V4SI_V4SI, IX86_BUILTIN_SHA1MSG2);
34089 def_builtin_const (OPTION_MASK_ISA_SHA, "__builtin_ia32_sha1nexte",
34090 V4SI_FTYPE_V4SI_V4SI, IX86_BUILTIN_SHA1NEXTE);
34091 def_builtin_const (OPTION_MASK_ISA_SHA, "__builtin_ia32_sha1rnds4",
34092 V4SI_FTYPE_V4SI_V4SI_INT, IX86_BUILTIN_SHA1RNDS4);
34093 def_builtin_const (OPTION_MASK_ISA_SHA, "__builtin_ia32_sha256msg1",
34094 V4SI_FTYPE_V4SI_V4SI, IX86_BUILTIN_SHA256MSG1);
34095 def_builtin_const (OPTION_MASK_ISA_SHA, "__builtin_ia32_sha256msg2",
34096 V4SI_FTYPE_V4SI_V4SI, IX86_BUILTIN_SHA256MSG2);
34097 def_builtin_const (OPTION_MASK_ISA_SHA, "__builtin_ia32_sha256rnds2",
34098 V4SI_FTYPE_V4SI_V4SI_V4SI, IX86_BUILTIN_SHA256RNDS2);
34101 def_builtin (OPTION_MASK_ISA_RTM, "__builtin_ia32_xabort",
34102 VOID_FTYPE_UNSIGNED, IX86_BUILTIN_XABORT);
34104 /* MMX access to the vec_init patterns. */
34105 def_builtin_const (OPTION_MASK_ISA_MMX, "__builtin_ia32_vec_init_v2si",
34106 V2SI_FTYPE_INT_INT, IX86_BUILTIN_VEC_INIT_V2SI);
34108 def_builtin_const (OPTION_MASK_ISA_MMX, "__builtin_ia32_vec_init_v4hi",
34109 V4HI_FTYPE_HI_HI_HI_HI,
34110 IX86_BUILTIN_VEC_INIT_V4HI);
34112 def_builtin_const (OPTION_MASK_ISA_MMX, "__builtin_ia32_vec_init_v8qi",
34113 V8QI_FTYPE_QI_QI_QI_QI_QI_QI_QI_QI,
34114 IX86_BUILTIN_VEC_INIT_V8QI);
34116 /* Access to the vec_extract patterns. */
34117 def_builtin_const (OPTION_MASK_ISA_SSE2, "__builtin_ia32_vec_ext_v2df",
34118 DOUBLE_FTYPE_V2DF_INT, IX86_BUILTIN_VEC_EXT_V2DF);
34119 def_builtin_const (OPTION_MASK_ISA_SSE2, "__builtin_ia32_vec_ext_v2di",
34120 DI_FTYPE_V2DI_INT, IX86_BUILTIN_VEC_EXT_V2DI);
34121 def_builtin_const (OPTION_MASK_ISA_SSE, "__builtin_ia32_vec_ext_v4sf",
34122 FLOAT_FTYPE_V4SF_INT, IX86_BUILTIN_VEC_EXT_V4SF);
34123 def_builtin_const (OPTION_MASK_ISA_SSE2, "__builtin_ia32_vec_ext_v4si",
34124 SI_FTYPE_V4SI_INT, IX86_BUILTIN_VEC_EXT_V4SI);
34125 def_builtin_const (OPTION_MASK_ISA_SSE2, "__builtin_ia32_vec_ext_v8hi",
34126 HI_FTYPE_V8HI_INT, IX86_BUILTIN_VEC_EXT_V8HI);
34128 def_builtin_const (OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A,
34129 "__builtin_ia32_vec_ext_v4hi",
34130 HI_FTYPE_V4HI_INT, IX86_BUILTIN_VEC_EXT_V4HI);
34132 def_builtin_const (OPTION_MASK_ISA_MMX, "__builtin_ia32_vec_ext_v2si",
34133 SI_FTYPE_V2SI_INT, IX86_BUILTIN_VEC_EXT_V2SI);
34135 def_builtin_const (OPTION_MASK_ISA_SSE2, "__builtin_ia32_vec_ext_v16qi",
34136 QI_FTYPE_V16QI_INT, IX86_BUILTIN_VEC_EXT_V16QI);
34138 /* Access to the vec_set patterns. */
34139 def_builtin_const (OPTION_MASK_ISA_SSE4_1 | OPTION_MASK_ISA_64BIT,
34140 "__builtin_ia32_vec_set_v2di",
34141 V2DI_FTYPE_V2DI_DI_INT, IX86_BUILTIN_VEC_SET_V2DI);
34143 def_builtin_const (OPTION_MASK_ISA_SSE4_1, "__builtin_ia32_vec_set_v4sf",
34144 V4SF_FTYPE_V4SF_FLOAT_INT, IX86_BUILTIN_VEC_SET_V4SF);
34146 def_builtin_const (OPTION_MASK_ISA_SSE4_1, "__builtin_ia32_vec_set_v4si",
34147 V4SI_FTYPE_V4SI_SI_INT, IX86_BUILTIN_VEC_SET_V4SI);
34149 def_builtin_const (OPTION_MASK_ISA_SSE2, "__builtin_ia32_vec_set_v8hi",
34150 V8HI_FTYPE_V8HI_HI_INT, IX86_BUILTIN_VEC_SET_V8HI);
34152 def_builtin_const (OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A,
34153 "__builtin_ia32_vec_set_v4hi",
34154 V4HI_FTYPE_V4HI_HI_INT, IX86_BUILTIN_VEC_SET_V4HI);
34156 def_builtin_const (OPTION_MASK_ISA_SSE4_1, "__builtin_ia32_vec_set_v16qi",
34157 V16QI_FTYPE_V16QI_QI_INT, IX86_BUILTIN_VEC_SET_V16QI);
34160 def_builtin (OPTION_MASK_ISA_RDSEED, "__builtin_ia32_rdseed_hi_step",
34161 INT_FTYPE_PUSHORT, IX86_BUILTIN_RDSEED16_STEP);
34162 def_builtin (OPTION_MASK_ISA_RDSEED, "__builtin_ia32_rdseed_si_step",
34163 INT_FTYPE_PUNSIGNED, IX86_BUILTIN_RDSEED32_STEP);
34164 def_builtin (OPTION_MASK_ISA_RDSEED | OPTION_MASK_ISA_64BIT,
34165 "__builtin_ia32_rdseed_di_step",
34166 INT_FTYPE_PULONGLONG, IX86_BUILTIN_RDSEED64_STEP);
34169 def_builtin (0, "__builtin_ia32_addcarryx_u32",
34170 UCHAR_FTYPE_UCHAR_UINT_UINT_PUNSIGNED, IX86_BUILTIN_ADDCARRYX32);
34171 def_builtin (OPTION_MASK_ISA_64BIT,
34172 "__builtin_ia32_addcarryx_u64",
34173 UCHAR_FTYPE_UCHAR_ULONGLONG_ULONGLONG_PULONGLONG,
34174 IX86_BUILTIN_ADDCARRYX64);
34177 def_builtin (0, "__builtin_ia32_sbb_u32",
34178 UCHAR_FTYPE_UCHAR_UINT_UINT_PUNSIGNED, IX86_BUILTIN_SBB32);
34179 def_builtin (OPTION_MASK_ISA_64BIT,
34180 "__builtin_ia32_sbb_u64",
34181 UCHAR_FTYPE_UCHAR_ULONGLONG_ULONGLONG_PULONGLONG,
34182 IX86_BUILTIN_SBB64);
34184 /* Read/write FLAGS. */
34185 def_builtin (~OPTION_MASK_ISA_64BIT, "__builtin_ia32_readeflags_u32",
34186 UNSIGNED_FTYPE_VOID, IX86_BUILTIN_READ_FLAGS);
34187 def_builtin (OPTION_MASK_ISA_64BIT, "__builtin_ia32_readeflags_u64",
34188 UINT64_FTYPE_VOID, IX86_BUILTIN_READ_FLAGS);
34189 def_builtin (~OPTION_MASK_ISA_64BIT, "__builtin_ia32_writeeflags_u32",
34190 VOID_FTYPE_UNSIGNED, IX86_BUILTIN_WRITE_FLAGS);
34191 def_builtin (OPTION_MASK_ISA_64BIT, "__builtin_ia32_writeeflags_u64",
34192 VOID_FTYPE_UINT64, IX86_BUILTIN_WRITE_FLAGS);
34195 def_builtin (OPTION_MASK_ISA_CLFLUSHOPT, "__builtin_ia32_clflushopt",
34196 VOID_FTYPE_PCVOID, IX86_BUILTIN_CLFLUSHOPT);
34199 def_builtin (OPTION_MASK_ISA_CLWB, "__builtin_ia32_clwb",
34200 VOID_FTYPE_PCVOID, IX86_BUILTIN_CLWB);
34202 /* MONITORX and MWAITX. */
34203 def_builtin (OPTION_MASK_ISA_MWAITX, "__builtin_ia32_monitorx",
34204 VOID_FTYPE_PCVOID_UNSIGNED_UNSIGNED, IX86_BUILTIN_MONITORX);
34205 def_builtin (OPTION_MASK_ISA_MWAITX, "__builtin_ia32_mwaitx",
34206 VOID_FTYPE_UNSIGNED_UNSIGNED_UNSIGNED, IX86_BUILTIN_MWAITX);
34208 /* Add FMA4 multi-arg argument instructions */
34209 for (i = 0, d = bdesc_multi_arg; i < ARRAY_SIZE (bdesc_multi_arg); i++, d++)
34214 ftype = (enum ix86_builtin_func_type) d->flag;
34215 def_builtin_const (d->mask, d->name, ftype, d->code);
34220 ix86_init_mpx_builtins ()
34222 const struct builtin_description * d;
34223 enum ix86_builtin_func_type ftype;
34227 for (i = 0, d = bdesc_mpx;
34228 i < ARRAY_SIZE (bdesc_mpx);
34234 ftype = (enum ix86_builtin_func_type) d->flag;
34235 decl = def_builtin (d->mask, d->name, ftype, d->code);
34237 /* With no leaf and nothrow flags for MPX builtins
34238 abnormal edges may follow its call when setjmp
34239 presents in the function. Since we may have a lot
34240 of MPX builtins calls it causes lots of useless
34241 edges and enormous PHI nodes. To avoid this we mark
34242 MPX builtins as leaf and nothrow. */
34245 DECL_ATTRIBUTES (decl) = build_tree_list (get_identifier ("leaf"),
34247 TREE_NOTHROW (decl) = 1;
34251 ix86_builtins_isa[(int)d->code].leaf_p = true;
34252 ix86_builtins_isa[(int)d->code].nothrow_p = true;
34256 for (i = 0, d = bdesc_mpx_const;
34257 i < ARRAY_SIZE (bdesc_mpx_const);
34263 ftype = (enum ix86_builtin_func_type) d->flag;
34264 decl = def_builtin_const (d->mask, d->name, ftype, d->code);
34268 DECL_ATTRIBUTES (decl) = build_tree_list (get_identifier ("leaf"),
34270 TREE_NOTHROW (decl) = 1;
34274 ix86_builtins_isa[(int)d->code].leaf_p = true;
34275 ix86_builtins_isa[(int)d->code].nothrow_p = true;
34280 /* This adds a condition to the basic_block NEW_BB in function FUNCTION_DECL
34281 to return a pointer to VERSION_DECL if the outcome of the expression
34282 formed by PREDICATE_CHAIN is true. This function will be called during
34283 version dispatch to decide which function version to execute. It returns
34284 the basic block at the end, to which more conditions can be added. */
34287 add_condition_to_bb (tree function_decl, tree version_decl,
34288 tree predicate_chain, basic_block new_bb)
34290 gimple return_stmt;
34291 tree convert_expr, result_var;
34292 gimple convert_stmt;
34293 gimple call_cond_stmt;
34294 gimple if_else_stmt;
34296 basic_block bb1, bb2, bb3;
34299 tree cond_var, and_expr_var = NULL_TREE;
34302 tree predicate_decl, predicate_arg;
34304 push_cfun (DECL_STRUCT_FUNCTION (function_decl));
34306 gcc_assert (new_bb != NULL);
34307 gseq = bb_seq (new_bb);
34310 convert_expr = build1 (CONVERT_EXPR, ptr_type_node,
34311 build_fold_addr_expr (version_decl));
34312 result_var = create_tmp_var (ptr_type_node);
34313 convert_stmt = gimple_build_assign (result_var, convert_expr);
34314 return_stmt = gimple_build_return (result_var);
34316 if (predicate_chain == NULL_TREE)
34318 gimple_seq_add_stmt (&gseq, convert_stmt);
34319 gimple_seq_add_stmt (&gseq, return_stmt);
34320 set_bb_seq (new_bb, gseq);
34321 gimple_set_bb (convert_stmt, new_bb);
34322 gimple_set_bb (return_stmt, new_bb);
34327 while (predicate_chain != NULL)
34329 cond_var = create_tmp_var (integer_type_node);
34330 predicate_decl = TREE_PURPOSE (predicate_chain);
34331 predicate_arg = TREE_VALUE (predicate_chain);
34332 call_cond_stmt = gimple_build_call (predicate_decl, 1, predicate_arg);
34333 gimple_call_set_lhs (call_cond_stmt, cond_var);
34335 gimple_set_block (call_cond_stmt, DECL_INITIAL (function_decl));
34336 gimple_set_bb (call_cond_stmt, new_bb);
34337 gimple_seq_add_stmt (&gseq, call_cond_stmt);
34339 predicate_chain = TREE_CHAIN (predicate_chain);
34341 if (and_expr_var == NULL)
34342 and_expr_var = cond_var;
34345 gimple assign_stmt;
34346 /* Use MIN_EXPR to check if any integer is zero?.
34347 and_expr_var = min_expr <cond_var, and_expr_var> */
34348 assign_stmt = gimple_build_assign (and_expr_var,
34349 build2 (MIN_EXPR, integer_type_node,
34350 cond_var, and_expr_var));
34352 gimple_set_block (assign_stmt, DECL_INITIAL (function_decl));
34353 gimple_set_bb (assign_stmt, new_bb);
34354 gimple_seq_add_stmt (&gseq, assign_stmt);
34358 if_else_stmt = gimple_build_cond (GT_EXPR, and_expr_var,
34360 NULL_TREE, NULL_TREE);
34361 gimple_set_block (if_else_stmt, DECL_INITIAL (function_decl));
34362 gimple_set_bb (if_else_stmt, new_bb);
34363 gimple_seq_add_stmt (&gseq, if_else_stmt);
34365 gimple_seq_add_stmt (&gseq, convert_stmt);
34366 gimple_seq_add_stmt (&gseq, return_stmt);
34367 set_bb_seq (new_bb, gseq);
34370 e12 = split_block (bb1, if_else_stmt);
34372 e12->flags &= ~EDGE_FALLTHRU;
34373 e12->flags |= EDGE_TRUE_VALUE;
34375 e23 = split_block (bb2, return_stmt);
34377 gimple_set_bb (convert_stmt, bb2);
34378 gimple_set_bb (return_stmt, bb2);
34381 make_edge (bb1, bb3, EDGE_FALSE_VALUE);
34384 make_edge (bb2, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
34391 /* This parses the attribute arguments to target in DECL and determines
34392 the right builtin to use to match the platform specification.
34393 It returns the priority value for this version decl. If PREDICATE_LIST
34394 is not NULL, it stores the list of cpu features that need to be checked
34395 before dispatching this function. */
34397 static unsigned int
34398 get_builtin_code_for_version (tree decl, tree *predicate_list)
34401 struct cl_target_option cur_target;
34403 struct cl_target_option *new_target;
34404 const char *arg_str = NULL;
34405 const char *attrs_str = NULL;
34406 char *tok_str = NULL;
34409 /* Priority of i386 features, greater value is higher priority. This is
34410 used to decide the order in which function dispatch must happen. For
34411 instance, a version specialized for SSE4.2 should be checked for dispatch
34412 before a version for SSE3, as SSE4.2 implies SSE3. */
34413 enum feature_priority
34444 enum feature_priority priority = P_ZERO;
34446 /* These are the target attribute strings for which a dispatcher is
34447 available, from fold_builtin_cpu. */
34449 static struct _feature_list
34451 const char *const name;
34452 const enum feature_priority priority;
34454 const feature_list[] =
34460 {"sse4a", P_SSE4_A},
34461 {"ssse3", P_SSSE3},
34462 {"sse4.1", P_SSE4_1},
34463 {"sse4.2", P_SSE4_2},
34464 {"popcnt", P_POPCNT},
34472 {"avx512f", P_AVX512F}
34476 static unsigned int NUM_FEATURES
34477 = sizeof (feature_list) / sizeof (struct _feature_list);
34481 tree predicate_chain = NULL_TREE;
34482 tree predicate_decl, predicate_arg;
34484 attrs = lookup_attribute ("target", DECL_ATTRIBUTES (decl));
34485 gcc_assert (attrs != NULL);
34487 attrs = TREE_VALUE (TREE_VALUE (attrs));
34489 gcc_assert (TREE_CODE (attrs) == STRING_CST);
34490 attrs_str = TREE_STRING_POINTER (attrs);
34492 /* Return priority zero for default function. */
34493 if (strcmp (attrs_str, "default") == 0)
34496 /* Handle arch= if specified. For priority, set it to be 1 more than
34497 the best instruction set the processor can handle. For instance, if
34498 there is a version for atom and a version for ssse3 (the highest ISA
34499 priority for atom), the atom version must be checked for dispatch
34500 before the ssse3 version. */
34501 if (strstr (attrs_str, "arch=") != NULL)
34503 cl_target_option_save (&cur_target, &global_options);
34504 target_node = ix86_valid_target_attribute_tree (attrs, &global_options,
34505 &global_options_set);
34507 gcc_assert (target_node);
34508 new_target = TREE_TARGET_OPTION (target_node);
34509 gcc_assert (new_target);
34511 if (new_target->arch_specified && new_target->arch > 0)
34513 switch (new_target->arch)
34515 case PROCESSOR_CORE2:
34517 priority = P_PROC_SSSE3;
34519 case PROCESSOR_NEHALEM:
34520 if (new_target->x_ix86_isa_flags & OPTION_MASK_ISA_AES)
34521 arg_str = "westmere";
34523 /* We translate "arch=corei7" and "arch=nehalem" to
34524 "corei7" so that it will be mapped to M_INTEL_COREI7
34525 as cpu type to cover all M_INTEL_COREI7_XXXs. */
34526 arg_str = "corei7";
34527 priority = P_PROC_SSE4_2;
34529 case PROCESSOR_SANDYBRIDGE:
34530 if (new_target->x_ix86_isa_flags & OPTION_MASK_ISA_F16C)
34531 arg_str = "ivybridge";
34533 arg_str = "sandybridge";
34534 priority = P_PROC_AVX;
34536 case PROCESSOR_HASWELL:
34537 if (new_target->x_ix86_isa_flags & OPTION_MASK_ISA_ADX)
34538 arg_str = "broadwell";
34540 arg_str = "haswell";
34541 priority = P_PROC_AVX2;
34543 case PROCESSOR_BONNELL:
34544 arg_str = "bonnell";
34545 priority = P_PROC_SSSE3;
34547 case PROCESSOR_KNL:
34549 priority = P_PROC_AVX512F;
34551 case PROCESSOR_SILVERMONT:
34552 arg_str = "silvermont";
34553 priority = P_PROC_SSE4_2;
34555 case PROCESSOR_AMDFAM10:
34556 arg_str = "amdfam10h";
34557 priority = P_PROC_SSE4_A;
34559 case PROCESSOR_BTVER1:
34560 arg_str = "btver1";
34561 priority = P_PROC_SSE4_A;
34563 case PROCESSOR_BTVER2:
34564 arg_str = "btver2";
34565 priority = P_PROC_BMI;
34567 case PROCESSOR_BDVER1:
34568 arg_str = "bdver1";
34569 priority = P_PROC_XOP;
34571 case PROCESSOR_BDVER2:
34572 arg_str = "bdver2";
34573 priority = P_PROC_FMA;
34575 case PROCESSOR_BDVER3:
34576 arg_str = "bdver3";
34577 priority = P_PROC_FMA;
34579 case PROCESSOR_BDVER4:
34580 arg_str = "bdver4";
34581 priority = P_PROC_AVX2;
34586 cl_target_option_restore (&global_options, &cur_target);
34588 if (predicate_list && arg_str == NULL)
34590 error_at (DECL_SOURCE_LOCATION (decl),
34591 "No dispatcher found for the versioning attributes");
34595 if (predicate_list)
34597 predicate_decl = ix86_builtins [(int) IX86_BUILTIN_CPU_IS];
34598 /* For a C string literal the length includes the trailing NULL. */
34599 predicate_arg = build_string_literal (strlen (arg_str) + 1, arg_str);
34600 predicate_chain = tree_cons (predicate_decl, predicate_arg,
34605 /* Process feature name. */
34606 tok_str = (char *) xmalloc (strlen (attrs_str) + 1);
34607 strcpy (tok_str, attrs_str);
34608 token = strtok (tok_str, ",");
34609 predicate_decl = ix86_builtins [(int) IX86_BUILTIN_CPU_SUPPORTS];
34611 while (token != NULL)
34613 /* Do not process "arch=" */
34614 if (strncmp (token, "arch=", 5) == 0)
34616 token = strtok (NULL, ",");
34619 for (i = 0; i < NUM_FEATURES; ++i)
34621 if (strcmp (token, feature_list[i].name) == 0)
34623 if (predicate_list)
34625 predicate_arg = build_string_literal (
34626 strlen (feature_list[i].name) + 1,
34627 feature_list[i].name);
34628 predicate_chain = tree_cons (predicate_decl, predicate_arg,
34631 /* Find the maximum priority feature. */
34632 if (feature_list[i].priority > priority)
34633 priority = feature_list[i].priority;
34638 if (predicate_list && i == NUM_FEATURES)
34640 error_at (DECL_SOURCE_LOCATION (decl),
34641 "No dispatcher found for %s", token);
34644 token = strtok (NULL, ",");
34648 if (predicate_list && predicate_chain == NULL_TREE)
34650 error_at (DECL_SOURCE_LOCATION (decl),
34651 "No dispatcher found for the versioning attributes : %s",
34655 else if (predicate_list)
34657 predicate_chain = nreverse (predicate_chain);
34658 *predicate_list = predicate_chain;
34664 /* This compares the priority of target features in function DECL1
34665 and DECL2. It returns positive value if DECL1 is higher priority,
34666 negative value if DECL2 is higher priority and 0 if they are the
34670 ix86_compare_version_priority (tree decl1, tree decl2)
34672 unsigned int priority1 = get_builtin_code_for_version (decl1, NULL);
34673 unsigned int priority2 = get_builtin_code_for_version (decl2, NULL);
34675 return (int)priority1 - (int)priority2;
34678 /* V1 and V2 point to function versions with different priorities
34679 based on the target ISA. This function compares their priorities. */
34682 feature_compare (const void *v1, const void *v2)
34684 typedef struct _function_version_info
34687 tree predicate_chain;
34688 unsigned int dispatch_priority;
34689 } function_version_info;
34691 const function_version_info c1 = *(const function_version_info *)v1;
34692 const function_version_info c2 = *(const function_version_info *)v2;
34693 return (c2.dispatch_priority - c1.dispatch_priority);
34696 /* This function generates the dispatch function for
34697 multi-versioned functions. DISPATCH_DECL is the function which will
34698 contain the dispatch logic. FNDECLS are the function choices for
34699 dispatch, and is a tree chain. EMPTY_BB is the basic block pointer
34700 in DISPATCH_DECL in which the dispatch code is generated. */
34703 dispatch_function_versions (tree dispatch_decl,
34705 basic_block *empty_bb)
34708 gimple ifunc_cpu_init_stmt;
34712 vec<tree> *fndecls;
34713 unsigned int num_versions = 0;
34714 unsigned int actual_versions = 0;
34717 struct _function_version_info
34720 tree predicate_chain;
34721 unsigned int dispatch_priority;
34722 }*function_version_info;
34724 gcc_assert (dispatch_decl != NULL
34725 && fndecls_p != NULL
34726 && empty_bb != NULL);
34728 /*fndecls_p is actually a vector. */
34729 fndecls = static_cast<vec<tree> *> (fndecls_p);
34731 /* At least one more version other than the default. */
34732 num_versions = fndecls->length ();
34733 gcc_assert (num_versions >= 2);
34735 function_version_info = (struct _function_version_info *)
34736 XNEWVEC (struct _function_version_info, (num_versions - 1));
34738 /* The first version in the vector is the default decl. */
34739 default_decl = (*fndecls)[0];
34741 push_cfun (DECL_STRUCT_FUNCTION (dispatch_decl));
34743 gseq = bb_seq (*empty_bb);
34744 /* Function version dispatch is via IFUNC. IFUNC resolvers fire before
34745 constructors, so explicity call __builtin_cpu_init here. */
34746 ifunc_cpu_init_stmt = gimple_build_call_vec (
34747 ix86_builtins [(int) IX86_BUILTIN_CPU_INIT], vNULL);
34748 gimple_seq_add_stmt (&gseq, ifunc_cpu_init_stmt);
34749 gimple_set_bb (ifunc_cpu_init_stmt, *empty_bb);
34750 set_bb_seq (*empty_bb, gseq);
34755 for (ix = 1; fndecls->iterate (ix, &ele); ++ix)
34757 tree version_decl = ele;
34758 tree predicate_chain = NULL_TREE;
34759 unsigned int priority;
34760 /* Get attribute string, parse it and find the right predicate decl.
34761 The predicate function could be a lengthy combination of many
34762 features, like arch-type and various isa-variants. */
34763 priority = get_builtin_code_for_version (version_decl,
34766 if (predicate_chain == NULL_TREE)
34769 function_version_info [actual_versions].version_decl = version_decl;
34770 function_version_info [actual_versions].predicate_chain
34772 function_version_info [actual_versions].dispatch_priority = priority;
34776 /* Sort the versions according to descending order of dispatch priority. The
34777 priority is based on the ISA. This is not a perfect solution. There
34778 could still be ambiguity. If more than one function version is suitable
34779 to execute, which one should be dispatched? In future, allow the user
34780 to specify a dispatch priority next to the version. */
34781 qsort (function_version_info, actual_versions,
34782 sizeof (struct _function_version_info), feature_compare);
34784 for (i = 0; i < actual_versions; ++i)
34785 *empty_bb = add_condition_to_bb (dispatch_decl,
34786 function_version_info[i].version_decl,
34787 function_version_info[i].predicate_chain,
34790 /* dispatch default version at the end. */
34791 *empty_bb = add_condition_to_bb (dispatch_decl, default_decl,
34794 free (function_version_info);
34798 /* Comparator function to be used in qsort routine to sort attribute
34799 specification strings to "target". */
34802 attr_strcmp (const void *v1, const void *v2)
34804 const char *c1 = *(char *const*)v1;
34805 const char *c2 = *(char *const*)v2;
34806 return strcmp (c1, c2);
34809 /* ARGLIST is the argument to target attribute. This function tokenizes
34810 the comma separated arguments, sorts them and returns a string which
34811 is a unique identifier for the comma separated arguments. It also
34812 replaces non-identifier characters "=,-" with "_". */
34815 sorted_attr_string (tree arglist)
34818 size_t str_len_sum = 0;
34819 char **args = NULL;
34820 char *attr_str, *ret_str;
34822 unsigned int argnum = 1;
34825 for (arg = arglist; arg; arg = TREE_CHAIN (arg))
34827 const char *str = TREE_STRING_POINTER (TREE_VALUE (arg));
34828 size_t len = strlen (str);
34829 str_len_sum += len + 1;
34830 if (arg != arglist)
34832 for (i = 0; i < strlen (str); i++)
34837 attr_str = XNEWVEC (char, str_len_sum);
34839 for (arg = arglist; arg; arg = TREE_CHAIN (arg))
34841 const char *str = TREE_STRING_POINTER (TREE_VALUE (arg));
34842 size_t len = strlen (str);
34843 memcpy (attr_str + str_len_sum, str, len);
34844 attr_str[str_len_sum + len] = TREE_CHAIN (arg) ? ',' : '\0';
34845 str_len_sum += len + 1;
34848 /* Replace "=,-" with "_". */
34849 for (i = 0; i < strlen (attr_str); i++)
34850 if (attr_str[i] == '=' || attr_str[i]== '-')
34856 args = XNEWVEC (char *, argnum);
34859 attr = strtok (attr_str, ",");
34860 while (attr != NULL)
34864 attr = strtok (NULL, ",");
34867 qsort (args, argnum, sizeof (char *), attr_strcmp);
34869 ret_str = XNEWVEC (char, str_len_sum);
34871 for (i = 0; i < argnum; i++)
34873 size_t len = strlen (args[i]);
34874 memcpy (ret_str + str_len_sum, args[i], len);
34875 ret_str[str_len_sum + len] = i < argnum - 1 ? '_' : '\0';
34876 str_len_sum += len + 1;
34880 XDELETEVEC (attr_str);
34884 /* This function changes the assembler name for functions that are
34885 versions. If DECL is a function version and has a "target"
34886 attribute, it appends the attribute string to its assembler name. */
34889 ix86_mangle_function_version_assembler_name (tree decl, tree id)
34892 const char *orig_name, *version_string;
34893 char *attr_str, *assembler_name;
34895 if (DECL_DECLARED_INLINE_P (decl)
34896 && lookup_attribute ("gnu_inline",
34897 DECL_ATTRIBUTES (decl)))
34898 error_at (DECL_SOURCE_LOCATION (decl),
34899 "Function versions cannot be marked as gnu_inline,"
34900 " bodies have to be generated");
34902 if (DECL_VIRTUAL_P (decl)
34903 || DECL_VINDEX (decl))
34904 sorry ("Virtual function multiversioning not supported");
34906 version_attr = lookup_attribute ("target", DECL_ATTRIBUTES (decl));
34908 /* target attribute string cannot be NULL. */
34909 gcc_assert (version_attr != NULL_TREE);
34911 orig_name = IDENTIFIER_POINTER (id);
34913 = TREE_STRING_POINTER (TREE_VALUE (TREE_VALUE (version_attr)));
34915 if (strcmp (version_string, "default") == 0)
34918 attr_str = sorted_attr_string (TREE_VALUE (version_attr));
34919 assembler_name = XNEWVEC (char, strlen (orig_name) + strlen (attr_str) + 2);
34921 sprintf (assembler_name, "%s.%s", orig_name, attr_str);
34923 /* Allow assembler name to be modified if already set. */
34924 if (DECL_ASSEMBLER_NAME_SET_P (decl))
34925 SET_DECL_RTL (decl, NULL);
34927 tree ret = get_identifier (assembler_name);
34928 XDELETEVEC (attr_str);
34929 XDELETEVEC (assembler_name);
34933 /* This function returns true if FN1 and FN2 are versions of the same function,
34934 that is, the target strings of the function decls are different. This assumes
34935 that FN1 and FN2 have the same signature. */
34938 ix86_function_versions (tree fn1, tree fn2)
34941 char *target1, *target2;
34944 if (TREE_CODE (fn1) != FUNCTION_DECL
34945 || TREE_CODE (fn2) != FUNCTION_DECL)
34948 attr1 = lookup_attribute ("target", DECL_ATTRIBUTES (fn1));
34949 attr2 = lookup_attribute ("target", DECL_ATTRIBUTES (fn2));
34951 /* At least one function decl should have the target attribute specified. */
34952 if (attr1 == NULL_TREE && attr2 == NULL_TREE)
34955 /* Diagnose missing target attribute if one of the decls is already
34956 multi-versioned. */
34957 if (attr1 == NULL_TREE || attr2 == NULL_TREE)
34959 if (DECL_FUNCTION_VERSIONED (fn1) || DECL_FUNCTION_VERSIONED (fn2))
34961 if (attr2 != NULL_TREE)
34968 error_at (DECL_SOURCE_LOCATION (fn2),
34969 "missing %<target%> attribute for multi-versioned %D",
34971 inform (DECL_SOURCE_LOCATION (fn1),
34972 "previous declaration of %D", fn1);
34973 /* Prevent diagnosing of the same error multiple times. */
34974 DECL_ATTRIBUTES (fn2)
34975 = tree_cons (get_identifier ("target"),
34976 copy_node (TREE_VALUE (attr1)),
34977 DECL_ATTRIBUTES (fn2));
34982 target1 = sorted_attr_string (TREE_VALUE (attr1));
34983 target2 = sorted_attr_string (TREE_VALUE (attr2));
34985 /* The sorted target strings must be different for fn1 and fn2
34987 if (strcmp (target1, target2) == 0)
34992 XDELETEVEC (target1);
34993 XDELETEVEC (target2);
34999 ix86_mangle_decl_assembler_name (tree decl, tree id)
35001 /* For function version, add the target suffix to the assembler name. */
35002 if (TREE_CODE (decl) == FUNCTION_DECL
35003 && DECL_FUNCTION_VERSIONED (decl))
35004 id = ix86_mangle_function_version_assembler_name (decl, id);
35005 #ifdef SUBTARGET_MANGLE_DECL_ASSEMBLER_NAME
35006 id = SUBTARGET_MANGLE_DECL_ASSEMBLER_NAME (decl, id);
35012 /* Return a new name by appending SUFFIX to the DECL name. If make_unique
35013 is true, append the full path name of the source file. */
35016 make_name (tree decl, const char *suffix, bool make_unique)
35018 char *global_var_name;
35021 const char *unique_name = NULL;
35023 name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl));
35025 /* Get a unique name that can be used globally without any chances
35026 of collision at link time. */
35028 unique_name = IDENTIFIER_POINTER (get_file_function_name ("\0"));
35030 name_len = strlen (name) + strlen (suffix) + 2;
35033 name_len += strlen (unique_name) + 1;
35034 global_var_name = XNEWVEC (char, name_len);
35036 /* Use '.' to concatenate names as it is demangler friendly. */
35038 snprintf (global_var_name, name_len, "%s.%s.%s", name, unique_name,
35041 snprintf (global_var_name, name_len, "%s.%s", name, suffix);
35043 return global_var_name;
35046 #if defined (ASM_OUTPUT_TYPE_DIRECTIVE)
35048 /* Make a dispatcher declaration for the multi-versioned function DECL.
35049 Calls to DECL function will be replaced with calls to the dispatcher
35050 by the front-end. Return the decl created. */
35053 make_dispatcher_decl (const tree decl)
35057 tree fn_type, func_type;
35058 bool is_uniq = false;
35060 if (TREE_PUBLIC (decl) == 0)
35063 func_name = make_name (decl, "ifunc", is_uniq);
35065 fn_type = TREE_TYPE (decl);
35066 func_type = build_function_type (TREE_TYPE (fn_type),
35067 TYPE_ARG_TYPES (fn_type));
35069 func_decl = build_fn_decl (func_name, func_type);
35070 XDELETEVEC (func_name);
35071 TREE_USED (func_decl) = 1;
35072 DECL_CONTEXT (func_decl) = NULL_TREE;
35073 DECL_INITIAL (func_decl) = error_mark_node;
35074 DECL_ARTIFICIAL (func_decl) = 1;
35075 /* Mark this func as external, the resolver will flip it again if
35076 it gets generated. */
35077 DECL_EXTERNAL (func_decl) = 1;
35078 /* This will be of type IFUNCs have to be externally visible. */
35079 TREE_PUBLIC (func_decl) = 1;
35086 /* Returns true if decl is multi-versioned and DECL is the default function,
35087 that is it is not tagged with target specific optimization. */
35090 is_function_default_version (const tree decl)
35092 if (TREE_CODE (decl) != FUNCTION_DECL
35093 || !DECL_FUNCTION_VERSIONED (decl))
35095 tree attr = lookup_attribute ("target", DECL_ATTRIBUTES (decl));
35097 attr = TREE_VALUE (TREE_VALUE (attr));
35098 return (TREE_CODE (attr) == STRING_CST
35099 && strcmp (TREE_STRING_POINTER (attr), "default") == 0);
35102 /* Make a dispatcher declaration for the multi-versioned function DECL.
35103 Calls to DECL function will be replaced with calls to the dispatcher
35104 by the front-end. Returns the decl of the dispatcher function. */
35107 ix86_get_function_versions_dispatcher (void *decl)
35109 tree fn = (tree) decl;
35110 struct cgraph_node *node = NULL;
35111 struct cgraph_node *default_node = NULL;
35112 struct cgraph_function_version_info *node_v = NULL;
35113 struct cgraph_function_version_info *first_v = NULL;
35115 tree dispatch_decl = NULL;
35117 struct cgraph_function_version_info *default_version_info = NULL;
35119 gcc_assert (fn != NULL && DECL_FUNCTION_VERSIONED (fn));
35121 node = cgraph_node::get (fn);
35122 gcc_assert (node != NULL);
35124 node_v = node->function_version ();
35125 gcc_assert (node_v != NULL);
35127 if (node_v->dispatcher_resolver != NULL)
35128 return node_v->dispatcher_resolver;
35130 /* Find the default version and make it the first node. */
35132 /* Go to the beginning of the chain. */
35133 while (first_v->prev != NULL)
35134 first_v = first_v->prev;
35135 default_version_info = first_v;
35136 while (default_version_info != NULL)
35138 if (is_function_default_version
35139 (default_version_info->this_node->decl))
35141 default_version_info = default_version_info->next;
35144 /* If there is no default node, just return NULL. */
35145 if (default_version_info == NULL)
35148 /* Make default info the first node. */
35149 if (first_v != default_version_info)
35151 default_version_info->prev->next = default_version_info->next;
35152 if (default_version_info->next)
35153 default_version_info->next->prev = default_version_info->prev;
35154 first_v->prev = default_version_info;
35155 default_version_info->next = first_v;
35156 default_version_info->prev = NULL;
35159 default_node = default_version_info->this_node;
35161 #if defined (ASM_OUTPUT_TYPE_DIRECTIVE)
35162 if (targetm.has_ifunc_p ())
35164 struct cgraph_function_version_info *it_v = NULL;
35165 struct cgraph_node *dispatcher_node = NULL;
35166 struct cgraph_function_version_info *dispatcher_version_info = NULL;
35168 /* Right now, the dispatching is done via ifunc. */
35169 dispatch_decl = make_dispatcher_decl (default_node->decl);
35171 dispatcher_node = cgraph_node::get_create (dispatch_decl);
35172 gcc_assert (dispatcher_node != NULL);
35173 dispatcher_node->dispatcher_function = 1;
35174 dispatcher_version_info
35175 = dispatcher_node->insert_new_function_version ();
35176 dispatcher_version_info->next = default_version_info;
35177 dispatcher_node->definition = 1;
35179 /* Set the dispatcher for all the versions. */
35180 it_v = default_version_info;
35181 while (it_v != NULL)
35183 it_v->dispatcher_resolver = dispatch_decl;
35190 error_at (DECL_SOURCE_LOCATION (default_node->decl),
35191 "multiversioning needs ifunc which is not supported "
35195 return dispatch_decl;
35198 /* Makes a function attribute of the form NAME(ARG_NAME) and chains
35202 make_attribute (const char *name, const char *arg_name, tree chain)
35205 tree attr_arg_name;
35209 attr_name = get_identifier (name);
35210 attr_arg_name = build_string (strlen (arg_name), arg_name);
35211 attr_args = tree_cons (NULL_TREE, attr_arg_name, NULL_TREE);
35212 attr = tree_cons (attr_name, attr_args, chain);
35216 /* Make the resolver function decl to dispatch the versions of
35217 a multi-versioned function, DEFAULT_DECL. Create an
35218 empty basic block in the resolver and store the pointer in
35219 EMPTY_BB. Return the decl of the resolver function. */
35222 make_resolver_func (const tree default_decl,
35223 const tree dispatch_decl,
35224 basic_block *empty_bb)
35226 char *resolver_name;
35227 tree decl, type, decl_name, t;
35228 bool is_uniq = false;
35230 /* IFUNC's have to be globally visible. So, if the default_decl is
35231 not, then the name of the IFUNC should be made unique. */
35232 if (TREE_PUBLIC (default_decl) == 0)
35235 /* Append the filename to the resolver function if the versions are
35236 not externally visible. This is because the resolver function has
35237 to be externally visible for the loader to find it. So, appending
35238 the filename will prevent conflicts with a resolver function from
35239 another module which is based on the same version name. */
35240 resolver_name = make_name (default_decl, "resolver", is_uniq);
35242 /* The resolver function should return a (void *). */
35243 type = build_function_type_list (ptr_type_node, NULL_TREE);
35245 decl = build_fn_decl (resolver_name, type);
35246 decl_name = get_identifier (resolver_name);
35247 SET_DECL_ASSEMBLER_NAME (decl, decl_name);
35249 DECL_NAME (decl) = decl_name;
35250 TREE_USED (decl) = 1;
35251 DECL_ARTIFICIAL (decl) = 1;
35252 DECL_IGNORED_P (decl) = 0;
35253 /* IFUNC resolvers have to be externally visible. */
35254 TREE_PUBLIC (decl) = 1;
35255 DECL_UNINLINABLE (decl) = 1;
35257 /* Resolver is not external, body is generated. */
35258 DECL_EXTERNAL (decl) = 0;
35259 DECL_EXTERNAL (dispatch_decl) = 0;
35261 DECL_CONTEXT (decl) = NULL_TREE;
35262 DECL_INITIAL (decl) = make_node (BLOCK);
35263 DECL_STATIC_CONSTRUCTOR (decl) = 0;
35265 if (DECL_COMDAT_GROUP (default_decl)
35266 || TREE_PUBLIC (default_decl))
35268 /* In this case, each translation unit with a call to this
35269 versioned function will put out a resolver. Ensure it
35270 is comdat to keep just one copy. */
35271 DECL_COMDAT (decl) = 1;
35272 make_decl_one_only (decl, DECL_ASSEMBLER_NAME (decl));
35274 /* Build result decl and add to function_decl. */
35275 t = build_decl (UNKNOWN_LOCATION, RESULT_DECL, NULL_TREE, ptr_type_node);
35276 DECL_ARTIFICIAL (t) = 1;
35277 DECL_IGNORED_P (t) = 1;
35278 DECL_RESULT (decl) = t;
35280 gimplify_function_tree (decl);
35281 push_cfun (DECL_STRUCT_FUNCTION (decl));
35282 *empty_bb = init_lowered_empty_function (decl, false, 0);
35284 cgraph_node::add_new_function (decl, true);
35285 symtab->call_cgraph_insertion_hooks (cgraph_node::get_create (decl));
35289 gcc_assert (dispatch_decl != NULL);
35290 /* Mark dispatch_decl as "ifunc" with resolver as resolver_name. */
35291 DECL_ATTRIBUTES (dispatch_decl)
35292 = make_attribute ("ifunc", resolver_name, DECL_ATTRIBUTES (dispatch_decl));
35294 /* Create the alias for dispatch to resolver here. */
35295 /*cgraph_create_function_alias (dispatch_decl, decl);*/
35296 cgraph_node::create_same_body_alias (dispatch_decl, decl);
35297 XDELETEVEC (resolver_name);
35301 /* Generate the dispatching code body to dispatch multi-versioned function
35302 DECL. The target hook is called to process the "target" attributes and
35303 provide the code to dispatch the right function at run-time. NODE points
35304 to the dispatcher decl whose body will be created. */
35307 ix86_generate_version_dispatcher_body (void *node_p)
35309 tree resolver_decl;
35310 basic_block empty_bb;
35311 tree default_ver_decl;
35312 struct cgraph_node *versn;
35313 struct cgraph_node *node;
35315 struct cgraph_function_version_info *node_version_info = NULL;
35316 struct cgraph_function_version_info *versn_info = NULL;
35318 node = (cgraph_node *)node_p;
35320 node_version_info = node->function_version ();
35321 gcc_assert (node->dispatcher_function
35322 && node_version_info != NULL);
35324 if (node_version_info->dispatcher_resolver)
35325 return node_version_info->dispatcher_resolver;
35327 /* The first version in the chain corresponds to the default version. */
35328 default_ver_decl = node_version_info->next->this_node->decl;
35330 /* node is going to be an alias, so remove the finalized bit. */
35331 node->definition = false;
35333 resolver_decl = make_resolver_func (default_ver_decl,
35334 node->decl, &empty_bb);
35336 node_version_info->dispatcher_resolver = resolver_decl;
35338 push_cfun (DECL_STRUCT_FUNCTION (resolver_decl));
35340 auto_vec<tree, 2> fn_ver_vec;
35342 for (versn_info = node_version_info->next; versn_info;
35343 versn_info = versn_info->next)
35345 versn = versn_info->this_node;
35346 /* Check for virtual functions here again, as by this time it should
35347 have been determined if this function needs a vtable index or
35348 not. This happens for methods in derived classes that override
35349 virtual methods in base classes but are not explicitly marked as
35351 if (DECL_VINDEX (versn->decl))
35352 sorry ("Virtual function multiversioning not supported");
35354 fn_ver_vec.safe_push (versn->decl);
35357 dispatch_function_versions (resolver_decl, &fn_ver_vec, &empty_bb);
35358 cgraph_edge::rebuild_edges ();
35360 return resolver_decl;
35362 /* This builds the processor_model struct type defined in
35363 libgcc/config/i386/cpuinfo.c */
35366 build_processor_model_struct (void)
35368 const char *field_name[] = {"__cpu_vendor", "__cpu_type", "__cpu_subtype",
35370 tree field = NULL_TREE, field_chain = NULL_TREE;
35372 tree type = make_node (RECORD_TYPE);
35374 /* The first 3 fields are unsigned int. */
35375 for (i = 0; i < 3; ++i)
35377 field = build_decl (UNKNOWN_LOCATION, FIELD_DECL,
35378 get_identifier (field_name[i]), unsigned_type_node);
35379 if (field_chain != NULL_TREE)
35380 DECL_CHAIN (field) = field_chain;
35381 field_chain = field;
35384 /* The last field is an array of unsigned integers of size one. */
35385 field = build_decl (UNKNOWN_LOCATION, FIELD_DECL,
35386 get_identifier (field_name[3]),
35387 build_array_type (unsigned_type_node,
35388 build_index_type (size_one_node)));
35389 if (field_chain != NULL_TREE)
35390 DECL_CHAIN (field) = field_chain;
35391 field_chain = field;
35393 finish_builtin_struct (type, "__processor_model", field_chain, NULL_TREE);
35397 /* Returns a extern, comdat VAR_DECL of type TYPE and name NAME. */
35400 make_var_decl (tree type, const char *name)
35404 new_decl = build_decl (UNKNOWN_LOCATION,
35406 get_identifier(name),
35409 DECL_EXTERNAL (new_decl) = 1;
35410 TREE_STATIC (new_decl) = 1;
35411 TREE_PUBLIC (new_decl) = 1;
35412 DECL_INITIAL (new_decl) = 0;
35413 DECL_ARTIFICIAL (new_decl) = 0;
35414 DECL_PRESERVE_P (new_decl) = 1;
35416 make_decl_one_only (new_decl, DECL_ASSEMBLER_NAME (new_decl));
35417 assemble_variable (new_decl, 0, 0, 0);
35422 /* FNDECL is a __builtin_cpu_is or a __builtin_cpu_supports call that is folded
35423 into an integer defined in libgcc/config/i386/cpuinfo.c */
35426 fold_builtin_cpu (tree fndecl, tree *args)
35429 enum ix86_builtins fn_code = (enum ix86_builtins)
35430 DECL_FUNCTION_CODE (fndecl);
35431 tree param_string_cst = NULL;
35433 /* This is the order of bit-fields in __processor_features in cpuinfo.c */
35434 enum processor_features
35457 /* These are the values for vendor types and cpu types and subtypes
35458 in cpuinfo.c. Cpu types and subtypes should be subtracted by
35459 the corresponding start value. */
35460 enum processor_model
35470 M_INTEL_SILVERMONT,
35474 M_CPU_SUBTYPE_START,
35475 M_INTEL_COREI7_NEHALEM,
35476 M_INTEL_COREI7_WESTMERE,
35477 M_INTEL_COREI7_SANDYBRIDGE,
35478 M_AMDFAM10H_BARCELONA,
35479 M_AMDFAM10H_SHANGHAI,
35480 M_AMDFAM10H_ISTANBUL,
35481 M_AMDFAM15H_BDVER1,
35482 M_AMDFAM15H_BDVER2,
35483 M_AMDFAM15H_BDVER3,
35484 M_AMDFAM15H_BDVER4,
35485 M_INTEL_COREI7_IVYBRIDGE,
35486 M_INTEL_COREI7_HASWELL,
35487 M_INTEL_COREI7_BROADWELL
35490 static struct _arch_names_table
35492 const char *const name;
35493 const enum processor_model model;
35495 const arch_names_table[] =
35498 {"intel", M_INTEL},
35499 {"atom", M_INTEL_BONNELL},
35500 {"slm", M_INTEL_SILVERMONT},
35501 {"core2", M_INTEL_CORE2},
35502 {"corei7", M_INTEL_COREI7},
35503 {"nehalem", M_INTEL_COREI7_NEHALEM},
35504 {"westmere", M_INTEL_COREI7_WESTMERE},
35505 {"sandybridge", M_INTEL_COREI7_SANDYBRIDGE},
35506 {"ivybridge", M_INTEL_COREI7_IVYBRIDGE},
35507 {"haswell", M_INTEL_COREI7_HASWELL},
35508 {"broadwell", M_INTEL_COREI7_BROADWELL},
35509 {"bonnell", M_INTEL_BONNELL},
35510 {"silvermont", M_INTEL_SILVERMONT},
35511 {"knl", M_INTEL_KNL},
35512 {"amdfam10h", M_AMDFAM10H},
35513 {"barcelona", M_AMDFAM10H_BARCELONA},
35514 {"shanghai", M_AMDFAM10H_SHANGHAI},
35515 {"istanbul", M_AMDFAM10H_ISTANBUL},
35516 {"btver1", M_AMD_BTVER1},
35517 {"amdfam15h", M_AMDFAM15H},
35518 {"bdver1", M_AMDFAM15H_BDVER1},
35519 {"bdver2", M_AMDFAM15H_BDVER2},
35520 {"bdver3", M_AMDFAM15H_BDVER3},
35521 {"bdver4", M_AMDFAM15H_BDVER4},
35522 {"btver2", M_AMD_BTVER2},
35525 static struct _isa_names_table
35527 const char *const name;
35528 const enum processor_features feature;
35530 const isa_names_table[] =
35534 {"popcnt", F_POPCNT},
35538 {"ssse3", F_SSSE3},
35539 {"sse4a", F_SSE4_A},
35540 {"sse4.1", F_SSE4_1},
35541 {"sse4.2", F_SSE4_2},
35547 {"avx512f",F_AVX512F},
35552 tree __processor_model_type = build_processor_model_struct ();
35553 tree __cpu_model_var = make_var_decl (__processor_model_type,
35557 varpool_node::add (__cpu_model_var);
35559 gcc_assert ((args != NULL) && (*args != NULL));
35561 param_string_cst = *args;
35562 while (param_string_cst
35563 && TREE_CODE (param_string_cst) != STRING_CST)
35565 /* *args must be a expr that can contain other EXPRS leading to a
35567 if (!EXPR_P (param_string_cst))
35569 error ("Parameter to builtin must be a string constant or literal");
35570 return integer_zero_node;
35572 param_string_cst = TREE_OPERAND (EXPR_CHECK (param_string_cst), 0);
35575 gcc_assert (param_string_cst);
35577 if (fn_code == IX86_BUILTIN_CPU_IS)
35583 unsigned int field_val = 0;
35584 unsigned int NUM_ARCH_NAMES
35585 = sizeof (arch_names_table) / sizeof (struct _arch_names_table);
35587 for (i = 0; i < NUM_ARCH_NAMES; i++)
35588 if (strcmp (arch_names_table[i].name,
35589 TREE_STRING_POINTER (param_string_cst)) == 0)
35592 if (i == NUM_ARCH_NAMES)
35594 error ("Parameter to builtin not valid: %s",
35595 TREE_STRING_POINTER (param_string_cst));
35596 return integer_zero_node;
35599 field = TYPE_FIELDS (__processor_model_type);
35600 field_val = arch_names_table[i].model;
35602 /* CPU types are stored in the next field. */
35603 if (field_val > M_CPU_TYPE_START
35604 && field_val < M_CPU_SUBTYPE_START)
35606 field = DECL_CHAIN (field);
35607 field_val -= M_CPU_TYPE_START;
35610 /* CPU subtypes are stored in the next field. */
35611 if (field_val > M_CPU_SUBTYPE_START)
35613 field = DECL_CHAIN ( DECL_CHAIN (field));
35614 field_val -= M_CPU_SUBTYPE_START;
35617 /* Get the appropriate field in __cpu_model. */
35618 ref = build3 (COMPONENT_REF, TREE_TYPE (field), __cpu_model_var,
35621 /* Check the value. */
35622 final = build2 (EQ_EXPR, unsigned_type_node, ref,
35623 build_int_cstu (unsigned_type_node, field_val));
35624 return build1 (CONVERT_EXPR, integer_type_node, final);
35626 else if (fn_code == IX86_BUILTIN_CPU_SUPPORTS)
35633 unsigned int field_val = 0;
35634 unsigned int NUM_ISA_NAMES
35635 = sizeof (isa_names_table) / sizeof (struct _isa_names_table);
35637 for (i = 0; i < NUM_ISA_NAMES; i++)
35638 if (strcmp (isa_names_table[i].name,
35639 TREE_STRING_POINTER (param_string_cst)) == 0)
35642 if (i == NUM_ISA_NAMES)
35644 error ("Parameter to builtin not valid: %s",
35645 TREE_STRING_POINTER (param_string_cst));
35646 return integer_zero_node;
35649 field = TYPE_FIELDS (__processor_model_type);
35650 /* Get the last field, which is __cpu_features. */
35651 while (DECL_CHAIN (field))
35652 field = DECL_CHAIN (field);
35654 /* Get the appropriate field: __cpu_model.__cpu_features */
35655 ref = build3 (COMPONENT_REF, TREE_TYPE (field), __cpu_model_var,
35658 /* Access the 0th element of __cpu_features array. */
35659 array_elt = build4 (ARRAY_REF, unsigned_type_node, ref,
35660 integer_zero_node, NULL_TREE, NULL_TREE);
35662 field_val = (1 << isa_names_table[i].feature);
35663 /* Return __cpu_model.__cpu_features[0] & field_val */
35664 final = build2 (BIT_AND_EXPR, unsigned_type_node, array_elt,
35665 build_int_cstu (unsigned_type_node, field_val));
35666 return build1 (CONVERT_EXPR, integer_type_node, final);
35668 gcc_unreachable ();
35672 ix86_fold_builtin (tree fndecl, int n_args,
35673 tree *args, bool ignore ATTRIBUTE_UNUSED)
35675 if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_MD)
35677 enum ix86_builtins fn_code = (enum ix86_builtins)
35678 DECL_FUNCTION_CODE (fndecl);
35679 if (fn_code == IX86_BUILTIN_CPU_IS
35680 || fn_code == IX86_BUILTIN_CPU_SUPPORTS)
35682 gcc_assert (n_args == 1);
35683 return fold_builtin_cpu (fndecl, args);
35687 #ifdef SUBTARGET_FOLD_BUILTIN
35688 return SUBTARGET_FOLD_BUILTIN (fndecl, n_args, args, ignore);
35694 /* Make builtins to detect cpu type and features supported. NAME is
35695 the builtin name, CODE is the builtin code, and FTYPE is the function
35696 type of the builtin. */
35699 make_cpu_type_builtin (const char* name, int code,
35700 enum ix86_builtin_func_type ftype, bool is_const)
35705 type = ix86_get_builtin_func_type (ftype);
35706 decl = add_builtin_function (name, type, code, BUILT_IN_MD,
35708 gcc_assert (decl != NULL_TREE);
35709 ix86_builtins[(int) code] = decl;
35710 TREE_READONLY (decl) = is_const;
35713 /* Make builtins to get CPU type and features supported. The created
35716 __builtin_cpu_init (), to detect cpu type and features,
35717 __builtin_cpu_is ("<CPUNAME>"), to check if cpu is of type <CPUNAME>,
35718 __builtin_cpu_supports ("<FEATURE>"), to check if cpu supports <FEATURE>
35722 ix86_init_platform_type_builtins (void)
35724 make_cpu_type_builtin ("__builtin_cpu_init", IX86_BUILTIN_CPU_INIT,
35725 INT_FTYPE_VOID, false);
35726 make_cpu_type_builtin ("__builtin_cpu_is", IX86_BUILTIN_CPU_IS,
35727 INT_FTYPE_PCCHAR, true);
35728 make_cpu_type_builtin ("__builtin_cpu_supports", IX86_BUILTIN_CPU_SUPPORTS,
35729 INT_FTYPE_PCCHAR, true);
35732 /* Internal method for ix86_init_builtins. */
35735 ix86_init_builtins_va_builtins_abi (void)
35737 tree ms_va_ref, sysv_va_ref;
35738 tree fnvoid_va_end_ms, fnvoid_va_end_sysv;
35739 tree fnvoid_va_start_ms, fnvoid_va_start_sysv;
35740 tree fnvoid_va_copy_ms, fnvoid_va_copy_sysv;
35741 tree fnattr_ms = NULL_TREE, fnattr_sysv = NULL_TREE;
35745 fnattr_ms = build_tree_list (get_identifier ("ms_abi"), NULL_TREE);
35746 fnattr_sysv = build_tree_list (get_identifier ("sysv_abi"), NULL_TREE);
35747 ms_va_ref = build_reference_type (ms_va_list_type_node);
35749 build_pointer_type (TREE_TYPE (sysv_va_list_type_node));
35752 build_function_type_list (void_type_node, ms_va_ref, NULL_TREE);
35753 fnvoid_va_start_ms =
35754 build_varargs_function_type_list (void_type_node, ms_va_ref, NULL_TREE);
35755 fnvoid_va_end_sysv =
35756 build_function_type_list (void_type_node, sysv_va_ref, NULL_TREE);
35757 fnvoid_va_start_sysv =
35758 build_varargs_function_type_list (void_type_node, sysv_va_ref,
35760 fnvoid_va_copy_ms =
35761 build_function_type_list (void_type_node, ms_va_ref, ms_va_list_type_node,
35763 fnvoid_va_copy_sysv =
35764 build_function_type_list (void_type_node, sysv_va_ref,
35765 sysv_va_ref, NULL_TREE);
35767 add_builtin_function ("__builtin_ms_va_start", fnvoid_va_start_ms,
35768 BUILT_IN_VA_START, BUILT_IN_NORMAL, NULL, fnattr_ms);
35769 add_builtin_function ("__builtin_ms_va_end", fnvoid_va_end_ms,
35770 BUILT_IN_VA_END, BUILT_IN_NORMAL, NULL, fnattr_ms);
35771 add_builtin_function ("__builtin_ms_va_copy", fnvoid_va_copy_ms,
35772 BUILT_IN_VA_COPY, BUILT_IN_NORMAL, NULL, fnattr_ms);
35773 add_builtin_function ("__builtin_sysv_va_start", fnvoid_va_start_sysv,
35774 BUILT_IN_VA_START, BUILT_IN_NORMAL, NULL, fnattr_sysv);
35775 add_builtin_function ("__builtin_sysv_va_end", fnvoid_va_end_sysv,
35776 BUILT_IN_VA_END, BUILT_IN_NORMAL, NULL, fnattr_sysv);
35777 add_builtin_function ("__builtin_sysv_va_copy", fnvoid_va_copy_sysv,
35778 BUILT_IN_VA_COPY, BUILT_IN_NORMAL, NULL, fnattr_sysv);
35782 ix86_init_builtin_types (void)
35784 tree float128_type_node, float80_type_node;
35786 /* The __float80 type. */
35787 float80_type_node = long_double_type_node;
35788 if (TYPE_MODE (float80_type_node) != XFmode)
35790 /* The __float80 type. */
35791 float80_type_node = make_node (REAL_TYPE);
35793 TYPE_PRECISION (float80_type_node) = 80;
35794 layout_type (float80_type_node);
35796 lang_hooks.types.register_builtin_type (float80_type_node, "__float80");
35798 /* The __float128 type. */
35799 float128_type_node = make_node (REAL_TYPE);
35800 TYPE_PRECISION (float128_type_node) = 128;
35801 layout_type (float128_type_node);
35802 lang_hooks.types.register_builtin_type (float128_type_node, "__float128");
35804 /* This macro is built by i386-builtin-types.awk. */
35805 DEFINE_BUILTIN_PRIMITIVE_TYPES;
35809 ix86_init_builtins (void)
35813 ix86_init_builtin_types ();
35815 /* Builtins to get CPU type and features. */
35816 ix86_init_platform_type_builtins ();
35818 /* TFmode support builtins. */
35819 def_builtin_const (0, "__builtin_infq",
35820 FLOAT128_FTYPE_VOID, IX86_BUILTIN_INFQ);
35821 def_builtin_const (0, "__builtin_huge_valq",
35822 FLOAT128_FTYPE_VOID, IX86_BUILTIN_HUGE_VALQ);
35824 /* We will expand them to normal call if SSE isn't available since
35825 they are used by libgcc. */
35826 t = ix86_get_builtin_func_type (FLOAT128_FTYPE_FLOAT128);
35827 t = add_builtin_function ("__builtin_fabsq", t, IX86_BUILTIN_FABSQ,
35828 BUILT_IN_MD, "__fabstf2", NULL_TREE);
35829 TREE_READONLY (t) = 1;
35830 ix86_builtins[(int) IX86_BUILTIN_FABSQ] = t;
35832 t = ix86_get_builtin_func_type (FLOAT128_FTYPE_FLOAT128_FLOAT128);
35833 t = add_builtin_function ("__builtin_copysignq", t, IX86_BUILTIN_COPYSIGNQ,
35834 BUILT_IN_MD, "__copysigntf3", NULL_TREE);
35835 TREE_READONLY (t) = 1;
35836 ix86_builtins[(int) IX86_BUILTIN_COPYSIGNQ] = t;
35838 ix86_init_tm_builtins ();
35839 ix86_init_mmx_sse_builtins ();
35840 ix86_init_mpx_builtins ();
35843 ix86_init_builtins_va_builtins_abi ();
35845 #ifdef SUBTARGET_INIT_BUILTINS
35846 SUBTARGET_INIT_BUILTINS;
35850 /* Return the ix86 builtin for CODE. */
35853 ix86_builtin_decl (unsigned code, bool)
35855 if (code >= IX86_BUILTIN_MAX)
35856 return error_mark_node;
35858 return ix86_builtins[code];
35861 /* Errors in the source file can cause expand_expr to return const0_rtx
35862 where we expect a vector. To avoid crashing, use one of the vector
35863 clear instructions. */
35865 safe_vector_operand (rtx x, machine_mode mode)
35867 if (x == const0_rtx)
35868 x = CONST0_RTX (mode);
35872 /* Fixup modeless constants to fit required mode. */
35874 fixup_modeless_constant (rtx x, machine_mode mode)
35876 if (GET_MODE (x) == VOIDmode)
35877 x = convert_to_mode (mode, x, 1);
35881 /* Subroutine of ix86_expand_builtin to take care of binop insns. */
35884 ix86_expand_binop_builtin (enum insn_code icode, tree exp, rtx target)
35887 tree arg0 = CALL_EXPR_ARG (exp, 0);
35888 tree arg1 = CALL_EXPR_ARG (exp, 1);
35889 rtx op0 = expand_normal (arg0);
35890 rtx op1 = expand_normal (arg1);
35891 machine_mode tmode = insn_data[icode].operand[0].mode;
35892 machine_mode mode0 = insn_data[icode].operand[1].mode;
35893 machine_mode mode1 = insn_data[icode].operand[2].mode;
35895 if (VECTOR_MODE_P (mode0))
35896 op0 = safe_vector_operand (op0, mode0);
35897 if (VECTOR_MODE_P (mode1))
35898 op1 = safe_vector_operand (op1, mode1);
35900 if (optimize || !target
35901 || GET_MODE (target) != tmode
35902 || !insn_data[icode].operand[0].predicate (target, tmode))
35903 target = gen_reg_rtx (tmode);
35905 if (GET_MODE (op1) == SImode && mode1 == TImode)
35907 rtx x = gen_reg_rtx (V4SImode);
35908 emit_insn (gen_sse2_loadd (x, op1));
35909 op1 = gen_lowpart (TImode, x);
35912 if (!insn_data[icode].operand[1].predicate (op0, mode0))
35913 op0 = copy_to_mode_reg (mode0, op0);
35914 if (!insn_data[icode].operand[2].predicate (op1, mode1))
35915 op1 = copy_to_mode_reg (mode1, op1);
35917 pat = GEN_FCN (icode) (target, op0, op1);
35926 /* Subroutine of ix86_expand_builtin to take care of 2-4 argument insns. */
35929 ix86_expand_multi_arg_builtin (enum insn_code icode, tree exp, rtx target,
35930 enum ix86_builtin_func_type m_type,
35931 enum rtx_code sub_code)
35936 bool comparison_p = false;
35938 bool last_arg_constant = false;
35939 int num_memory = 0;
35945 machine_mode tmode = insn_data[icode].operand[0].mode;
35949 case MULTI_ARG_4_DF2_DI_I:
35950 case MULTI_ARG_4_DF2_DI_I1:
35951 case MULTI_ARG_4_SF2_SI_I:
35952 case MULTI_ARG_4_SF2_SI_I1:
35954 last_arg_constant = true;
35957 case MULTI_ARG_3_SF:
35958 case MULTI_ARG_3_DF:
35959 case MULTI_ARG_3_SF2:
35960 case MULTI_ARG_3_DF2:
35961 case MULTI_ARG_3_DI:
35962 case MULTI_ARG_3_SI:
35963 case MULTI_ARG_3_SI_DI:
35964 case MULTI_ARG_3_HI:
35965 case MULTI_ARG_3_HI_SI:
35966 case MULTI_ARG_3_QI:
35967 case MULTI_ARG_3_DI2:
35968 case MULTI_ARG_3_SI2:
35969 case MULTI_ARG_3_HI2:
35970 case MULTI_ARG_3_QI2:
35974 case MULTI_ARG_2_SF:
35975 case MULTI_ARG_2_DF:
35976 case MULTI_ARG_2_DI:
35977 case MULTI_ARG_2_SI:
35978 case MULTI_ARG_2_HI:
35979 case MULTI_ARG_2_QI:
35983 case MULTI_ARG_2_DI_IMM:
35984 case MULTI_ARG_2_SI_IMM:
35985 case MULTI_ARG_2_HI_IMM:
35986 case MULTI_ARG_2_QI_IMM:
35988 last_arg_constant = true;
35991 case MULTI_ARG_1_SF:
35992 case MULTI_ARG_1_DF:
35993 case MULTI_ARG_1_SF2:
35994 case MULTI_ARG_1_DF2:
35995 case MULTI_ARG_1_DI:
35996 case MULTI_ARG_1_SI:
35997 case MULTI_ARG_1_HI:
35998 case MULTI_ARG_1_QI:
35999 case MULTI_ARG_1_SI_DI:
36000 case MULTI_ARG_1_HI_DI:
36001 case MULTI_ARG_1_HI_SI:
36002 case MULTI_ARG_1_QI_DI:
36003 case MULTI_ARG_1_QI_SI:
36004 case MULTI_ARG_1_QI_HI:
36008 case MULTI_ARG_2_DI_CMP:
36009 case MULTI_ARG_2_SI_CMP:
36010 case MULTI_ARG_2_HI_CMP:
36011 case MULTI_ARG_2_QI_CMP:
36013 comparison_p = true;
36016 case MULTI_ARG_2_SF_TF:
36017 case MULTI_ARG_2_DF_TF:
36018 case MULTI_ARG_2_DI_TF:
36019 case MULTI_ARG_2_SI_TF:
36020 case MULTI_ARG_2_HI_TF:
36021 case MULTI_ARG_2_QI_TF:
36027 gcc_unreachable ();
36030 if (optimize || !target
36031 || GET_MODE (target) != tmode
36032 || !insn_data[icode].operand[0].predicate (target, tmode))
36033 target = gen_reg_rtx (tmode);
36035 gcc_assert (nargs <= 4);
36037 for (i = 0; i < nargs; i++)
36039 tree arg = CALL_EXPR_ARG (exp, i);
36040 rtx op = expand_normal (arg);
36041 int adjust = (comparison_p) ? 1 : 0;
36042 machine_mode mode = insn_data[icode].operand[i+adjust+1].mode;
36044 if (last_arg_constant && i == nargs - 1)
36046 if (!insn_data[icode].operand[i + 1].predicate (op, mode))
36048 enum insn_code new_icode = icode;
36051 case CODE_FOR_xop_vpermil2v2df3:
36052 case CODE_FOR_xop_vpermil2v4sf3:
36053 case CODE_FOR_xop_vpermil2v4df3:
36054 case CODE_FOR_xop_vpermil2v8sf3:
36055 error ("the last argument must be a 2-bit immediate");
36056 return gen_reg_rtx (tmode);
36057 case CODE_FOR_xop_rotlv2di3:
36058 new_icode = CODE_FOR_rotlv2di3;
36060 case CODE_FOR_xop_rotlv4si3:
36061 new_icode = CODE_FOR_rotlv4si3;
36063 case CODE_FOR_xop_rotlv8hi3:
36064 new_icode = CODE_FOR_rotlv8hi3;
36066 case CODE_FOR_xop_rotlv16qi3:
36067 new_icode = CODE_FOR_rotlv16qi3;
36069 if (CONST_INT_P (op))
36071 int mask = GET_MODE_BITSIZE (GET_MODE_INNER (tmode)) - 1;
36072 op = GEN_INT (INTVAL (op) & mask);
36073 gcc_checking_assert
36074 (insn_data[icode].operand[i + 1].predicate (op, mode));
36078 gcc_checking_assert
36080 && insn_data[new_icode].operand[0].mode == tmode
36081 && insn_data[new_icode].operand[1].mode == tmode
36082 && insn_data[new_icode].operand[2].mode == mode
36083 && insn_data[new_icode].operand[0].predicate
36084 == insn_data[icode].operand[0].predicate
36085 && insn_data[new_icode].operand[1].predicate
36086 == insn_data[icode].operand[1].predicate);
36092 gcc_unreachable ();
36099 if (VECTOR_MODE_P (mode))
36100 op = safe_vector_operand (op, mode);
36102 /* If we aren't optimizing, only allow one memory operand to be
36104 if (memory_operand (op, mode))
36107 gcc_assert (GET_MODE (op) == mode || GET_MODE (op) == VOIDmode);
36110 || !insn_data[icode].operand[i+adjust+1].predicate (op, mode)
36112 op = force_reg (mode, op);
36116 args[i].mode = mode;
36122 pat = GEN_FCN (icode) (target, args[0].op);
36127 pat = GEN_FCN (icode) (target, args[0].op, args[1].op,
36128 GEN_INT ((int)sub_code));
36129 else if (! comparison_p)
36130 pat = GEN_FCN (icode) (target, args[0].op, args[1].op);
36133 rtx cmp_op = gen_rtx_fmt_ee (sub_code, GET_MODE (target),
36137 pat = GEN_FCN (icode) (target, cmp_op, args[0].op, args[1].op);
36142 pat = GEN_FCN (icode) (target, args[0].op, args[1].op, args[2].op);
36146 pat = GEN_FCN (icode) (target, args[0].op, args[1].op, args[2].op, args[3].op);
36150 gcc_unreachable ();
36160 /* Subroutine of ix86_expand_args_builtin to take care of scalar unop
36161 insns with vec_merge. */
36164 ix86_expand_unop_vec_merge_builtin (enum insn_code icode, tree exp,
36168 tree arg0 = CALL_EXPR_ARG (exp, 0);
36169 rtx op1, op0 = expand_normal (arg0);
36170 machine_mode tmode = insn_data[icode].operand[0].mode;
36171 machine_mode mode0 = insn_data[icode].operand[1].mode;
36173 if (optimize || !target
36174 || GET_MODE (target) != tmode
36175 || !insn_data[icode].operand[0].predicate (target, tmode))
36176 target = gen_reg_rtx (tmode);
36178 if (VECTOR_MODE_P (mode0))
36179 op0 = safe_vector_operand (op0, mode0);
36181 if ((optimize && !register_operand (op0, mode0))
36182 || !insn_data[icode].operand[1].predicate (op0, mode0))
36183 op0 = copy_to_mode_reg (mode0, op0);
36186 if (!insn_data[icode].operand[2].predicate (op1, mode0))
36187 op1 = copy_to_mode_reg (mode0, op1);
36189 pat = GEN_FCN (icode) (target, op0, op1);
36196 /* Subroutine of ix86_expand_builtin to take care of comparison insns. */
36199 ix86_expand_sse_compare (const struct builtin_description *d,
36200 tree exp, rtx target, bool swap)
36203 tree arg0 = CALL_EXPR_ARG (exp, 0);
36204 tree arg1 = CALL_EXPR_ARG (exp, 1);
36205 rtx op0 = expand_normal (arg0);
36206 rtx op1 = expand_normal (arg1);
36208 machine_mode tmode = insn_data[d->icode].operand[0].mode;
36209 machine_mode mode0 = insn_data[d->icode].operand[1].mode;
36210 machine_mode mode1 = insn_data[d->icode].operand[2].mode;
36211 enum rtx_code comparison = d->comparison;
36213 if (VECTOR_MODE_P (mode0))
36214 op0 = safe_vector_operand (op0, mode0);
36215 if (VECTOR_MODE_P (mode1))
36216 op1 = safe_vector_operand (op1, mode1);
36218 /* Swap operands if we have a comparison that isn't available in
36221 std::swap (op0, op1);
36223 if (optimize || !target
36224 || GET_MODE (target) != tmode
36225 || !insn_data[d->icode].operand[0].predicate (target, tmode))
36226 target = gen_reg_rtx (tmode);
36228 if ((optimize && !register_operand (op0, mode0))
36229 || !insn_data[d->icode].operand[1].predicate (op0, mode0))
36230 op0 = copy_to_mode_reg (mode0, op0);
36231 if ((optimize && !register_operand (op1, mode1))
36232 || !insn_data[d->icode].operand[2].predicate (op1, mode1))
36233 op1 = copy_to_mode_reg (mode1, op1);
36235 op2 = gen_rtx_fmt_ee (comparison, mode0, op0, op1);
36236 pat = GEN_FCN (d->icode) (target, op0, op1, op2);
36243 /* Subroutine of ix86_expand_builtin to take care of comi insns. */
36246 ix86_expand_sse_comi (const struct builtin_description *d, tree exp,
36250 tree arg0 = CALL_EXPR_ARG (exp, 0);
36251 tree arg1 = CALL_EXPR_ARG (exp, 1);
36252 rtx op0 = expand_normal (arg0);
36253 rtx op1 = expand_normal (arg1);
36254 machine_mode mode0 = insn_data[d->icode].operand[0].mode;
36255 machine_mode mode1 = insn_data[d->icode].operand[1].mode;
36256 enum rtx_code comparison = d->comparison;
36258 if (VECTOR_MODE_P (mode0))
36259 op0 = safe_vector_operand (op0, mode0);
36260 if (VECTOR_MODE_P (mode1))
36261 op1 = safe_vector_operand (op1, mode1);
36263 /* Swap operands if we have a comparison that isn't available in
36265 if (d->flag & BUILTIN_DESC_SWAP_OPERANDS)
36266 std::swap (op0, op1);
36268 target = gen_reg_rtx (SImode);
36269 emit_move_insn (target, const0_rtx);
36270 target = gen_rtx_SUBREG (QImode, target, 0);
36272 if ((optimize && !register_operand (op0, mode0))
36273 || !insn_data[d->icode].operand[0].predicate (op0, mode0))
36274 op0 = copy_to_mode_reg (mode0, op0);
36275 if ((optimize && !register_operand (op1, mode1))
36276 || !insn_data[d->icode].operand[1].predicate (op1, mode1))
36277 op1 = copy_to_mode_reg (mode1, op1);
36279 pat = GEN_FCN (d->icode) (op0, op1);
36283 emit_insn (gen_rtx_SET (gen_rtx_STRICT_LOW_PART (VOIDmode, target),
36284 gen_rtx_fmt_ee (comparison, QImode,
36288 return SUBREG_REG (target);
36291 /* Subroutines of ix86_expand_args_builtin to take care of round insns. */
36294 ix86_expand_sse_round (const struct builtin_description *d, tree exp,
36298 tree arg0 = CALL_EXPR_ARG (exp, 0);
36299 rtx op1, op0 = expand_normal (arg0);
36300 machine_mode tmode = insn_data[d->icode].operand[0].mode;
36301 machine_mode mode0 = insn_data[d->icode].operand[1].mode;
36303 if (optimize || target == 0
36304 || GET_MODE (target) != tmode
36305 || !insn_data[d->icode].operand[0].predicate (target, tmode))
36306 target = gen_reg_rtx (tmode);
36308 if (VECTOR_MODE_P (mode0))
36309 op0 = safe_vector_operand (op0, mode0);
36311 if ((optimize && !register_operand (op0, mode0))
36312 || !insn_data[d->icode].operand[0].predicate (op0, mode0))
36313 op0 = copy_to_mode_reg (mode0, op0);
36315 op1 = GEN_INT (d->comparison);
36317 pat = GEN_FCN (d->icode) (target, op0, op1);
36325 ix86_expand_sse_round_vec_pack_sfix (const struct builtin_description *d,
36326 tree exp, rtx target)
36329 tree arg0 = CALL_EXPR_ARG (exp, 0);
36330 tree arg1 = CALL_EXPR_ARG (exp, 1);
36331 rtx op0 = expand_normal (arg0);
36332 rtx op1 = expand_normal (arg1);
36334 machine_mode tmode = insn_data[d->icode].operand[0].mode;
36335 machine_mode mode0 = insn_data[d->icode].operand[1].mode;
36336 machine_mode mode1 = insn_data[d->icode].operand[2].mode;
36338 if (optimize || target == 0
36339 || GET_MODE (target) != tmode
36340 || !insn_data[d->icode].operand[0].predicate (target, tmode))
36341 target = gen_reg_rtx (tmode);
36343 op0 = safe_vector_operand (op0, mode0);
36344 op1 = safe_vector_operand (op1, mode1);
36346 if ((optimize && !register_operand (op0, mode0))
36347 || !insn_data[d->icode].operand[0].predicate (op0, mode0))
36348 op0 = copy_to_mode_reg (mode0, op0);
36349 if ((optimize && !register_operand (op1, mode1))
36350 || !insn_data[d->icode].operand[1].predicate (op1, mode1))
36351 op1 = copy_to_mode_reg (mode1, op1);
36353 op2 = GEN_INT (d->comparison);
36355 pat = GEN_FCN (d->icode) (target, op0, op1, op2);
36362 /* Subroutine of ix86_expand_builtin to take care of ptest insns. */
36365 ix86_expand_sse_ptest (const struct builtin_description *d, tree exp,
36369 tree arg0 = CALL_EXPR_ARG (exp, 0);
36370 tree arg1 = CALL_EXPR_ARG (exp, 1);
36371 rtx op0 = expand_normal (arg0);
36372 rtx op1 = expand_normal (arg1);
36373 machine_mode mode0 = insn_data[d->icode].operand[0].mode;
36374 machine_mode mode1 = insn_data[d->icode].operand[1].mode;
36375 enum rtx_code comparison = d->comparison;
36377 if (VECTOR_MODE_P (mode0))
36378 op0 = safe_vector_operand (op0, mode0);
36379 if (VECTOR_MODE_P (mode1))
36380 op1 = safe_vector_operand (op1, mode1);
36382 target = gen_reg_rtx (SImode);
36383 emit_move_insn (target, const0_rtx);
36384 target = gen_rtx_SUBREG (QImode, target, 0);
36386 if ((optimize && !register_operand (op0, mode0))
36387 || !insn_data[d->icode].operand[0].predicate (op0, mode0))
36388 op0 = copy_to_mode_reg (mode0, op0);
36389 if ((optimize && !register_operand (op1, mode1))
36390 || !insn_data[d->icode].operand[1].predicate (op1, mode1))
36391 op1 = copy_to_mode_reg (mode1, op1);
36393 pat = GEN_FCN (d->icode) (op0, op1);
36397 emit_insn (gen_rtx_SET (gen_rtx_STRICT_LOW_PART (VOIDmode, target),
36398 gen_rtx_fmt_ee (comparison, QImode,
36402 return SUBREG_REG (target);
36405 /* Subroutine of ix86_expand_builtin to take care of pcmpestr[im] insns. */
36408 ix86_expand_sse_pcmpestr (const struct builtin_description *d,
36409 tree exp, rtx target)
36412 tree arg0 = CALL_EXPR_ARG (exp, 0);
36413 tree arg1 = CALL_EXPR_ARG (exp, 1);
36414 tree arg2 = CALL_EXPR_ARG (exp, 2);
36415 tree arg3 = CALL_EXPR_ARG (exp, 3);
36416 tree arg4 = CALL_EXPR_ARG (exp, 4);
36417 rtx scratch0, scratch1;
36418 rtx op0 = expand_normal (arg0);
36419 rtx op1 = expand_normal (arg1);
36420 rtx op2 = expand_normal (arg2);
36421 rtx op3 = expand_normal (arg3);
36422 rtx op4 = expand_normal (arg4);
36423 machine_mode tmode0, tmode1, modev2, modei3, modev4, modei5, modeimm;
36425 tmode0 = insn_data[d->icode].operand[0].mode;
36426 tmode1 = insn_data[d->icode].operand[1].mode;
36427 modev2 = insn_data[d->icode].operand[2].mode;
36428 modei3 = insn_data[d->icode].operand[3].mode;
36429 modev4 = insn_data[d->icode].operand[4].mode;
36430 modei5 = insn_data[d->icode].operand[5].mode;
36431 modeimm = insn_data[d->icode].operand[6].mode;
36433 if (VECTOR_MODE_P (modev2))
36434 op0 = safe_vector_operand (op0, modev2);
36435 if (VECTOR_MODE_P (modev4))
36436 op2 = safe_vector_operand (op2, modev4);
36438 if (!insn_data[d->icode].operand[2].predicate (op0, modev2))
36439 op0 = copy_to_mode_reg (modev2, op0);
36440 if (!insn_data[d->icode].operand[3].predicate (op1, modei3))
36441 op1 = copy_to_mode_reg (modei3, op1);
36442 if ((optimize && !register_operand (op2, modev4))
36443 || !insn_data[d->icode].operand[4].predicate (op2, modev4))
36444 op2 = copy_to_mode_reg (modev4, op2);
36445 if (!insn_data[d->icode].operand[5].predicate (op3, modei5))
36446 op3 = copy_to_mode_reg (modei5, op3);
36448 if (!insn_data[d->icode].operand[6].predicate (op4, modeimm))
36450 error ("the fifth argument must be an 8-bit immediate");
36454 if (d->code == IX86_BUILTIN_PCMPESTRI128)
36456 if (optimize || !target
36457 || GET_MODE (target) != tmode0
36458 || !insn_data[d->icode].operand[0].predicate (target, tmode0))
36459 target = gen_reg_rtx (tmode0);
36461 scratch1 = gen_reg_rtx (tmode1);
36463 pat = GEN_FCN (d->icode) (target, scratch1, op0, op1, op2, op3, op4);
36465 else if (d->code == IX86_BUILTIN_PCMPESTRM128)
36467 if (optimize || !target
36468 || GET_MODE (target) != tmode1
36469 || !insn_data[d->icode].operand[1].predicate (target, tmode1))
36470 target = gen_reg_rtx (tmode1);
36472 scratch0 = gen_reg_rtx (tmode0);
36474 pat = GEN_FCN (d->icode) (scratch0, target, op0, op1, op2, op3, op4);
36478 gcc_assert (d->flag);
36480 scratch0 = gen_reg_rtx (tmode0);
36481 scratch1 = gen_reg_rtx (tmode1);
36483 pat = GEN_FCN (d->icode) (scratch0, scratch1, op0, op1, op2, op3, op4);
36493 target = gen_reg_rtx (SImode);
36494 emit_move_insn (target, const0_rtx);
36495 target = gen_rtx_SUBREG (QImode, target, 0);
36498 (gen_rtx_SET (gen_rtx_STRICT_LOW_PART (VOIDmode, target),
36499 gen_rtx_fmt_ee (EQ, QImode,
36500 gen_rtx_REG ((machine_mode) d->flag,
36503 return SUBREG_REG (target);
36510 /* Subroutine of ix86_expand_builtin to take care of pcmpistr[im] insns. */
36513 ix86_expand_sse_pcmpistr (const struct builtin_description *d,
36514 tree exp, rtx target)
36517 tree arg0 = CALL_EXPR_ARG (exp, 0);
36518 tree arg1 = CALL_EXPR_ARG (exp, 1);
36519 tree arg2 = CALL_EXPR_ARG (exp, 2);
36520 rtx scratch0, scratch1;
36521 rtx op0 = expand_normal (arg0);
36522 rtx op1 = expand_normal (arg1);
36523 rtx op2 = expand_normal (arg2);
36524 machine_mode tmode0, tmode1, modev2, modev3, modeimm;
36526 tmode0 = insn_data[d->icode].operand[0].mode;
36527 tmode1 = insn_data[d->icode].operand[1].mode;
36528 modev2 = insn_data[d->icode].operand[2].mode;
36529 modev3 = insn_data[d->icode].operand[3].mode;
36530 modeimm = insn_data[d->icode].operand[4].mode;
36532 if (VECTOR_MODE_P (modev2))
36533 op0 = safe_vector_operand (op0, modev2);
36534 if (VECTOR_MODE_P (modev3))
36535 op1 = safe_vector_operand (op1, modev3);
36537 if (!insn_data[d->icode].operand[2].predicate (op0, modev2))
36538 op0 = copy_to_mode_reg (modev2, op0);
36539 if ((optimize && !register_operand (op1, modev3))
36540 || !insn_data[d->icode].operand[3].predicate (op1, modev3))
36541 op1 = copy_to_mode_reg (modev3, op1);
36543 if (!insn_data[d->icode].operand[4].predicate (op2, modeimm))
36545 error ("the third argument must be an 8-bit immediate");
36549 if (d->code == IX86_BUILTIN_PCMPISTRI128)
36551 if (optimize || !target
36552 || GET_MODE (target) != tmode0
36553 || !insn_data[d->icode].operand[0].predicate (target, tmode0))
36554 target = gen_reg_rtx (tmode0);
36556 scratch1 = gen_reg_rtx (tmode1);
36558 pat = GEN_FCN (d->icode) (target, scratch1, op0, op1, op2);
36560 else if (d->code == IX86_BUILTIN_PCMPISTRM128)
36562 if (optimize || !target
36563 || GET_MODE (target) != tmode1
36564 || !insn_data[d->icode].operand[1].predicate (target, tmode1))
36565 target = gen_reg_rtx (tmode1);
36567 scratch0 = gen_reg_rtx (tmode0);
36569 pat = GEN_FCN (d->icode) (scratch0, target, op0, op1, op2);
36573 gcc_assert (d->flag);
36575 scratch0 = gen_reg_rtx (tmode0);
36576 scratch1 = gen_reg_rtx (tmode1);
36578 pat = GEN_FCN (d->icode) (scratch0, scratch1, op0, op1, op2);
36588 target = gen_reg_rtx (SImode);
36589 emit_move_insn (target, const0_rtx);
36590 target = gen_rtx_SUBREG (QImode, target, 0);
36593 (gen_rtx_SET (gen_rtx_STRICT_LOW_PART (VOIDmode, target),
36594 gen_rtx_fmt_ee (EQ, QImode,
36595 gen_rtx_REG ((machine_mode) d->flag,
36598 return SUBREG_REG (target);
36604 /* Subroutine of ix86_expand_builtin to take care of insns with
36605 variable number of operands. */
36608 ix86_expand_args_builtin (const struct builtin_description *d,
36609 tree exp, rtx target)
36611 rtx pat, real_target;
36612 unsigned int i, nargs;
36613 unsigned int nargs_constant = 0;
36614 unsigned int mask_pos = 0;
36615 int num_memory = 0;
36621 bool last_arg_count = false;
36622 enum insn_code icode = d->icode;
36623 const struct insn_data_d *insn_p = &insn_data[icode];
36624 machine_mode tmode = insn_p->operand[0].mode;
36625 machine_mode rmode = VOIDmode;
36627 enum rtx_code comparison = d->comparison;
36629 switch ((enum ix86_builtin_func_type) d->flag)
36631 case V2DF_FTYPE_V2DF_ROUND:
36632 case V4DF_FTYPE_V4DF_ROUND:
36633 case V4SF_FTYPE_V4SF_ROUND:
36634 case V8SF_FTYPE_V8SF_ROUND:
36635 case V4SI_FTYPE_V4SF_ROUND:
36636 case V8SI_FTYPE_V8SF_ROUND:
36637 return ix86_expand_sse_round (d, exp, target);
36638 case V4SI_FTYPE_V2DF_V2DF_ROUND:
36639 case V8SI_FTYPE_V4DF_V4DF_ROUND:
36640 case V16SI_FTYPE_V8DF_V8DF_ROUND:
36641 return ix86_expand_sse_round_vec_pack_sfix (d, exp, target);
36642 case INT_FTYPE_V8SF_V8SF_PTEST:
36643 case INT_FTYPE_V4DI_V4DI_PTEST:
36644 case INT_FTYPE_V4DF_V4DF_PTEST:
36645 case INT_FTYPE_V4SF_V4SF_PTEST:
36646 case INT_FTYPE_V2DI_V2DI_PTEST:
36647 case INT_FTYPE_V2DF_V2DF_PTEST:
36648 return ix86_expand_sse_ptest (d, exp, target);
36649 case FLOAT128_FTYPE_FLOAT128:
36650 case FLOAT_FTYPE_FLOAT:
36651 case INT_FTYPE_INT:
36652 case UINT64_FTYPE_INT:
36653 case UINT16_FTYPE_UINT16:
36654 case INT64_FTYPE_INT64:
36655 case INT64_FTYPE_V4SF:
36656 case INT64_FTYPE_V2DF:
36657 case INT_FTYPE_V16QI:
36658 case INT_FTYPE_V8QI:
36659 case INT_FTYPE_V8SF:
36660 case INT_FTYPE_V4DF:
36661 case INT_FTYPE_V4SF:
36662 case INT_FTYPE_V2DF:
36663 case INT_FTYPE_V32QI:
36664 case V16QI_FTYPE_V16QI:
36665 case V8SI_FTYPE_V8SF:
36666 case V8SI_FTYPE_V4SI:
36667 case V8HI_FTYPE_V8HI:
36668 case V8HI_FTYPE_V16QI:
36669 case V8QI_FTYPE_V8QI:
36670 case V8SF_FTYPE_V8SF:
36671 case V8SF_FTYPE_V8SI:
36672 case V8SF_FTYPE_V4SF:
36673 case V8SF_FTYPE_V8HI:
36674 case V4SI_FTYPE_V4SI:
36675 case V4SI_FTYPE_V16QI:
36676 case V4SI_FTYPE_V4SF:
36677 case V4SI_FTYPE_V8SI:
36678 case V4SI_FTYPE_V8HI:
36679 case V4SI_FTYPE_V4DF:
36680 case V4SI_FTYPE_V2DF:
36681 case V4HI_FTYPE_V4HI:
36682 case V4DF_FTYPE_V4DF:
36683 case V4DF_FTYPE_V4SI:
36684 case V4DF_FTYPE_V4SF:
36685 case V4DF_FTYPE_V2DF:
36686 case V4SF_FTYPE_V4SF:
36687 case V4SF_FTYPE_V4SI:
36688 case V4SF_FTYPE_V8SF:
36689 case V4SF_FTYPE_V4DF:
36690 case V4SF_FTYPE_V8HI:
36691 case V4SF_FTYPE_V2DF:
36692 case V2DI_FTYPE_V2DI:
36693 case V2DI_FTYPE_V16QI:
36694 case V2DI_FTYPE_V8HI:
36695 case V2DI_FTYPE_V4SI:
36696 case V2DF_FTYPE_V2DF:
36697 case V2DF_FTYPE_V4SI:
36698 case V2DF_FTYPE_V4DF:
36699 case V2DF_FTYPE_V4SF:
36700 case V2DF_FTYPE_V2SI:
36701 case V2SI_FTYPE_V2SI:
36702 case V2SI_FTYPE_V4SF:
36703 case V2SI_FTYPE_V2SF:
36704 case V2SI_FTYPE_V2DF:
36705 case V2SF_FTYPE_V2SF:
36706 case V2SF_FTYPE_V2SI:
36707 case V32QI_FTYPE_V32QI:
36708 case V32QI_FTYPE_V16QI:
36709 case V16HI_FTYPE_V16HI:
36710 case V16HI_FTYPE_V8HI:
36711 case V8SI_FTYPE_V8SI:
36712 case V16HI_FTYPE_V16QI:
36713 case V8SI_FTYPE_V16QI:
36714 case V4DI_FTYPE_V16QI:
36715 case V8SI_FTYPE_V8HI:
36716 case V4DI_FTYPE_V8HI:
36717 case V4DI_FTYPE_V4SI:
36718 case V4DI_FTYPE_V2DI:
36720 case HI_FTYPE_V16QI:
36721 case SI_FTYPE_V32QI:
36722 case DI_FTYPE_V64QI:
36723 case V16QI_FTYPE_HI:
36724 case V32QI_FTYPE_SI:
36725 case V64QI_FTYPE_DI:
36726 case V8HI_FTYPE_QI:
36727 case V16HI_FTYPE_HI:
36728 case V32HI_FTYPE_SI:
36729 case V4SI_FTYPE_QI:
36730 case V8SI_FTYPE_QI:
36731 case V4SI_FTYPE_HI:
36732 case V8SI_FTYPE_HI:
36733 case QI_FTYPE_V8HI:
36734 case HI_FTYPE_V16HI:
36735 case SI_FTYPE_V32HI:
36736 case QI_FTYPE_V4SI:
36737 case QI_FTYPE_V8SI:
36738 case HI_FTYPE_V16SI:
36739 case QI_FTYPE_V2DI:
36740 case QI_FTYPE_V4DI:
36741 case QI_FTYPE_V8DI:
36742 case UINT_FTYPE_V2DF:
36743 case UINT_FTYPE_V4SF:
36744 case UINT64_FTYPE_V2DF:
36745 case UINT64_FTYPE_V4SF:
36746 case V16QI_FTYPE_V8DI:
36747 case V16HI_FTYPE_V16SI:
36748 case V16SI_FTYPE_HI:
36749 case V2DI_FTYPE_QI:
36750 case V4DI_FTYPE_QI:
36751 case V16SI_FTYPE_V16SI:
36752 case V16SI_FTYPE_INT:
36753 case V16SF_FTYPE_FLOAT:
36754 case V16SF_FTYPE_V8SF:
36755 case V16SI_FTYPE_V8SI:
36756 case V16SF_FTYPE_V4SF:
36757 case V16SI_FTYPE_V4SI:
36758 case V16SF_FTYPE_V16SF:
36759 case V8HI_FTYPE_V8DI:
36760 case V8UHI_FTYPE_V8UHI:
36761 case V8SI_FTYPE_V8DI:
36762 case V8SF_FTYPE_V8DF:
36763 case V8DI_FTYPE_QI:
36764 case V8DI_FTYPE_INT64:
36765 case V8DI_FTYPE_V4DI:
36766 case V8DI_FTYPE_V8DI:
36767 case V8DF_FTYPE_DOUBLE:
36768 case V8DF_FTYPE_V4DF:
36769 case V8DF_FTYPE_V2DF:
36770 case V8DF_FTYPE_V8DF:
36771 case V8DF_FTYPE_V8SI:
36774 case V4SF_FTYPE_V4SF_VEC_MERGE:
36775 case V2DF_FTYPE_V2DF_VEC_MERGE:
36776 return ix86_expand_unop_vec_merge_builtin (icode, exp, target);
36777 case FLOAT128_FTYPE_FLOAT128_FLOAT128:
36778 case V16QI_FTYPE_V16QI_V16QI:
36779 case V16QI_FTYPE_V8HI_V8HI:
36780 case V16SI_FTYPE_V16SI_V16SI:
36781 case V16SF_FTYPE_V16SF_V16SF:
36782 case V16SF_FTYPE_V16SF_V16SI:
36783 case V8QI_FTYPE_V8QI_V8QI:
36784 case V8QI_FTYPE_V4HI_V4HI:
36785 case V8HI_FTYPE_V8HI_V8HI:
36786 case V8HI_FTYPE_V16QI_V16QI:
36787 case V8HI_FTYPE_V4SI_V4SI:
36788 case V8SF_FTYPE_V8SF_V8SF:
36789 case V8SF_FTYPE_V8SF_V8SI:
36790 case V8DI_FTYPE_V8DI_V8DI:
36791 case V8DF_FTYPE_V8DF_V8DF:
36792 case V8DF_FTYPE_V8DF_V8DI:
36793 case V4SI_FTYPE_V4SI_V4SI:
36794 case V4SI_FTYPE_V8HI_V8HI:
36795 case V4SI_FTYPE_V4SF_V4SF:
36796 case V4SI_FTYPE_V2DF_V2DF:
36797 case V4HI_FTYPE_V4HI_V4HI:
36798 case V4HI_FTYPE_V8QI_V8QI:
36799 case V4HI_FTYPE_V2SI_V2SI:
36800 case V4DF_FTYPE_V4DF_V4DF:
36801 case V4DF_FTYPE_V4DF_V4DI:
36802 case V4SF_FTYPE_V4SF_V4SF:
36803 case V4SF_FTYPE_V4SF_V4SI:
36804 case V4SF_FTYPE_V4SF_V2SI:
36805 case V4SF_FTYPE_V4SF_V2DF:
36806 case V4SF_FTYPE_V4SF_UINT:
36807 case V4SF_FTYPE_V4SF_UINT64:
36808 case V4SF_FTYPE_V4SF_DI:
36809 case V4SF_FTYPE_V4SF_SI:
36810 case V2DI_FTYPE_V2DI_V2DI:
36811 case V2DI_FTYPE_V16QI_V16QI:
36812 case V2DI_FTYPE_V4SI_V4SI:
36813 case V2UDI_FTYPE_V4USI_V4USI:
36814 case V2DI_FTYPE_V2DI_V16QI:
36815 case V2DI_FTYPE_V2DF_V2DF:
36816 case V2SI_FTYPE_V2SI_V2SI:
36817 case V2SI_FTYPE_V4HI_V4HI:
36818 case V2SI_FTYPE_V2SF_V2SF:
36819 case V2DF_FTYPE_V2DF_V2DF:
36820 case V2DF_FTYPE_V2DF_V4SF:
36821 case V2DF_FTYPE_V2DF_V2DI:
36822 case V2DF_FTYPE_V2DF_DI:
36823 case V2DF_FTYPE_V2DF_SI:
36824 case V2DF_FTYPE_V2DF_UINT:
36825 case V2DF_FTYPE_V2DF_UINT64:
36826 case V2SF_FTYPE_V2SF_V2SF:
36827 case V1DI_FTYPE_V1DI_V1DI:
36828 case V1DI_FTYPE_V8QI_V8QI:
36829 case V1DI_FTYPE_V2SI_V2SI:
36830 case V32QI_FTYPE_V16HI_V16HI:
36831 case V16HI_FTYPE_V8SI_V8SI:
36832 case V32QI_FTYPE_V32QI_V32QI:
36833 case V16HI_FTYPE_V32QI_V32QI:
36834 case V16HI_FTYPE_V16HI_V16HI:
36835 case V8SI_FTYPE_V4DF_V4DF:
36836 case V8SI_FTYPE_V8SI_V8SI:
36837 case V8SI_FTYPE_V16HI_V16HI:
36838 case V4DI_FTYPE_V4DI_V4DI:
36839 case V4DI_FTYPE_V8SI_V8SI:
36840 case V4UDI_FTYPE_V8USI_V8USI:
36841 case QI_FTYPE_V8DI_V8DI:
36842 case V8DI_FTYPE_V64QI_V64QI:
36843 case HI_FTYPE_V16SI_V16SI:
36844 if (comparison == UNKNOWN)
36845 return ix86_expand_binop_builtin (icode, exp, target);
36848 case V4SF_FTYPE_V4SF_V4SF_SWAP:
36849 case V2DF_FTYPE_V2DF_V2DF_SWAP:
36850 gcc_assert (comparison != UNKNOWN);
36854 case V16HI_FTYPE_V16HI_V8HI_COUNT:
36855 case V16HI_FTYPE_V16HI_SI_COUNT:
36856 case V8SI_FTYPE_V8SI_V4SI_COUNT:
36857 case V8SI_FTYPE_V8SI_SI_COUNT:
36858 case V4DI_FTYPE_V4DI_V2DI_COUNT:
36859 case V4DI_FTYPE_V4DI_INT_COUNT:
36860 case V8HI_FTYPE_V8HI_V8HI_COUNT:
36861 case V8HI_FTYPE_V8HI_SI_COUNT:
36862 case V4SI_FTYPE_V4SI_V4SI_COUNT:
36863 case V4SI_FTYPE_V4SI_SI_COUNT:
36864 case V4HI_FTYPE_V4HI_V4HI_COUNT:
36865 case V4HI_FTYPE_V4HI_SI_COUNT:
36866 case V2DI_FTYPE_V2DI_V2DI_COUNT:
36867 case V2DI_FTYPE_V2DI_SI_COUNT:
36868 case V2SI_FTYPE_V2SI_V2SI_COUNT:
36869 case V2SI_FTYPE_V2SI_SI_COUNT:
36870 case V1DI_FTYPE_V1DI_V1DI_COUNT:
36871 case V1DI_FTYPE_V1DI_SI_COUNT:
36873 last_arg_count = true;
36875 case UINT64_FTYPE_UINT64_UINT64:
36876 case UINT_FTYPE_UINT_UINT:
36877 case UINT_FTYPE_UINT_USHORT:
36878 case UINT_FTYPE_UINT_UCHAR:
36879 case UINT16_FTYPE_UINT16_INT:
36880 case UINT8_FTYPE_UINT8_INT:
36881 case HI_FTYPE_HI_HI:
36882 case SI_FTYPE_SI_SI:
36883 case DI_FTYPE_DI_DI:
36884 case V16SI_FTYPE_V8DF_V8DF:
36887 case V2DI_FTYPE_V2DI_INT_CONVERT:
36890 nargs_constant = 1;
36892 case V4DI_FTYPE_V4DI_INT_CONVERT:
36895 nargs_constant = 1;
36897 case V8DI_FTYPE_V8DI_INT_CONVERT:
36900 nargs_constant = 1;
36902 case V8HI_FTYPE_V8HI_INT:
36903 case V8HI_FTYPE_V8SF_INT:
36904 case V16HI_FTYPE_V16SF_INT:
36905 case V8HI_FTYPE_V4SF_INT:
36906 case V8SF_FTYPE_V8SF_INT:
36907 case V4SF_FTYPE_V16SF_INT:
36908 case V16SF_FTYPE_V16SF_INT:
36909 case V4SI_FTYPE_V4SI_INT:
36910 case V4SI_FTYPE_V8SI_INT:
36911 case V4HI_FTYPE_V4HI_INT:
36912 case V4DF_FTYPE_V4DF_INT:
36913 case V4DF_FTYPE_V8DF_INT:
36914 case V4SF_FTYPE_V4SF_INT:
36915 case V4SF_FTYPE_V8SF_INT:
36916 case V2DI_FTYPE_V2DI_INT:
36917 case V2DF_FTYPE_V2DF_INT:
36918 case V2DF_FTYPE_V4DF_INT:
36919 case V16HI_FTYPE_V16HI_INT:
36920 case V8SI_FTYPE_V8SI_INT:
36921 case V16SI_FTYPE_V16SI_INT:
36922 case V4SI_FTYPE_V16SI_INT:
36923 case V4DI_FTYPE_V4DI_INT:
36924 case V2DI_FTYPE_V4DI_INT:
36925 case V4DI_FTYPE_V8DI_INT:
36926 case HI_FTYPE_HI_INT:
36927 case QI_FTYPE_V4SF_INT:
36928 case QI_FTYPE_V2DF_INT:
36930 nargs_constant = 1;
36932 case V16QI_FTYPE_V16QI_V16QI_V16QI:
36933 case V8SF_FTYPE_V8SF_V8SF_V8SF:
36934 case V4DF_FTYPE_V4DF_V4DF_V4DF:
36935 case V4SF_FTYPE_V4SF_V4SF_V4SF:
36936 case V2DF_FTYPE_V2DF_V2DF_V2DF:
36937 case V32QI_FTYPE_V32QI_V32QI_V32QI:
36938 case HI_FTYPE_V16SI_V16SI_HI:
36939 case QI_FTYPE_V8DI_V8DI_QI:
36940 case V16HI_FTYPE_V16SI_V16HI_HI:
36941 case V16QI_FTYPE_V16SI_V16QI_HI:
36942 case V16QI_FTYPE_V8DI_V16QI_QI:
36943 case V16SF_FTYPE_V16SF_V16SF_HI:
36944 case V16SF_FTYPE_V16SF_V16SF_V16SF:
36945 case V16SF_FTYPE_V16SF_V16SI_V16SF:
36946 case V16SF_FTYPE_V16SI_V16SF_HI:
36947 case V16SF_FTYPE_V16SI_V16SF_V16SF:
36948 case V16SF_FTYPE_V4SF_V16SF_HI:
36949 case V16SI_FTYPE_SI_V16SI_HI:
36950 case V16SI_FTYPE_V16HI_V16SI_HI:
36951 case V16SI_FTYPE_V16QI_V16SI_HI:
36952 case V16SI_FTYPE_V16SF_V16SI_HI:
36953 case V8SF_FTYPE_V4SF_V8SF_QI:
36954 case V4DF_FTYPE_V2DF_V4DF_QI:
36955 case V8SI_FTYPE_V4SI_V8SI_QI:
36956 case V8SI_FTYPE_SI_V8SI_QI:
36957 case V4SI_FTYPE_V4SI_V4SI_QI:
36958 case V4SI_FTYPE_SI_V4SI_QI:
36959 case V4DI_FTYPE_V2DI_V4DI_QI:
36960 case V4DI_FTYPE_DI_V4DI_QI:
36961 case V2DI_FTYPE_V2DI_V2DI_QI:
36962 case V2DI_FTYPE_DI_V2DI_QI:
36963 case V64QI_FTYPE_V64QI_V64QI_DI:
36964 case V64QI_FTYPE_V16QI_V64QI_DI:
36965 case V64QI_FTYPE_QI_V64QI_DI:
36966 case V32QI_FTYPE_V32QI_V32QI_SI:
36967 case V32QI_FTYPE_V16QI_V32QI_SI:
36968 case V32QI_FTYPE_QI_V32QI_SI:
36969 case V16QI_FTYPE_V16QI_V16QI_HI:
36970 case V16QI_FTYPE_QI_V16QI_HI:
36971 case V32HI_FTYPE_V8HI_V32HI_SI:
36972 case V32HI_FTYPE_HI_V32HI_SI:
36973 case V16HI_FTYPE_V8HI_V16HI_HI:
36974 case V16HI_FTYPE_HI_V16HI_HI:
36975 case V8HI_FTYPE_V8HI_V8HI_QI:
36976 case V8HI_FTYPE_HI_V8HI_QI:
36977 case V8SF_FTYPE_V8HI_V8SF_QI:
36978 case V4SF_FTYPE_V8HI_V4SF_QI:
36979 case V8SI_FTYPE_V8SF_V8SI_QI:
36980 case V4SI_FTYPE_V4SF_V4SI_QI:
36981 case V8DI_FTYPE_V8SF_V8DI_QI:
36982 case V4DI_FTYPE_V4SF_V4DI_QI:
36983 case V2DI_FTYPE_V4SF_V2DI_QI:
36984 case V8SF_FTYPE_V8DI_V8SF_QI:
36985 case V4SF_FTYPE_V4DI_V4SF_QI:
36986 case V4SF_FTYPE_V2DI_V4SF_QI:
36987 case V8DF_FTYPE_V8DI_V8DF_QI:
36988 case V4DF_FTYPE_V4DI_V4DF_QI:
36989 case V2DF_FTYPE_V2DI_V2DF_QI:
36990 case V16QI_FTYPE_V8HI_V16QI_QI:
36991 case V16QI_FTYPE_V16HI_V16QI_HI:
36992 case V16QI_FTYPE_V4SI_V16QI_QI:
36993 case V16QI_FTYPE_V8SI_V16QI_QI:
36994 case V8HI_FTYPE_V4SI_V8HI_QI:
36995 case V8HI_FTYPE_V8SI_V8HI_QI:
36996 case V16QI_FTYPE_V2DI_V16QI_QI:
36997 case V16QI_FTYPE_V4DI_V16QI_QI:
36998 case V8HI_FTYPE_V2DI_V8HI_QI:
36999 case V8HI_FTYPE_V4DI_V8HI_QI:
37000 case V4SI_FTYPE_V2DI_V4SI_QI:
37001 case V4SI_FTYPE_V4DI_V4SI_QI:
37002 case V32QI_FTYPE_V32HI_V32QI_SI:
37003 case HI_FTYPE_V16QI_V16QI_HI:
37004 case SI_FTYPE_V32QI_V32QI_SI:
37005 case DI_FTYPE_V64QI_V64QI_DI:
37006 case QI_FTYPE_V8HI_V8HI_QI:
37007 case HI_FTYPE_V16HI_V16HI_HI:
37008 case SI_FTYPE_V32HI_V32HI_SI:
37009 case QI_FTYPE_V4SI_V4SI_QI:
37010 case QI_FTYPE_V8SI_V8SI_QI:
37011 case QI_FTYPE_V2DI_V2DI_QI:
37012 case QI_FTYPE_V4DI_V4DI_QI:
37013 case V4SF_FTYPE_V2DF_V4SF_QI:
37014 case V4SF_FTYPE_V4DF_V4SF_QI:
37015 case V16SI_FTYPE_V16SI_V16SI_HI:
37016 case V16SI_FTYPE_V16SI_V16SI_V16SI:
37017 case V16SI_FTYPE_V4SI_V16SI_HI:
37018 case V2DI_FTYPE_V2DI_V2DI_V2DI:
37019 case V2DI_FTYPE_V4SI_V2DI_QI:
37020 case V2DI_FTYPE_V8HI_V2DI_QI:
37021 case V2DI_FTYPE_V16QI_V2DI_QI:
37022 case V4DI_FTYPE_V4DI_V4DI_QI:
37023 case V4DI_FTYPE_V4SI_V4DI_QI:
37024 case V4DI_FTYPE_V8HI_V4DI_QI:
37025 case V4DI_FTYPE_V16QI_V4DI_QI:
37026 case V8DI_FTYPE_V8DF_V8DI_QI:
37027 case V4DI_FTYPE_V4DF_V4DI_QI:
37028 case V2DI_FTYPE_V2DF_V2DI_QI:
37029 case V4SI_FTYPE_V4DF_V4SI_QI:
37030 case V4SI_FTYPE_V2DF_V4SI_QI:
37031 case V4SI_FTYPE_V8HI_V4SI_QI:
37032 case V4SI_FTYPE_V16QI_V4SI_QI:
37033 case V8SI_FTYPE_V8SI_V8SI_V8SI:
37034 case V4DI_FTYPE_V4DI_V4DI_V4DI:
37035 case V8DF_FTYPE_V2DF_V8DF_QI:
37036 case V8DF_FTYPE_V4DF_V8DF_QI:
37037 case V8DF_FTYPE_V8DF_V8DF_QI:
37038 case V8DF_FTYPE_V8DF_V8DF_V8DF:
37039 case V8SF_FTYPE_V8SF_V8SF_QI:
37040 case V8SF_FTYPE_V8SI_V8SF_QI:
37041 case V4DF_FTYPE_V4DF_V4DF_QI:
37042 case V4SF_FTYPE_V4SF_V4SF_QI:
37043 case V2DF_FTYPE_V2DF_V2DF_QI:
37044 case V2DF_FTYPE_V4SF_V2DF_QI:
37045 case V2DF_FTYPE_V4SI_V2DF_QI:
37046 case V4SF_FTYPE_V4SI_V4SF_QI:
37047 case V4DF_FTYPE_V4SF_V4DF_QI:
37048 case V4DF_FTYPE_V4SI_V4DF_QI:
37049 case V8SI_FTYPE_V8SI_V8SI_QI:
37050 case V8SI_FTYPE_V8HI_V8SI_QI:
37051 case V8SI_FTYPE_V16QI_V8SI_QI:
37052 case V8DF_FTYPE_V8DF_V8DI_V8DF:
37053 case V8DF_FTYPE_V8DI_V8DF_V8DF:
37054 case V8DF_FTYPE_V8SF_V8DF_QI:
37055 case V8DF_FTYPE_V8SI_V8DF_QI:
37056 case V8DI_FTYPE_DI_V8DI_QI:
37057 case V16SF_FTYPE_V8SF_V16SF_HI:
37058 case V16SI_FTYPE_V8SI_V16SI_HI:
37059 case V16HI_FTYPE_V16HI_V16HI_HI:
37060 case V8HI_FTYPE_V16QI_V8HI_QI:
37061 case V16HI_FTYPE_V16QI_V16HI_HI:
37062 case V32HI_FTYPE_V32HI_V32HI_SI:
37063 case V32HI_FTYPE_V32QI_V32HI_SI:
37064 case V8DI_FTYPE_V16QI_V8DI_QI:
37065 case V8DI_FTYPE_V2DI_V8DI_QI:
37066 case V8DI_FTYPE_V4DI_V8DI_QI:
37067 case V8DI_FTYPE_V8DI_V8DI_QI:
37068 case V8DI_FTYPE_V8DI_V8DI_V8DI:
37069 case V8DI_FTYPE_V8HI_V8DI_QI:
37070 case V8DI_FTYPE_V8SI_V8DI_QI:
37071 case V8HI_FTYPE_V8DI_V8HI_QI:
37072 case V8SF_FTYPE_V8DF_V8SF_QI:
37073 case V8SI_FTYPE_V8DF_V8SI_QI:
37074 case V8SI_FTYPE_V8DI_V8SI_QI:
37075 case V4SI_FTYPE_V4SI_V4SI_V4SI:
37078 case V32QI_FTYPE_V32QI_V32QI_INT:
37079 case V16HI_FTYPE_V16HI_V16HI_INT:
37080 case V16QI_FTYPE_V16QI_V16QI_INT:
37081 case V4DI_FTYPE_V4DI_V4DI_INT:
37082 case V8HI_FTYPE_V8HI_V8HI_INT:
37083 case V8SI_FTYPE_V8SI_V8SI_INT:
37084 case V8SI_FTYPE_V8SI_V4SI_INT:
37085 case V8SF_FTYPE_V8SF_V8SF_INT:
37086 case V8SF_FTYPE_V8SF_V4SF_INT:
37087 case V4SI_FTYPE_V4SI_V4SI_INT:
37088 case V4DF_FTYPE_V4DF_V4DF_INT:
37089 case V16SF_FTYPE_V16SF_V16SF_INT:
37090 case V16SF_FTYPE_V16SF_V4SF_INT:
37091 case V16SI_FTYPE_V16SI_V4SI_INT:
37092 case V4DF_FTYPE_V4DF_V2DF_INT:
37093 case V4SF_FTYPE_V4SF_V4SF_INT:
37094 case V2DI_FTYPE_V2DI_V2DI_INT:
37095 case V4DI_FTYPE_V4DI_V2DI_INT:
37096 case V2DF_FTYPE_V2DF_V2DF_INT:
37097 case QI_FTYPE_V8DI_V8DI_INT:
37098 case QI_FTYPE_V8DF_V8DF_INT:
37099 case QI_FTYPE_V2DF_V2DF_INT:
37100 case QI_FTYPE_V4SF_V4SF_INT:
37101 case HI_FTYPE_V16SI_V16SI_INT:
37102 case HI_FTYPE_V16SF_V16SF_INT:
37104 nargs_constant = 1;
37106 case V4DI_FTYPE_V4DI_V4DI_INT_CONVERT:
37109 nargs_constant = 1;
37111 case V2DI_FTYPE_V2DI_V2DI_INT_CONVERT:
37114 nargs_constant = 1;
37116 case V1DI_FTYPE_V1DI_V1DI_INT_CONVERT:
37119 nargs_constant = 1;
37121 case V2DI_FTYPE_V2DI_UINT_UINT:
37123 nargs_constant = 2;
37125 case V8DI_FTYPE_V8DI_V8DI_INT_CONVERT:
37128 nargs_constant = 1;
37130 case V8DI_FTYPE_V8DI_V8DI_INT_V8DI_DI_CONVERT:
37134 nargs_constant = 1;
37136 case QI_FTYPE_V8DF_INT_QI:
37137 case QI_FTYPE_V4DF_INT_QI:
37138 case QI_FTYPE_V2DF_INT_QI:
37139 case HI_FTYPE_V16SF_INT_HI:
37140 case QI_FTYPE_V8SF_INT_QI:
37141 case QI_FTYPE_V4SF_INT_QI:
37144 nargs_constant = 1;
37146 case V4DI_FTYPE_V4DI_V4DI_INT_V4DI_SI_CONVERT:
37150 nargs_constant = 1;
37152 case V2DI_FTYPE_V2DI_V2DI_INT_V2DI_HI_CONVERT:
37156 nargs_constant = 1;
37158 case V32QI_FTYPE_V32QI_V32QI_V32QI_SI:
37159 case V32HI_FTYPE_V32HI_V32HI_V32HI_SI:
37160 case V32HI_FTYPE_V64QI_V64QI_V32HI_SI:
37161 case V16SI_FTYPE_V32HI_V32HI_V16SI_HI:
37162 case V64QI_FTYPE_V64QI_V64QI_V64QI_DI:
37163 case V32HI_FTYPE_V32HI_V8HI_V32HI_SI:
37164 case V16HI_FTYPE_V16HI_V8HI_V16HI_HI:
37165 case V8SI_FTYPE_V8SI_V4SI_V8SI_QI:
37166 case V4DI_FTYPE_V4DI_V2DI_V4DI_QI:
37167 case V64QI_FTYPE_V32HI_V32HI_V64QI_DI:
37168 case V32QI_FTYPE_V16HI_V16HI_V32QI_SI:
37169 case V16QI_FTYPE_V8HI_V8HI_V16QI_HI:
37170 case V32HI_FTYPE_V16SI_V16SI_V32HI_SI:
37171 case V16HI_FTYPE_V8SI_V8SI_V16HI_HI:
37172 case V8HI_FTYPE_V4SI_V4SI_V8HI_QI:
37173 case V4DF_FTYPE_V4DF_V4DI_V4DF_QI:
37174 case V8SF_FTYPE_V8SF_V8SI_V8SF_QI:
37175 case V4SF_FTYPE_V4SF_V4SI_V4SF_QI:
37176 case V2DF_FTYPE_V2DF_V2DI_V2DF_QI:
37177 case V2DI_FTYPE_V4SI_V4SI_V2DI_QI:
37178 case V4DI_FTYPE_V8SI_V8SI_V4DI_QI:
37179 case V4DF_FTYPE_V4DI_V4DF_V4DF_QI:
37180 case V8SF_FTYPE_V8SI_V8SF_V8SF_QI:
37181 case V2DF_FTYPE_V2DI_V2DF_V2DF_QI:
37182 case V4SF_FTYPE_V4SI_V4SF_V4SF_QI:
37183 case V16SF_FTYPE_V16SF_V16SF_V16SF_HI:
37184 case V16SF_FTYPE_V16SF_V16SI_V16SF_HI:
37185 case V16SF_FTYPE_V16SI_V16SF_V16SF_HI:
37186 case V16SI_FTYPE_V16SI_V16SI_V16SI_HI:
37187 case V16SI_FTYPE_V16SI_V4SI_V16SI_HI:
37188 case V8HI_FTYPE_V8HI_V8HI_V8HI_QI:
37189 case V8SI_FTYPE_V8SI_V8SI_V8SI_QI:
37190 case V4SI_FTYPE_V4SI_V4SI_V4SI_QI:
37191 case V8SF_FTYPE_V8SF_V8SF_V8SF_QI:
37192 case V16QI_FTYPE_V16QI_V16QI_V16QI_HI:
37193 case V16HI_FTYPE_V16HI_V16HI_V16HI_HI:
37194 case V2DI_FTYPE_V2DI_V2DI_V2DI_QI:
37195 case V2DF_FTYPE_V2DF_V2DF_V2DF_QI:
37196 case V2DF_FTYPE_V2DF_V4SF_V2DF_QI:
37197 case V4DI_FTYPE_V4DI_V4DI_V4DI_QI:
37198 case V4DF_FTYPE_V4DF_V4DF_V4DF_QI:
37199 case V4SF_FTYPE_V4SF_V2DF_V4SF_QI:
37200 case V4SF_FTYPE_V4SF_V4SF_V4SF_QI:
37201 case V8DF_FTYPE_V8DF_V8DF_V8DF_QI:
37202 case V8DF_FTYPE_V8DF_V8DI_V8DF_QI:
37203 case V8DF_FTYPE_V8DI_V8DF_V8DF_QI:
37204 case V8DI_FTYPE_V16SI_V16SI_V8DI_QI:
37205 case V8DI_FTYPE_V8DI_SI_V8DI_V8DI:
37206 case V8DI_FTYPE_V8DI_V2DI_V8DI_QI:
37207 case V8DI_FTYPE_V8DI_V8DI_V8DI_QI:
37208 case V8HI_FTYPE_V16QI_V16QI_V8HI_QI:
37209 case V16HI_FTYPE_V32QI_V32QI_V16HI_HI:
37210 case V8SI_FTYPE_V16HI_V16HI_V8SI_QI:
37211 case V4SI_FTYPE_V8HI_V8HI_V4SI_QI:
37214 case V2DF_FTYPE_V2DF_V2DF_V2DI_INT:
37215 case V4DF_FTYPE_V4DF_V4DF_V4DI_INT:
37216 case V4SF_FTYPE_V4SF_V4SF_V4SI_INT:
37217 case V8SF_FTYPE_V8SF_V8SF_V8SI_INT:
37218 case V16SF_FTYPE_V16SF_V16SF_V16SI_INT:
37220 nargs_constant = 1;
37222 case QI_FTYPE_V4DI_V4DI_INT_QI:
37223 case QI_FTYPE_V8SI_V8SI_INT_QI:
37224 case QI_FTYPE_V4DF_V4DF_INT_QI:
37225 case QI_FTYPE_V8SF_V8SF_INT_QI:
37226 case QI_FTYPE_V2DI_V2DI_INT_QI:
37227 case QI_FTYPE_V4SI_V4SI_INT_QI:
37228 case QI_FTYPE_V2DF_V2DF_INT_QI:
37229 case QI_FTYPE_V4SF_V4SF_INT_QI:
37230 case DI_FTYPE_V64QI_V64QI_INT_DI:
37231 case SI_FTYPE_V32QI_V32QI_INT_SI:
37232 case HI_FTYPE_V16QI_V16QI_INT_HI:
37233 case SI_FTYPE_V32HI_V32HI_INT_SI:
37234 case HI_FTYPE_V16HI_V16HI_INT_HI:
37235 case QI_FTYPE_V8HI_V8HI_INT_QI:
37238 nargs_constant = 1;
37240 case V2DI_FTYPE_V2DI_V2DI_UINT_UINT:
37242 nargs_constant = 2;
37244 case UCHAR_FTYPE_UCHAR_UINT_UINT_PUNSIGNED:
37245 case UCHAR_FTYPE_UCHAR_ULONGLONG_ULONGLONG_PULONGLONG:
37248 case QI_FTYPE_V8DI_V8DI_INT_QI:
37249 case HI_FTYPE_V16SI_V16SI_INT_HI:
37250 case QI_FTYPE_V8DF_V8DF_INT_QI:
37251 case HI_FTYPE_V16SF_V16SF_INT_HI:
37254 nargs_constant = 1;
37256 case V8SF_FTYPE_V8SF_INT_V8SF_QI:
37257 case V4SF_FTYPE_V4SF_INT_V4SF_QI:
37258 case V2DF_FTYPE_V4DF_INT_V2DF_QI:
37259 case V2DI_FTYPE_V4DI_INT_V2DI_QI:
37260 case V8SF_FTYPE_V16SF_INT_V8SF_QI:
37261 case V8SI_FTYPE_V16SI_INT_V8SI_QI:
37262 case V2DF_FTYPE_V8DF_INT_V2DF_QI:
37263 case V2DI_FTYPE_V8DI_INT_V2DI_QI:
37264 case V4SF_FTYPE_V8SF_INT_V4SF_QI:
37265 case V4SI_FTYPE_V8SI_INT_V4SI_QI:
37266 case V8HI_FTYPE_V8SF_INT_V8HI_QI:
37267 case V8HI_FTYPE_V4SF_INT_V8HI_QI:
37268 case V32HI_FTYPE_V32HI_INT_V32HI_SI:
37269 case V16HI_FTYPE_V16HI_INT_V16HI_HI:
37270 case V8HI_FTYPE_V8HI_INT_V8HI_QI:
37271 case V4DI_FTYPE_V4DI_INT_V4DI_QI:
37272 case V2DI_FTYPE_V2DI_INT_V2DI_QI:
37273 case V8SI_FTYPE_V8SI_INT_V8SI_QI:
37274 case V4SI_FTYPE_V4SI_INT_V4SI_QI:
37275 case V4DF_FTYPE_V4DF_INT_V4DF_QI:
37276 case V2DF_FTYPE_V2DF_INT_V2DF_QI:
37277 case V8DF_FTYPE_V8DF_INT_V8DF_QI:
37278 case V16SF_FTYPE_V16SF_INT_V16SF_HI:
37279 case V16HI_FTYPE_V16SF_INT_V16HI_HI:
37280 case V16SI_FTYPE_V16SI_INT_V16SI_HI:
37281 case V4SI_FTYPE_V16SI_INT_V4SI_QI:
37282 case V4DI_FTYPE_V8DI_INT_V4DI_QI:
37283 case V4DF_FTYPE_V8DF_INT_V4DF_QI:
37284 case V4SF_FTYPE_V16SF_INT_V4SF_QI:
37285 case V8DI_FTYPE_V8DI_INT_V8DI_QI:
37288 nargs_constant = 1;
37290 case V16SF_FTYPE_V16SF_V4SF_INT_V16SF_HI:
37291 case V16SI_FTYPE_V16SI_V4SI_INT_V16SI_HI:
37292 case V8DF_FTYPE_V8DF_V8DF_INT_V8DF_QI:
37293 case V8DI_FTYPE_V8DI_V8DI_INT_V8DI_QI:
37294 case V16SF_FTYPE_V16SF_V16SF_INT_V16SF_HI:
37295 case V16SI_FTYPE_V16SI_V16SI_INT_V16SI_HI:
37296 case V4SF_FTYPE_V4SF_V4SF_INT_V4SF_QI:
37297 case V2DF_FTYPE_V2DF_V2DF_INT_V2DF_QI:
37298 case V8DF_FTYPE_V8DF_V4DF_INT_V8DF_QI:
37299 case V8DI_FTYPE_V8DI_V4DI_INT_V8DI_QI:
37300 case V4DF_FTYPE_V4DF_V4DF_INT_V4DF_QI:
37301 case V8SF_FTYPE_V8SF_V8SF_INT_V8SF_QI:
37302 case V8DF_FTYPE_V8DF_V2DF_INT_V8DF_QI:
37303 case V8DI_FTYPE_V8DI_V2DI_INT_V8DI_QI:
37304 case V8SI_FTYPE_V8SI_V8SI_INT_V8SI_QI:
37305 case V4DI_FTYPE_V4DI_V4DI_INT_V4DI_QI:
37306 case V4SI_FTYPE_V4SI_V4SI_INT_V4SI_QI:
37307 case V2DI_FTYPE_V2DI_V2DI_INT_V2DI_QI:
37308 case V32HI_FTYPE_V64QI_V64QI_INT_V32HI_SI:
37309 case V16HI_FTYPE_V32QI_V32QI_INT_V16HI_HI:
37310 case V8HI_FTYPE_V16QI_V16QI_INT_V8HI_QI:
37311 case V16SF_FTYPE_V16SF_V8SF_INT_V16SF_HI:
37312 case V16SI_FTYPE_V16SI_V8SI_INT_V16SI_HI:
37313 case V8SF_FTYPE_V8SF_V4SF_INT_V8SF_QI:
37314 case V8SI_FTYPE_V8SI_V4SI_INT_V8SI_QI:
37315 case V4DI_FTYPE_V4DI_V2DI_INT_V4DI_QI:
37316 case V4DF_FTYPE_V4DF_V2DF_INT_V4DF_QI:
37319 nargs_constant = 1;
37321 case V8DI_FTYPE_V8DI_V8DI_V8DI_INT_QI:
37322 case V16SF_FTYPE_V16SF_V16SF_V16SI_INT_HI:
37323 case V16SI_FTYPE_V16SI_V16SI_V16SI_INT_HI:
37324 case V2DF_FTYPE_V2DF_V2DF_V2DI_INT_QI:
37325 case V4SF_FTYPE_V4SF_V4SF_V4SI_INT_QI:
37326 case V8SF_FTYPE_V8SF_V8SF_V8SI_INT_QI:
37327 case V8SI_FTYPE_V8SI_V8SI_V8SI_INT_QI:
37328 case V4DF_FTYPE_V4DF_V4DF_V4DI_INT_QI:
37329 case V4DI_FTYPE_V4DI_V4DI_V4DI_INT_QI:
37330 case V4SI_FTYPE_V4SI_V4SI_V4SI_INT_QI:
37331 case V2DI_FTYPE_V2DI_V2DI_V2DI_INT_QI:
37335 nargs_constant = 1;
37339 gcc_unreachable ();
37342 gcc_assert (nargs <= ARRAY_SIZE (args));
37344 if (comparison != UNKNOWN)
37346 gcc_assert (nargs == 2);
37347 return ix86_expand_sse_compare (d, exp, target, swap);
37350 if (rmode == VOIDmode || rmode == tmode)
37354 || GET_MODE (target) != tmode
37355 || !insn_p->operand[0].predicate (target, tmode))
37356 target = gen_reg_rtx (tmode);
37357 real_target = target;
37361 real_target = gen_reg_rtx (tmode);
37362 target = simplify_gen_subreg (rmode, real_target, tmode, 0);
37365 for (i = 0; i < nargs; i++)
37367 tree arg = CALL_EXPR_ARG (exp, i);
37368 rtx op = expand_normal (arg);
37369 machine_mode mode = insn_p->operand[i + 1].mode;
37370 bool match = insn_p->operand[i + 1].predicate (op, mode);
37372 if (last_arg_count && (i + 1) == nargs)
37374 /* SIMD shift insns take either an 8-bit immediate or
37375 register as count. But builtin functions take int as
37376 count. If count doesn't match, we put it in register. */
37379 op = simplify_gen_subreg (SImode, op, GET_MODE (op), 0);
37380 if (!insn_p->operand[i + 1].predicate (op, mode))
37381 op = copy_to_reg (op);
37384 else if ((mask_pos && (nargs - i - mask_pos) == nargs_constant) ||
37385 (!mask_pos && (nargs - i) <= nargs_constant))
37390 case CODE_FOR_avx_vinsertf128v4di:
37391 case CODE_FOR_avx_vextractf128v4di:
37392 error ("the last argument must be an 1-bit immediate");
37395 case CODE_FOR_avx512f_cmpv8di3_mask:
37396 case CODE_FOR_avx512f_cmpv16si3_mask:
37397 case CODE_FOR_avx512f_ucmpv8di3_mask:
37398 case CODE_FOR_avx512f_ucmpv16si3_mask:
37399 case CODE_FOR_avx512vl_cmpv4di3_mask:
37400 case CODE_FOR_avx512vl_cmpv8si3_mask:
37401 case CODE_FOR_avx512vl_ucmpv4di3_mask:
37402 case CODE_FOR_avx512vl_ucmpv8si3_mask:
37403 case CODE_FOR_avx512vl_cmpv2di3_mask:
37404 case CODE_FOR_avx512vl_cmpv4si3_mask:
37405 case CODE_FOR_avx512vl_ucmpv2di3_mask:
37406 case CODE_FOR_avx512vl_ucmpv4si3_mask:
37407 error ("the last argument must be a 3-bit immediate");
37410 case CODE_FOR_sse4_1_roundsd:
37411 case CODE_FOR_sse4_1_roundss:
37413 case CODE_FOR_sse4_1_roundpd:
37414 case CODE_FOR_sse4_1_roundps:
37415 case CODE_FOR_avx_roundpd256:
37416 case CODE_FOR_avx_roundps256:
37418 case CODE_FOR_sse4_1_roundpd_vec_pack_sfix:
37419 case CODE_FOR_sse4_1_roundps_sfix:
37420 case CODE_FOR_avx_roundpd_vec_pack_sfix256:
37421 case CODE_FOR_avx_roundps_sfix256:
37423 case CODE_FOR_sse4_1_blendps:
37424 case CODE_FOR_avx_blendpd256:
37425 case CODE_FOR_avx_vpermilv4df:
37426 case CODE_FOR_avx_vpermilv4df_mask:
37427 case CODE_FOR_avx512f_getmantv8df_mask:
37428 case CODE_FOR_avx512f_getmantv16sf_mask:
37429 case CODE_FOR_avx512vl_getmantv8sf_mask:
37430 case CODE_FOR_avx512vl_getmantv4df_mask:
37431 case CODE_FOR_avx512vl_getmantv4sf_mask:
37432 case CODE_FOR_avx512vl_getmantv2df_mask:
37433 case CODE_FOR_avx512dq_rangepv8df_mask_round:
37434 case CODE_FOR_avx512dq_rangepv16sf_mask_round:
37435 case CODE_FOR_avx512dq_rangepv4df_mask:
37436 case CODE_FOR_avx512dq_rangepv8sf_mask:
37437 case CODE_FOR_avx512dq_rangepv2df_mask:
37438 case CODE_FOR_avx512dq_rangepv4sf_mask:
37439 case CODE_FOR_avx_shufpd256_mask:
37440 error ("the last argument must be a 4-bit immediate");
37443 case CODE_FOR_sha1rnds4:
37444 case CODE_FOR_sse4_1_blendpd:
37445 case CODE_FOR_avx_vpermilv2df:
37446 case CODE_FOR_avx_vpermilv2df_mask:
37447 case CODE_FOR_xop_vpermil2v2df3:
37448 case CODE_FOR_xop_vpermil2v4sf3:
37449 case CODE_FOR_xop_vpermil2v4df3:
37450 case CODE_FOR_xop_vpermil2v8sf3:
37451 case CODE_FOR_avx512f_vinsertf32x4_mask:
37452 case CODE_FOR_avx512f_vinserti32x4_mask:
37453 case CODE_FOR_avx512f_vextractf32x4_mask:
37454 case CODE_FOR_avx512f_vextracti32x4_mask:
37455 case CODE_FOR_sse2_shufpd:
37456 case CODE_FOR_sse2_shufpd_mask:
37457 case CODE_FOR_avx512dq_shuf_f64x2_mask:
37458 case CODE_FOR_avx512dq_shuf_i64x2_mask:
37459 case CODE_FOR_avx512vl_shuf_i32x4_mask:
37460 case CODE_FOR_avx512vl_shuf_f32x4_mask:
37461 error ("the last argument must be a 2-bit immediate");
37464 case CODE_FOR_avx_vextractf128v4df:
37465 case CODE_FOR_avx_vextractf128v8sf:
37466 case CODE_FOR_avx_vextractf128v8si:
37467 case CODE_FOR_avx_vinsertf128v4df:
37468 case CODE_FOR_avx_vinsertf128v8sf:
37469 case CODE_FOR_avx_vinsertf128v8si:
37470 case CODE_FOR_avx512f_vinsertf64x4_mask:
37471 case CODE_FOR_avx512f_vinserti64x4_mask:
37472 case CODE_FOR_avx512f_vextractf64x4_mask:
37473 case CODE_FOR_avx512f_vextracti64x4_mask:
37474 case CODE_FOR_avx512dq_vinsertf32x8_mask:
37475 case CODE_FOR_avx512dq_vinserti32x8_mask:
37476 case CODE_FOR_avx512vl_vinsertv4df:
37477 case CODE_FOR_avx512vl_vinsertv4di:
37478 case CODE_FOR_avx512vl_vinsertv8sf:
37479 case CODE_FOR_avx512vl_vinsertv8si:
37480 error ("the last argument must be a 1-bit immediate");
37483 case CODE_FOR_avx_vmcmpv2df3:
37484 case CODE_FOR_avx_vmcmpv4sf3:
37485 case CODE_FOR_avx_cmpv2df3:
37486 case CODE_FOR_avx_cmpv4sf3:
37487 case CODE_FOR_avx_cmpv4df3:
37488 case CODE_FOR_avx_cmpv8sf3:
37489 case CODE_FOR_avx512f_cmpv8df3_mask:
37490 case CODE_FOR_avx512f_cmpv16sf3_mask:
37491 case CODE_FOR_avx512f_vmcmpv2df3_mask:
37492 case CODE_FOR_avx512f_vmcmpv4sf3_mask:
37493 error ("the last argument must be a 5-bit immediate");
37497 switch (nargs_constant)
37500 if ((mask_pos && (nargs - i - mask_pos) == nargs_constant) ||
37501 (!mask_pos && (nargs - i) == nargs_constant))
37503 error ("the next to last argument must be an 8-bit immediate");
37507 error ("the last argument must be an 8-bit immediate");
37510 gcc_unreachable ();
37517 if (VECTOR_MODE_P (mode))
37518 op = safe_vector_operand (op, mode);
37520 /* If we aren't optimizing, only allow one memory operand to
37522 if (memory_operand (op, mode))
37525 op = fixup_modeless_constant (op, mode);
37527 if (GET_MODE (op) == mode || GET_MODE (op) == VOIDmode)
37529 if (optimize || !match || num_memory > 1)
37530 op = copy_to_mode_reg (mode, op);
37534 op = copy_to_reg (op);
37535 op = simplify_gen_subreg (mode, op, GET_MODE (op), 0);
37540 args[i].mode = mode;
37546 pat = GEN_FCN (icode) (real_target, args[0].op);
37549 pat = GEN_FCN (icode) (real_target, args[0].op, args[1].op);
37552 pat = GEN_FCN (icode) (real_target, args[0].op, args[1].op,
37556 pat = GEN_FCN (icode) (real_target, args[0].op, args[1].op,
37557 args[2].op, args[3].op);
37560 pat = GEN_FCN (icode) (real_target, args[0].op, args[1].op,
37561 args[2].op, args[3].op, args[4].op);
37563 pat = GEN_FCN (icode) (real_target, args[0].op, args[1].op,
37564 args[2].op, args[3].op, args[4].op,
37568 gcc_unreachable ();
37578 /* Transform pattern of following layout:
37581 (unspec [C] UNSPEC_EMBEDDED_ROUNDING)])
37589 (unspec [C] UNSPEC_EMBEDDED_ROUNDING)
37593 (parallel [ A B ... ]) */
37596 ix86_erase_embedded_rounding (rtx pat)
37598 if (GET_CODE (pat) == INSN)
37599 pat = PATTERN (pat);
37601 gcc_assert (GET_CODE (pat) == PARALLEL);
37603 if (XVECLEN (pat, 0) == 2)
37605 rtx p0 = XVECEXP (pat, 0, 0);
37606 rtx p1 = XVECEXP (pat, 0, 1);
37608 gcc_assert (GET_CODE (p0) == SET
37609 && GET_CODE (p1) == UNSPEC
37610 && XINT (p1, 1) == UNSPEC_EMBEDDED_ROUNDING);
37616 rtx *res = XALLOCAVEC (rtx, XVECLEN (pat, 0));
37620 for (; i < XVECLEN (pat, 0); ++i)
37622 rtx elem = XVECEXP (pat, 0, i);
37623 if (GET_CODE (elem) != UNSPEC
37624 || XINT (elem, 1) != UNSPEC_EMBEDDED_ROUNDING)
37628 /* No more than 1 occurence was removed. */
37629 gcc_assert (j >= XVECLEN (pat, 0) - 1);
37631 return gen_rtx_PARALLEL (GET_MODE (pat), gen_rtvec_v (j, res));
37635 /* Subroutine of ix86_expand_round_builtin to take care of comi insns
37638 ix86_expand_sse_comi_round (const struct builtin_description *d,
37639 tree exp, rtx target)
37642 tree arg0 = CALL_EXPR_ARG (exp, 0);
37643 tree arg1 = CALL_EXPR_ARG (exp, 1);
37644 tree arg2 = CALL_EXPR_ARG (exp, 2);
37645 tree arg3 = CALL_EXPR_ARG (exp, 3);
37646 rtx op0 = expand_normal (arg0);
37647 rtx op1 = expand_normal (arg1);
37648 rtx op2 = expand_normal (arg2);
37649 rtx op3 = expand_normal (arg3);
37650 enum insn_code icode = d->icode;
37651 const struct insn_data_d *insn_p = &insn_data[icode];
37652 machine_mode mode0 = insn_p->operand[0].mode;
37653 machine_mode mode1 = insn_p->operand[1].mode;
37654 enum rtx_code comparison = UNEQ;
37655 bool need_ucomi = false;
37657 /* See avxintrin.h for values. */
37658 enum rtx_code comi_comparisons[32] =
37660 UNEQ, GT, GE, UNORDERED, LTGT, UNLE, UNLT, ORDERED, UNEQ, UNLT,
37661 UNLE, LT, LTGT, GE, GT, LT, UNEQ, GT, GE, UNORDERED, LTGT, UNLE,
37662 UNLT, ORDERED, UNEQ, UNLT, UNLE, LT, LTGT, GE, GT, LT
37664 bool need_ucomi_values[32] =
37666 true, false, false, true, true, false, false, true,
37667 true, false, false, true, true, false, false, true,
37668 false, true, true, false, false, true, true, false,
37669 false, true, true, false, false, true, true, false
37672 if (!CONST_INT_P (op2))
37674 error ("the third argument must be comparison constant");
37677 if (INTVAL (op2) < 0 || INTVAL (op2) >= 32)
37679 error ("incorrect comparison mode");
37683 if (!insn_p->operand[2].predicate (op3, SImode))
37685 error ("incorrect rounding operand");
37689 comparison = comi_comparisons[INTVAL (op2)];
37690 need_ucomi = need_ucomi_values[INTVAL (op2)];
37692 if (VECTOR_MODE_P (mode0))
37693 op0 = safe_vector_operand (op0, mode0);
37694 if (VECTOR_MODE_P (mode1))
37695 op1 = safe_vector_operand (op1, mode1);
37697 target = gen_reg_rtx (SImode);
37698 emit_move_insn (target, const0_rtx);
37699 target = gen_rtx_SUBREG (QImode, target, 0);
37701 if ((optimize && !register_operand (op0, mode0))
37702 || !insn_p->operand[0].predicate (op0, mode0))
37703 op0 = copy_to_mode_reg (mode0, op0);
37704 if ((optimize && !register_operand (op1, mode1))
37705 || !insn_p->operand[1].predicate (op1, mode1))
37706 op1 = copy_to_mode_reg (mode1, op1);
37709 icode = icode == CODE_FOR_sse_comi_round
37710 ? CODE_FOR_sse_ucomi_round
37711 : CODE_FOR_sse2_ucomi_round;
37713 pat = GEN_FCN (icode) (op0, op1, op3);
37717 /* Rounding operand can be either NO_ROUND or ROUND_SAE at this point. */
37718 if (INTVAL (op3) == NO_ROUND)
37720 pat = ix86_erase_embedded_rounding (pat);
37724 set_dst = SET_DEST (pat);
37728 gcc_assert (GET_CODE (XVECEXP (pat, 0, 0)) == SET);
37729 set_dst = SET_DEST (XVECEXP (pat, 0, 0));
37733 emit_insn (gen_rtx_SET (gen_rtx_STRICT_LOW_PART (VOIDmode, target),
37734 gen_rtx_fmt_ee (comparison, QImode,
37738 return SUBREG_REG (target);
37742 ix86_expand_round_builtin (const struct builtin_description *d,
37743 tree exp, rtx target)
37746 unsigned int i, nargs;
37752 enum insn_code icode = d->icode;
37753 const struct insn_data_d *insn_p = &insn_data[icode];
37754 machine_mode tmode = insn_p->operand[0].mode;
37755 unsigned int nargs_constant = 0;
37756 unsigned int redundant_embed_rnd = 0;
37758 switch ((enum ix86_builtin_func_type) d->flag)
37760 case UINT64_FTYPE_V2DF_INT:
37761 case UINT64_FTYPE_V4SF_INT:
37762 case UINT_FTYPE_V2DF_INT:
37763 case UINT_FTYPE_V4SF_INT:
37764 case INT64_FTYPE_V2DF_INT:
37765 case INT64_FTYPE_V4SF_INT:
37766 case INT_FTYPE_V2DF_INT:
37767 case INT_FTYPE_V4SF_INT:
37770 case V4SF_FTYPE_V4SF_UINT_INT:
37771 case V4SF_FTYPE_V4SF_UINT64_INT:
37772 case V2DF_FTYPE_V2DF_UINT64_INT:
37773 case V4SF_FTYPE_V4SF_INT_INT:
37774 case V4SF_FTYPE_V4SF_INT64_INT:
37775 case V2DF_FTYPE_V2DF_INT64_INT:
37776 case V4SF_FTYPE_V4SF_V4SF_INT:
37777 case V2DF_FTYPE_V2DF_V2DF_INT:
37778 case V4SF_FTYPE_V4SF_V2DF_INT:
37779 case V2DF_FTYPE_V2DF_V4SF_INT:
37782 case V8SF_FTYPE_V8DF_V8SF_QI_INT:
37783 case V8DF_FTYPE_V8DF_V8DF_QI_INT:
37784 case V8SI_FTYPE_V8DF_V8SI_QI_INT:
37785 case V8DI_FTYPE_V8DF_V8DI_QI_INT:
37786 case V8SF_FTYPE_V8DI_V8SF_QI_INT:
37787 case V8DF_FTYPE_V8DI_V8DF_QI_INT:
37788 case V16SF_FTYPE_V16SF_V16SF_HI_INT:
37789 case V8DI_FTYPE_V8SF_V8DI_QI_INT:
37790 case V16SF_FTYPE_V16SI_V16SF_HI_INT:
37791 case V16SI_FTYPE_V16SF_V16SI_HI_INT:
37792 case V8DF_FTYPE_V8SF_V8DF_QI_INT:
37793 case V16SF_FTYPE_V16HI_V16SF_HI_INT:
37794 case V2DF_FTYPE_V2DF_V2DF_V2DF_INT:
37795 case V4SF_FTYPE_V4SF_V4SF_V4SF_INT:
37798 case V4SF_FTYPE_V4SF_V4SF_INT_INT:
37799 case V2DF_FTYPE_V2DF_V2DF_INT_INT:
37800 nargs_constant = 2;
37803 case INT_FTYPE_V4SF_V4SF_INT_INT:
37804 case INT_FTYPE_V2DF_V2DF_INT_INT:
37805 return ix86_expand_sse_comi_round (d, exp, target);
37806 case V8DF_FTYPE_V8DF_V8DF_V8DF_QI_INT:
37807 case V16SF_FTYPE_V16SF_V16SF_V16SF_HI_INT:
37808 case V2DF_FTYPE_V2DF_V2DF_V2DF_QI_INT:
37809 case V2DF_FTYPE_V2DF_V4SF_V2DF_QI_INT:
37810 case V4SF_FTYPE_V4SF_V4SF_V4SF_QI_INT:
37811 case V4SF_FTYPE_V4SF_V2DF_V4SF_QI_INT:
37814 case V16SF_FTYPE_V16SF_INT_V16SF_HI_INT:
37815 case V8DF_FTYPE_V8DF_INT_V8DF_QI_INT:
37816 nargs_constant = 4;
37819 case QI_FTYPE_V8DF_V8DF_INT_QI_INT:
37820 case QI_FTYPE_V2DF_V2DF_INT_QI_INT:
37821 case HI_FTYPE_V16SF_V16SF_INT_HI_INT:
37822 case QI_FTYPE_V4SF_V4SF_INT_QI_INT:
37823 nargs_constant = 3;
37826 case V16SF_FTYPE_V16SF_V16SF_INT_V16SF_HI_INT:
37827 case V8DF_FTYPE_V8DF_V8DF_INT_V8DF_QI_INT:
37828 case V4SF_FTYPE_V4SF_V4SF_INT_V4SF_QI_INT:
37829 case V2DF_FTYPE_V2DF_V2DF_INT_V2DF_QI_INT:
37831 nargs_constant = 4;
37833 case V8DF_FTYPE_V8DF_V8DF_V8DI_INT_QI_INT:
37834 case V16SF_FTYPE_V16SF_V16SF_V16SI_INT_HI_INT:
37835 case V2DF_FTYPE_V2DF_V2DF_V2DI_INT_QI_INT:
37836 case V4SF_FTYPE_V4SF_V4SF_V4SI_INT_QI_INT:
37838 nargs_constant = 3;
37841 gcc_unreachable ();
37843 gcc_assert (nargs <= ARRAY_SIZE (args));
37847 || GET_MODE (target) != tmode
37848 || !insn_p->operand[0].predicate (target, tmode))
37849 target = gen_reg_rtx (tmode);
37851 for (i = 0; i < nargs; i++)
37853 tree arg = CALL_EXPR_ARG (exp, i);
37854 rtx op = expand_normal (arg);
37855 machine_mode mode = insn_p->operand[i + 1].mode;
37856 bool match = insn_p->operand[i + 1].predicate (op, mode);
37858 if (i == nargs - nargs_constant)
37864 case CODE_FOR_avx512f_getmantv8df_mask_round:
37865 case CODE_FOR_avx512f_getmantv16sf_mask_round:
37866 case CODE_FOR_avx512f_vgetmantv2df_round:
37867 case CODE_FOR_avx512f_vgetmantv4sf_round:
37868 error ("the immediate argument must be a 4-bit immediate");
37870 case CODE_FOR_avx512f_cmpv8df3_mask_round:
37871 case CODE_FOR_avx512f_cmpv16sf3_mask_round:
37872 case CODE_FOR_avx512f_vmcmpv2df3_mask_round:
37873 case CODE_FOR_avx512f_vmcmpv4sf3_mask_round:
37874 error ("the immediate argument must be a 5-bit immediate");
37877 error ("the immediate argument must be an 8-bit immediate");
37882 else if (i == nargs-1)
37884 if (!insn_p->operand[nargs].predicate (op, SImode))
37886 error ("incorrect rounding operand");
37890 /* If there is no rounding use normal version of the pattern. */
37891 if (INTVAL (op) == NO_ROUND)
37892 redundant_embed_rnd = 1;
37896 if (VECTOR_MODE_P (mode))
37897 op = safe_vector_operand (op, mode);
37899 op = fixup_modeless_constant (op, mode);
37901 if (GET_MODE (op) == mode || GET_MODE (op) == VOIDmode)
37903 if (optimize || !match)
37904 op = copy_to_mode_reg (mode, op);
37908 op = copy_to_reg (op);
37909 op = simplify_gen_subreg (mode, op, GET_MODE (op), 0);
37914 args[i].mode = mode;
37920 pat = GEN_FCN (icode) (target, args[0].op);
37923 pat = GEN_FCN (icode) (target, args[0].op, args[1].op);
37926 pat = GEN_FCN (icode) (target, args[0].op, args[1].op,
37930 pat = GEN_FCN (icode) (target, args[0].op, args[1].op,
37931 args[2].op, args[3].op);
37934 pat = GEN_FCN (icode) (target, args[0].op, args[1].op,
37935 args[2].op, args[3].op, args[4].op);
37937 pat = GEN_FCN (icode) (target, args[0].op, args[1].op,
37938 args[2].op, args[3].op, args[4].op,
37942 gcc_unreachable ();
37948 if (redundant_embed_rnd)
37949 pat = ix86_erase_embedded_rounding (pat);
37955 /* Subroutine of ix86_expand_builtin to take care of special insns
37956 with variable number of operands. */
37959 ix86_expand_special_args_builtin (const struct builtin_description *d,
37960 tree exp, rtx target)
37964 unsigned int i, nargs, arg_adjust, memory;
37965 bool aligned_mem = false;
37971 enum insn_code icode = d->icode;
37972 bool last_arg_constant = false;
37973 const struct insn_data_d *insn_p = &insn_data[icode];
37974 machine_mode tmode = insn_p->operand[0].mode;
37975 enum { load, store } klass;
37977 switch ((enum ix86_builtin_func_type) d->flag)
37979 case VOID_FTYPE_VOID:
37980 emit_insn (GEN_FCN (icode) (target));
37982 case VOID_FTYPE_UINT64:
37983 case VOID_FTYPE_UNSIGNED:
37989 case INT_FTYPE_VOID:
37990 case USHORT_FTYPE_VOID:
37991 case UINT64_FTYPE_VOID:
37992 case UNSIGNED_FTYPE_VOID:
37997 case UINT64_FTYPE_PUNSIGNED:
37998 case V2DI_FTYPE_PV2DI:
37999 case V4DI_FTYPE_PV4DI:
38000 case V32QI_FTYPE_PCCHAR:
38001 case V16QI_FTYPE_PCCHAR:
38002 case V8SF_FTYPE_PCV4SF:
38003 case V8SF_FTYPE_PCFLOAT:
38004 case V4SF_FTYPE_PCFLOAT:
38005 case V4DF_FTYPE_PCV2DF:
38006 case V4DF_FTYPE_PCDOUBLE:
38007 case V2DF_FTYPE_PCDOUBLE:
38008 case VOID_FTYPE_PVOID:
38009 case V16SI_FTYPE_PV4SI:
38010 case V16SF_FTYPE_PV4SF:
38011 case V8DI_FTYPE_PV4DI:
38012 case V8DI_FTYPE_PV8DI:
38013 case V8DF_FTYPE_PV4DF:
38019 case CODE_FOR_sse4_1_movntdqa:
38020 case CODE_FOR_avx2_movntdqa:
38021 case CODE_FOR_avx512f_movntdqa:
38022 aligned_mem = true;
38028 case VOID_FTYPE_PV2SF_V4SF:
38029 case VOID_FTYPE_PV8DI_V8DI:
38030 case VOID_FTYPE_PV4DI_V4DI:
38031 case VOID_FTYPE_PV2DI_V2DI:
38032 case VOID_FTYPE_PCHAR_V32QI:
38033 case VOID_FTYPE_PCHAR_V16QI:
38034 case VOID_FTYPE_PFLOAT_V16SF:
38035 case VOID_FTYPE_PFLOAT_V8SF:
38036 case VOID_FTYPE_PFLOAT_V4SF:
38037 case VOID_FTYPE_PDOUBLE_V8DF:
38038 case VOID_FTYPE_PDOUBLE_V4DF:
38039 case VOID_FTYPE_PDOUBLE_V2DF:
38040 case VOID_FTYPE_PLONGLONG_LONGLONG:
38041 case VOID_FTYPE_PULONGLONG_ULONGLONG:
38042 case VOID_FTYPE_PINT_INT:
38045 /* Reserve memory operand for target. */
38046 memory = ARRAY_SIZE (args);
38049 /* These builtins and instructions require the memory
38050 to be properly aligned. */
38051 case CODE_FOR_avx_movntv4di:
38052 case CODE_FOR_sse2_movntv2di:
38053 case CODE_FOR_avx_movntv8sf:
38054 case CODE_FOR_sse_movntv4sf:
38055 case CODE_FOR_sse4a_vmmovntv4sf:
38056 case CODE_FOR_avx_movntv4df:
38057 case CODE_FOR_sse2_movntv2df:
38058 case CODE_FOR_sse4a_vmmovntv2df:
38059 case CODE_FOR_sse2_movntidi:
38060 case CODE_FOR_sse_movntq:
38061 case CODE_FOR_sse2_movntisi:
38062 case CODE_FOR_avx512f_movntv16sf:
38063 case CODE_FOR_avx512f_movntv8df:
38064 case CODE_FOR_avx512f_movntv8di:
38065 aligned_mem = true;
38071 case V4SF_FTYPE_V4SF_PCV2SF:
38072 case V2DF_FTYPE_V2DF_PCDOUBLE:
38077 case V8SF_FTYPE_PCV8SF_V8SI:
38078 case V4DF_FTYPE_PCV4DF_V4DI:
38079 case V4SF_FTYPE_PCV4SF_V4SI:
38080 case V2DF_FTYPE_PCV2DF_V2DI:
38081 case V8SI_FTYPE_PCV8SI_V8SI:
38082 case V4DI_FTYPE_PCV4DI_V4DI:
38083 case V4SI_FTYPE_PCV4SI_V4SI:
38084 case V2DI_FTYPE_PCV2DI_V2DI:
38089 case VOID_FTYPE_PV8DF_V8DF_QI:
38090 case VOID_FTYPE_PV16SF_V16SF_HI:
38091 case VOID_FTYPE_PV8DI_V8DI_QI:
38092 case VOID_FTYPE_PV4DI_V4DI_QI:
38093 case VOID_FTYPE_PV2DI_V2DI_QI:
38094 case VOID_FTYPE_PV16SI_V16SI_HI:
38095 case VOID_FTYPE_PV8SI_V8SI_QI:
38096 case VOID_FTYPE_PV4SI_V4SI_QI:
38099 /* These builtins and instructions require the memory
38100 to be properly aligned. */
38101 case CODE_FOR_avx512f_storev16sf_mask:
38102 case CODE_FOR_avx512f_storev16si_mask:
38103 case CODE_FOR_avx512f_storev8df_mask:
38104 case CODE_FOR_avx512f_storev8di_mask:
38105 case CODE_FOR_avx512vl_storev8sf_mask:
38106 case CODE_FOR_avx512vl_storev8si_mask:
38107 case CODE_FOR_avx512vl_storev4df_mask:
38108 case CODE_FOR_avx512vl_storev4di_mask:
38109 case CODE_FOR_avx512vl_storev4sf_mask:
38110 case CODE_FOR_avx512vl_storev4si_mask:
38111 case CODE_FOR_avx512vl_storev2df_mask:
38112 case CODE_FOR_avx512vl_storev2di_mask:
38113 aligned_mem = true;
38119 case VOID_FTYPE_PV8SF_V8SI_V8SF:
38120 case VOID_FTYPE_PV4DF_V4DI_V4DF:
38121 case VOID_FTYPE_PV4SF_V4SI_V4SF:
38122 case VOID_FTYPE_PV2DF_V2DI_V2DF:
38123 case VOID_FTYPE_PV8SI_V8SI_V8SI:
38124 case VOID_FTYPE_PV4DI_V4DI_V4DI:
38125 case VOID_FTYPE_PV4SI_V4SI_V4SI:
38126 case VOID_FTYPE_PV2DI_V2DI_V2DI:
38127 case VOID_FTYPE_PDOUBLE_V2DF_QI:
38128 case VOID_FTYPE_PFLOAT_V4SF_QI:
38129 case VOID_FTYPE_PV8SI_V8DI_QI:
38130 case VOID_FTYPE_PV8HI_V8DI_QI:
38131 case VOID_FTYPE_PV16HI_V16SI_HI:
38132 case VOID_FTYPE_PV16QI_V8DI_QI:
38133 case VOID_FTYPE_PV16QI_V16SI_HI:
38134 case VOID_FTYPE_PV4SI_V4DI_QI:
38135 case VOID_FTYPE_PV4SI_V2DI_QI:
38136 case VOID_FTYPE_PV8HI_V4DI_QI:
38137 case VOID_FTYPE_PV8HI_V2DI_QI:
38138 case VOID_FTYPE_PV8HI_V8SI_QI:
38139 case VOID_FTYPE_PV8HI_V4SI_QI:
38140 case VOID_FTYPE_PV16QI_V4DI_QI:
38141 case VOID_FTYPE_PV16QI_V2DI_QI:
38142 case VOID_FTYPE_PV16QI_V8SI_QI:
38143 case VOID_FTYPE_PV16QI_V4SI_QI:
38144 case VOID_FTYPE_PV8HI_V8HI_QI:
38145 case VOID_FTYPE_PV16HI_V16HI_HI:
38146 case VOID_FTYPE_PV32HI_V32HI_SI:
38147 case VOID_FTYPE_PV16QI_V16QI_HI:
38148 case VOID_FTYPE_PV32QI_V32QI_SI:
38149 case VOID_FTYPE_PV64QI_V64QI_DI:
38150 case VOID_FTYPE_PV4DF_V4DF_QI:
38151 case VOID_FTYPE_PV2DF_V2DF_QI:
38152 case VOID_FTYPE_PV8SF_V8SF_QI:
38153 case VOID_FTYPE_PV4SF_V4SF_QI:
38156 /* Reserve memory operand for target. */
38157 memory = ARRAY_SIZE (args);
38159 case V4SF_FTYPE_PCV4SF_V4SF_QI:
38160 case V8SF_FTYPE_PCV8SF_V8SF_QI:
38161 case V16SF_FTYPE_PCV16SF_V16SF_HI:
38162 case V4SI_FTYPE_PCV4SI_V4SI_QI:
38163 case V8SI_FTYPE_PCV8SI_V8SI_QI:
38164 case V16SI_FTYPE_PCV16SI_V16SI_HI:
38165 case V2DF_FTYPE_PCV2DF_V2DF_QI:
38166 case V4DF_FTYPE_PCV4DF_V4DF_QI:
38167 case V8DF_FTYPE_PCV8DF_V8DF_QI:
38168 case V2DI_FTYPE_PCV2DI_V2DI_QI:
38169 case V4DI_FTYPE_PCV4DI_V4DI_QI:
38170 case V8DI_FTYPE_PCV8DI_V8DI_QI:
38171 case V2DF_FTYPE_PCDOUBLE_V2DF_QI:
38172 case V4SF_FTYPE_PCFLOAT_V4SF_QI:
38173 case V8HI_FTYPE_PCV8HI_V8HI_QI:
38174 case V16HI_FTYPE_PCV16HI_V16HI_HI:
38175 case V32HI_FTYPE_PCV32HI_V32HI_SI:
38176 case V16QI_FTYPE_PCV16QI_V16QI_HI:
38177 case V32QI_FTYPE_PCV32QI_V32QI_SI:
38178 case V64QI_FTYPE_PCV64QI_V64QI_DI:
38184 /* These builtins and instructions require the memory
38185 to be properly aligned. */
38186 case CODE_FOR_avx512f_loadv16sf_mask:
38187 case CODE_FOR_avx512f_loadv16si_mask:
38188 case CODE_FOR_avx512f_loadv8df_mask:
38189 case CODE_FOR_avx512f_loadv8di_mask:
38190 case CODE_FOR_avx512vl_loadv8sf_mask:
38191 case CODE_FOR_avx512vl_loadv8si_mask:
38192 case CODE_FOR_avx512vl_loadv4df_mask:
38193 case CODE_FOR_avx512vl_loadv4di_mask:
38194 case CODE_FOR_avx512vl_loadv4sf_mask:
38195 case CODE_FOR_avx512vl_loadv4si_mask:
38196 case CODE_FOR_avx512vl_loadv2df_mask:
38197 case CODE_FOR_avx512vl_loadv2di_mask:
38198 case CODE_FOR_avx512bw_loadv64qi_mask:
38199 case CODE_FOR_avx512vl_loadv32qi_mask:
38200 case CODE_FOR_avx512vl_loadv16qi_mask:
38201 case CODE_FOR_avx512bw_loadv32hi_mask:
38202 case CODE_FOR_avx512vl_loadv16hi_mask:
38203 case CODE_FOR_avx512vl_loadv8hi_mask:
38204 aligned_mem = true;
38210 case VOID_FTYPE_UINT_UINT_UINT:
38211 case VOID_FTYPE_UINT64_UINT_UINT:
38212 case UCHAR_FTYPE_UINT_UINT_UINT:
38213 case UCHAR_FTYPE_UINT64_UINT_UINT:
38216 memory = ARRAY_SIZE (args);
38217 last_arg_constant = true;
38220 gcc_unreachable ();
38223 gcc_assert (nargs <= ARRAY_SIZE (args));
38225 if (klass == store)
38227 arg = CALL_EXPR_ARG (exp, 0);
38228 op = expand_normal (arg);
38229 gcc_assert (target == 0);
38232 op = ix86_zero_extend_to_Pmode (op);
38233 target = gen_rtx_MEM (tmode, op);
38234 /* target at this point has just BITS_PER_UNIT MEM_ALIGN
38235 on it. Try to improve it using get_pointer_alignment,
38236 and if the special builtin is one that requires strict
38237 mode alignment, also from it's GET_MODE_ALIGNMENT.
38238 Failure to do so could lead to ix86_legitimate_combined_insn
38239 rejecting all changes to such insns. */
38240 unsigned int align = get_pointer_alignment (arg);
38241 if (aligned_mem && align < GET_MODE_ALIGNMENT (tmode))
38242 align = GET_MODE_ALIGNMENT (tmode);
38243 if (MEM_ALIGN (target) < align)
38244 set_mem_align (target, align);
38247 target = force_reg (tmode, op);
38255 || !register_operand (target, tmode)
38256 || GET_MODE (target) != tmode)
38257 target = gen_reg_rtx (tmode);
38260 for (i = 0; i < nargs; i++)
38262 machine_mode mode = insn_p->operand[i + 1].mode;
38265 arg = CALL_EXPR_ARG (exp, i + arg_adjust);
38266 op = expand_normal (arg);
38267 match = insn_p->operand[i + 1].predicate (op, mode);
38269 if (last_arg_constant && (i + 1) == nargs)
38273 if (icode == CODE_FOR_lwp_lwpvalsi3
38274 || icode == CODE_FOR_lwp_lwpinssi3
38275 || icode == CODE_FOR_lwp_lwpvaldi3
38276 || icode == CODE_FOR_lwp_lwpinsdi3)
38277 error ("the last argument must be a 32-bit immediate");
38279 error ("the last argument must be an 8-bit immediate");
38287 /* This must be the memory operand. */
38288 op = ix86_zero_extend_to_Pmode (op);
38289 op = gen_rtx_MEM (mode, op);
38290 /* op at this point has just BITS_PER_UNIT MEM_ALIGN
38291 on it. Try to improve it using get_pointer_alignment,
38292 and if the special builtin is one that requires strict
38293 mode alignment, also from it's GET_MODE_ALIGNMENT.
38294 Failure to do so could lead to ix86_legitimate_combined_insn
38295 rejecting all changes to such insns. */
38296 unsigned int align = get_pointer_alignment (arg);
38297 if (aligned_mem && align < GET_MODE_ALIGNMENT (mode))
38298 align = GET_MODE_ALIGNMENT (mode);
38299 if (MEM_ALIGN (op) < align)
38300 set_mem_align (op, align);
38304 /* This must be register. */
38305 if (VECTOR_MODE_P (mode))
38306 op = safe_vector_operand (op, mode);
38308 op = fixup_modeless_constant (op, mode);
38310 if (GET_MODE (op) == mode || GET_MODE (op) == VOIDmode)
38311 op = copy_to_mode_reg (mode, op);
38314 op = copy_to_reg (op);
38315 op = simplify_gen_subreg (mode, op, GET_MODE (op), 0);
38321 args[i].mode = mode;
38327 pat = GEN_FCN (icode) (target);
38330 pat = GEN_FCN (icode) (target, args[0].op);
38333 pat = GEN_FCN (icode) (target, args[0].op, args[1].op);
38336 pat = GEN_FCN (icode) (target, args[0].op, args[1].op, args[2].op);
38339 gcc_unreachable ();
38345 return klass == store ? 0 : target;
38348 /* Return the integer constant in ARG. Constrain it to be in the range
38349 of the subparts of VEC_TYPE; issue an error if not. */
38352 get_element_number (tree vec_type, tree arg)
38354 unsigned HOST_WIDE_INT elt, max = TYPE_VECTOR_SUBPARTS (vec_type) - 1;
38356 if (!tree_fits_uhwi_p (arg)
38357 || (elt = tree_to_uhwi (arg), elt > max))
38359 error ("selector must be an integer constant in the range 0..%wi", max);
38366 /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
38367 ix86_expand_vector_init. We DO have language-level syntax for this, in
38368 the form of (type){ init-list }. Except that since we can't place emms
38369 instructions from inside the compiler, we can't allow the use of MMX
38370 registers unless the user explicitly asks for it. So we do *not* define
38371 vec_set/vec_extract/vec_init patterns for MMX modes in mmx.md. Instead
38372 we have builtins invoked by mmintrin.h that gives us license to emit
38373 these sorts of instructions. */
38376 ix86_expand_vec_init_builtin (tree type, tree exp, rtx target)
38378 machine_mode tmode = TYPE_MODE (type);
38379 machine_mode inner_mode = GET_MODE_INNER (tmode);
38380 int i, n_elt = GET_MODE_NUNITS (tmode);
38381 rtvec v = rtvec_alloc (n_elt);
38383 gcc_assert (VECTOR_MODE_P (tmode));
38384 gcc_assert (call_expr_nargs (exp) == n_elt);
38386 for (i = 0; i < n_elt; ++i)
38388 rtx x = expand_normal (CALL_EXPR_ARG (exp, i));
38389 RTVEC_ELT (v, i) = gen_lowpart (inner_mode, x);
38392 if (!target || !register_operand (target, tmode))
38393 target = gen_reg_rtx (tmode);
38395 ix86_expand_vector_init (true, target, gen_rtx_PARALLEL (tmode, v));
38399 /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
38400 ix86_expand_vector_extract. They would be redundant (for non-MMX) if we
38401 had a language-level syntax for referencing vector elements. */
38404 ix86_expand_vec_ext_builtin (tree exp, rtx target)
38406 machine_mode tmode, mode0;
38411 arg0 = CALL_EXPR_ARG (exp, 0);
38412 arg1 = CALL_EXPR_ARG (exp, 1);
38414 op0 = expand_normal (arg0);
38415 elt = get_element_number (TREE_TYPE (arg0), arg1);
38417 tmode = TYPE_MODE (TREE_TYPE (TREE_TYPE (arg0)));
38418 mode0 = TYPE_MODE (TREE_TYPE (arg0));
38419 gcc_assert (VECTOR_MODE_P (mode0));
38421 op0 = force_reg (mode0, op0);
38423 if (optimize || !target || !register_operand (target, tmode))
38424 target = gen_reg_rtx (tmode);
38426 ix86_expand_vector_extract (true, target, op0, elt);
38431 /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
38432 ix86_expand_vector_set. They would be redundant (for non-MMX) if we had
38433 a language-level syntax for referencing vector elements. */
38436 ix86_expand_vec_set_builtin (tree exp)
38438 machine_mode tmode, mode1;
38439 tree arg0, arg1, arg2;
38441 rtx op0, op1, target;
38443 arg0 = CALL_EXPR_ARG (exp, 0);
38444 arg1 = CALL_EXPR_ARG (exp, 1);
38445 arg2 = CALL_EXPR_ARG (exp, 2);
38447 tmode = TYPE_MODE (TREE_TYPE (arg0));
38448 mode1 = TYPE_MODE (TREE_TYPE (TREE_TYPE (arg0)));
38449 gcc_assert (VECTOR_MODE_P (tmode));
38451 op0 = expand_expr (arg0, NULL_RTX, tmode, EXPAND_NORMAL);
38452 op1 = expand_expr (arg1, NULL_RTX, mode1, EXPAND_NORMAL);
38453 elt = get_element_number (TREE_TYPE (arg0), arg2);
38455 if (GET_MODE (op1) != mode1 && GET_MODE (op1) != VOIDmode)
38456 op1 = convert_modes (mode1, GET_MODE (op1), op1, true);
38458 op0 = force_reg (tmode, op0);
38459 op1 = force_reg (mode1, op1);
38461 /* OP0 is the source of these builtin functions and shouldn't be
38462 modified. Create a copy, use it and return it as target. */
38463 target = gen_reg_rtx (tmode);
38464 emit_move_insn (target, op0);
38465 ix86_expand_vector_set (true, target, op1, elt);
38470 /* Emit conditional move of SRC to DST with condition
38473 ix86_emit_cmove (rtx dst, rtx src, enum rtx_code code, rtx op1, rtx op2)
38479 t = ix86_expand_compare (code, op1, op2);
38480 emit_insn (gen_rtx_SET (dst, gen_rtx_IF_THEN_ELSE (GET_MODE (dst), t,
38485 rtx_code_label *nomove = gen_label_rtx ();
38486 emit_cmp_and_jump_insns (op1, op2, reverse_condition (code),
38487 const0_rtx, GET_MODE (op1), 1, nomove);
38488 emit_move_insn (dst, src);
38489 emit_label (nomove);
38493 /* Choose max of DST and SRC and put it to DST. */
38495 ix86_emit_move_max (rtx dst, rtx src)
38497 ix86_emit_cmove (dst, src, LTU, dst, src);
38500 /* Expand an expression EXP that calls a built-in function,
38501 with result going to TARGET if that's convenient
38502 (and in mode MODE if that's convenient).
38503 SUBTARGET may be used as the target for computing one of EXP's operands.
38504 IGNORE is nonzero if the value is to be ignored. */
38507 ix86_expand_builtin (tree exp, rtx target, rtx subtarget,
38508 machine_mode mode, int ignore)
38510 const struct builtin_description *d;
38512 enum insn_code icode;
38513 tree fndecl = TREE_OPERAND (CALL_EXPR_FN (exp), 0);
38514 tree arg0, arg1, arg2, arg3, arg4;
38515 rtx op0, op1, op2, op3, op4, pat, insn;
38516 machine_mode mode0, mode1, mode2, mode3, mode4;
38517 unsigned int fcode = DECL_FUNCTION_CODE (fndecl);
38519 /* For CPU builtins that can be folded, fold first and expand the fold. */
38522 case IX86_BUILTIN_CPU_INIT:
38524 /* Make it call __cpu_indicator_init in libgcc. */
38525 tree call_expr, fndecl, type;
38526 type = build_function_type_list (integer_type_node, NULL_TREE);
38527 fndecl = build_fn_decl ("__cpu_indicator_init", type);
38528 call_expr = build_call_expr (fndecl, 0);
38529 return expand_expr (call_expr, target, mode, EXPAND_NORMAL);
38531 case IX86_BUILTIN_CPU_IS:
38532 case IX86_BUILTIN_CPU_SUPPORTS:
38534 tree arg0 = CALL_EXPR_ARG (exp, 0);
38535 tree fold_expr = fold_builtin_cpu (fndecl, &arg0);
38536 gcc_assert (fold_expr != NULL_TREE);
38537 return expand_expr (fold_expr, target, mode, EXPAND_NORMAL);
38541 /* Determine whether the builtin function is available under the current ISA.
38542 Originally the builtin was not created if it wasn't applicable to the
38543 current ISA based on the command line switches. With function specific
38544 options, we need to check in the context of the function making the call
38545 whether it is supported. */
38546 if (ix86_builtins_isa[fcode].isa
38547 && !(ix86_builtins_isa[fcode].isa & ix86_isa_flags))
38549 char *opts = ix86_target_string (ix86_builtins_isa[fcode].isa, 0, NULL,
38550 NULL, (enum fpmath_unit) 0, false);
38553 error ("%qE needs unknown isa option", fndecl);
38556 gcc_assert (opts != NULL);
38557 error ("%qE needs isa option %s", fndecl, opts);
38565 case IX86_BUILTIN_BNDMK:
38567 || GET_MODE (target) != BNDmode
38568 || !register_operand (target, BNDmode))
38569 target = gen_reg_rtx (BNDmode);
38571 arg0 = CALL_EXPR_ARG (exp, 0);
38572 arg1 = CALL_EXPR_ARG (exp, 1);
38574 op0 = expand_normal (arg0);
38575 op1 = expand_normal (arg1);
38577 if (!register_operand (op0, Pmode))
38578 op0 = ix86_zero_extend_to_Pmode (op0);
38579 if (!register_operand (op1, Pmode))
38580 op1 = ix86_zero_extend_to_Pmode (op1);
38582 /* Builtin arg1 is size of block but instruction op1 should
38584 op1 = expand_simple_binop (Pmode, PLUS, op1, constm1_rtx,
38585 NULL_RTX, 1, OPTAB_DIRECT);
38587 emit_insn (BNDmode == BND64mode
38588 ? gen_bnd64_mk (target, op0, op1)
38589 : gen_bnd32_mk (target, op0, op1));
38592 case IX86_BUILTIN_BNDSTX:
38593 arg0 = CALL_EXPR_ARG (exp, 0);
38594 arg1 = CALL_EXPR_ARG (exp, 1);
38595 arg2 = CALL_EXPR_ARG (exp, 2);
38597 op0 = expand_normal (arg0);
38598 op1 = expand_normal (arg1);
38599 op2 = expand_normal (arg2);
38601 if (!register_operand (op0, Pmode))
38602 op0 = ix86_zero_extend_to_Pmode (op0);
38603 if (!register_operand (op1, BNDmode))
38604 op1 = copy_to_mode_reg (BNDmode, op1);
38605 if (!register_operand (op2, Pmode))
38606 op2 = ix86_zero_extend_to_Pmode (op2);
38608 emit_insn (BNDmode == BND64mode
38609 ? gen_bnd64_stx (op2, op0, op1)
38610 : gen_bnd32_stx (op2, op0, op1));
38613 case IX86_BUILTIN_BNDLDX:
38615 || GET_MODE (target) != BNDmode
38616 || !register_operand (target, BNDmode))
38617 target = gen_reg_rtx (BNDmode);
38619 arg0 = CALL_EXPR_ARG (exp, 0);
38620 arg1 = CALL_EXPR_ARG (exp, 1);
38622 op0 = expand_normal (arg0);
38623 op1 = expand_normal (arg1);
38625 if (!register_operand (op0, Pmode))
38626 op0 = ix86_zero_extend_to_Pmode (op0);
38627 if (!register_operand (op1, Pmode))
38628 op1 = ix86_zero_extend_to_Pmode (op1);
38630 emit_insn (BNDmode == BND64mode
38631 ? gen_bnd64_ldx (target, op0, op1)
38632 : gen_bnd32_ldx (target, op0, op1));
38635 case IX86_BUILTIN_BNDCL:
38636 arg0 = CALL_EXPR_ARG (exp, 0);
38637 arg1 = CALL_EXPR_ARG (exp, 1);
38639 op0 = expand_normal (arg0);
38640 op1 = expand_normal (arg1);
38642 if (!register_operand (op0, Pmode))
38643 op0 = ix86_zero_extend_to_Pmode (op0);
38644 if (!register_operand (op1, BNDmode))
38645 op1 = copy_to_mode_reg (BNDmode, op1);
38647 emit_insn (BNDmode == BND64mode
38648 ? gen_bnd64_cl (op1, op0)
38649 : gen_bnd32_cl (op1, op0));
38652 case IX86_BUILTIN_BNDCU:
38653 arg0 = CALL_EXPR_ARG (exp, 0);
38654 arg1 = CALL_EXPR_ARG (exp, 1);
38656 op0 = expand_normal (arg0);
38657 op1 = expand_normal (arg1);
38659 if (!register_operand (op0, Pmode))
38660 op0 = ix86_zero_extend_to_Pmode (op0);
38661 if (!register_operand (op1, BNDmode))
38662 op1 = copy_to_mode_reg (BNDmode, op1);
38664 emit_insn (BNDmode == BND64mode
38665 ? gen_bnd64_cu (op1, op0)
38666 : gen_bnd32_cu (op1, op0));
38669 case IX86_BUILTIN_BNDRET:
38670 arg0 = CALL_EXPR_ARG (exp, 0);
38671 gcc_assert (TREE_CODE (arg0) == SSA_NAME);
38672 target = chkp_get_rtl_bounds (arg0);
38674 /* If no bounds were specified for returned value,
38675 then use INIT bounds. It usually happens when
38676 some built-in function is expanded. */
38679 rtx t1 = gen_reg_rtx (Pmode);
38680 rtx t2 = gen_reg_rtx (Pmode);
38681 target = gen_reg_rtx (BNDmode);
38682 emit_move_insn (t1, const0_rtx);
38683 emit_move_insn (t2, constm1_rtx);
38684 emit_insn (BNDmode == BND64mode
38685 ? gen_bnd64_mk (target, t1, t2)
38686 : gen_bnd32_mk (target, t1, t2));
38689 gcc_assert (target && REG_P (target));
38692 case IX86_BUILTIN_BNDNARROW:
38694 rtx m1, m1h1, m1h2, lb, ub, t1;
38696 /* Return value and lb. */
38697 arg0 = CALL_EXPR_ARG (exp, 0);
38699 arg1 = CALL_EXPR_ARG (exp, 1);
38701 arg2 = CALL_EXPR_ARG (exp, 2);
38703 lb = expand_normal (arg0);
38704 op1 = expand_normal (arg1);
38705 op2 = expand_normal (arg2);
38707 /* Size was passed but we need to use (size - 1) as for bndmk. */
38708 op2 = expand_simple_binop (Pmode, PLUS, op2, constm1_rtx,
38709 NULL_RTX, 1, OPTAB_DIRECT);
38711 /* Add LB to size and inverse to get UB. */
38712 op2 = expand_simple_binop (Pmode, PLUS, op2, lb,
38713 op2, 1, OPTAB_DIRECT);
38714 ub = expand_simple_unop (Pmode, NOT, op2, op2, 1);
38716 if (!register_operand (lb, Pmode))
38717 lb = ix86_zero_extend_to_Pmode (lb);
38718 if (!register_operand (ub, Pmode))
38719 ub = ix86_zero_extend_to_Pmode (ub);
38721 /* We need to move bounds to memory before any computations. */
38726 m1 = assign_386_stack_local (BNDmode, SLOT_TEMP);
38727 emit_move_insn (m1, op1);
38730 /* Generate mem expression to be used for access to LB and UB. */
38731 m1h1 = adjust_address (m1, Pmode, 0);
38732 m1h2 = adjust_address (m1, Pmode, GET_MODE_SIZE (Pmode));
38734 t1 = gen_reg_rtx (Pmode);
38737 emit_move_insn (t1, m1h1);
38738 ix86_emit_move_max (t1, lb);
38739 emit_move_insn (m1h1, t1);
38741 /* Compute UB. UB is stored in 1's complement form. Therefore
38742 we also use max here. */
38743 emit_move_insn (t1, m1h2);
38744 ix86_emit_move_max (t1, ub);
38745 emit_move_insn (m1h2, t1);
38747 op2 = gen_reg_rtx (BNDmode);
38748 emit_move_insn (op2, m1);
38750 return chkp_join_splitted_slot (lb, op2);
38753 case IX86_BUILTIN_BNDINT:
38755 rtx res, rh1, rh2, lb1, lb2, ub1, ub2;
38758 || GET_MODE (target) != BNDmode
38759 || !register_operand (target, BNDmode))
38760 target = gen_reg_rtx (BNDmode);
38762 arg0 = CALL_EXPR_ARG (exp, 0);
38763 arg1 = CALL_EXPR_ARG (exp, 1);
38765 op0 = expand_normal (arg0);
38766 op1 = expand_normal (arg1);
38768 res = assign_386_stack_local (BNDmode, SLOT_TEMP);
38769 rh1 = adjust_address (res, Pmode, 0);
38770 rh2 = adjust_address (res, Pmode, GET_MODE_SIZE (Pmode));
38772 /* Put first bounds to temporaries. */
38773 lb1 = gen_reg_rtx (Pmode);
38774 ub1 = gen_reg_rtx (Pmode);
38777 emit_move_insn (lb1, adjust_address (op0, Pmode, 0));
38778 emit_move_insn (ub1, adjust_address (op0, Pmode,
38779 GET_MODE_SIZE (Pmode)));
38783 emit_move_insn (res, op0);
38784 emit_move_insn (lb1, rh1);
38785 emit_move_insn (ub1, rh2);
38788 /* Put second bounds to temporaries. */
38789 lb2 = gen_reg_rtx (Pmode);
38790 ub2 = gen_reg_rtx (Pmode);
38793 emit_move_insn (lb2, adjust_address (op1, Pmode, 0));
38794 emit_move_insn (ub2, adjust_address (op1, Pmode,
38795 GET_MODE_SIZE (Pmode)));
38799 emit_move_insn (res, op1);
38800 emit_move_insn (lb2, rh1);
38801 emit_move_insn (ub2, rh2);
38805 ix86_emit_move_max (lb1, lb2);
38806 emit_move_insn (rh1, lb1);
38808 /* Compute UB. UB is stored in 1's complement form. Therefore
38809 we also use max here. */
38810 ix86_emit_move_max (ub1, ub2);
38811 emit_move_insn (rh2, ub1);
38813 emit_move_insn (target, res);
38818 case IX86_BUILTIN_SIZEOF:
38824 || GET_MODE (target) != Pmode
38825 || !register_operand (target, Pmode))
38826 target = gen_reg_rtx (Pmode);
38828 arg0 = CALL_EXPR_ARG (exp, 0);
38829 gcc_assert (TREE_CODE (arg0) == VAR_DECL);
38831 name = DECL_ASSEMBLER_NAME (arg0);
38832 symbol = gen_rtx_SYMBOL_REF (Pmode, IDENTIFIER_POINTER (name));
38834 emit_insn (Pmode == SImode
38835 ? gen_move_size_reloc_si (target, symbol)
38836 : gen_move_size_reloc_di (target, symbol));
38841 case IX86_BUILTIN_BNDLOWER:
38846 || GET_MODE (target) != Pmode
38847 || !register_operand (target, Pmode))
38848 target = gen_reg_rtx (Pmode);
38850 arg0 = CALL_EXPR_ARG (exp, 0);
38851 op0 = expand_normal (arg0);
38853 /* We need to move bounds to memory first. */
38858 mem = assign_386_stack_local (BNDmode, SLOT_TEMP);
38859 emit_move_insn (mem, op0);
38862 /* Generate mem expression to access LB and load it. */
38863 hmem = adjust_address (mem, Pmode, 0);
38864 emit_move_insn (target, hmem);
38869 case IX86_BUILTIN_BNDUPPER:
38871 rtx mem, hmem, res;
38874 || GET_MODE (target) != Pmode
38875 || !register_operand (target, Pmode))
38876 target = gen_reg_rtx (Pmode);
38878 arg0 = CALL_EXPR_ARG (exp, 0);
38879 op0 = expand_normal (arg0);
38881 /* We need to move bounds to memory first. */
38886 mem = assign_386_stack_local (BNDmode, SLOT_TEMP);
38887 emit_move_insn (mem, op0);
38890 /* Generate mem expression to access UB. */
38891 hmem = adjust_address (mem, Pmode, GET_MODE_SIZE (Pmode));
38893 /* We need to inverse all bits of UB. */
38894 res = expand_simple_unop (Pmode, NOT, hmem, target, 1);
38897 emit_move_insn (target, res);
38902 case IX86_BUILTIN_MASKMOVQ:
38903 case IX86_BUILTIN_MASKMOVDQU:
38904 icode = (fcode == IX86_BUILTIN_MASKMOVQ
38905 ? CODE_FOR_mmx_maskmovq
38906 : CODE_FOR_sse2_maskmovdqu);
38907 /* Note the arg order is different from the operand order. */
38908 arg1 = CALL_EXPR_ARG (exp, 0);
38909 arg2 = CALL_EXPR_ARG (exp, 1);
38910 arg0 = CALL_EXPR_ARG (exp, 2);
38911 op0 = expand_normal (arg0);
38912 op1 = expand_normal (arg1);
38913 op2 = expand_normal (arg2);
38914 mode0 = insn_data[icode].operand[0].mode;
38915 mode1 = insn_data[icode].operand[1].mode;
38916 mode2 = insn_data[icode].operand[2].mode;
38918 op0 = ix86_zero_extend_to_Pmode (op0);
38919 op0 = gen_rtx_MEM (mode1, op0);
38921 if (!insn_data[icode].operand[0].predicate (op0, mode0))
38922 op0 = copy_to_mode_reg (mode0, op0);
38923 if (!insn_data[icode].operand[1].predicate (op1, mode1))
38924 op1 = copy_to_mode_reg (mode1, op1);
38925 if (!insn_data[icode].operand[2].predicate (op2, mode2))
38926 op2 = copy_to_mode_reg (mode2, op2);
38927 pat = GEN_FCN (icode) (op0, op1, op2);
38933 case IX86_BUILTIN_LDMXCSR:
38934 op0 = expand_normal (CALL_EXPR_ARG (exp, 0));
38935 target = assign_386_stack_local (SImode, SLOT_TEMP);
38936 emit_move_insn (target, op0);
38937 emit_insn (gen_sse_ldmxcsr (target));
38940 case IX86_BUILTIN_STMXCSR:
38941 target = assign_386_stack_local (SImode, SLOT_TEMP);
38942 emit_insn (gen_sse_stmxcsr (target));
38943 return copy_to_mode_reg (SImode, target);
38945 case IX86_BUILTIN_CLFLUSH:
38946 arg0 = CALL_EXPR_ARG (exp, 0);
38947 op0 = expand_normal (arg0);
38948 icode = CODE_FOR_sse2_clflush;
38949 if (!insn_data[icode].operand[0].predicate (op0, Pmode))
38950 op0 = ix86_zero_extend_to_Pmode (op0);
38952 emit_insn (gen_sse2_clflush (op0));
38955 case IX86_BUILTIN_CLWB:
38956 arg0 = CALL_EXPR_ARG (exp, 0);
38957 op0 = expand_normal (arg0);
38958 icode = CODE_FOR_clwb;
38959 if (!insn_data[icode].operand[0].predicate (op0, Pmode))
38960 op0 = ix86_zero_extend_to_Pmode (op0);
38962 emit_insn (gen_clwb (op0));
38965 case IX86_BUILTIN_CLFLUSHOPT:
38966 arg0 = CALL_EXPR_ARG (exp, 0);
38967 op0 = expand_normal (arg0);
38968 icode = CODE_FOR_clflushopt;
38969 if (!insn_data[icode].operand[0].predicate (op0, Pmode))
38970 op0 = ix86_zero_extend_to_Pmode (op0);
38972 emit_insn (gen_clflushopt (op0));
38975 case IX86_BUILTIN_MONITOR:
38976 case IX86_BUILTIN_MONITORX:
38977 arg0 = CALL_EXPR_ARG (exp, 0);
38978 arg1 = CALL_EXPR_ARG (exp, 1);
38979 arg2 = CALL_EXPR_ARG (exp, 2);
38980 op0 = expand_normal (arg0);
38981 op1 = expand_normal (arg1);
38982 op2 = expand_normal (arg2);
38984 op0 = ix86_zero_extend_to_Pmode (op0);
38986 op1 = copy_to_mode_reg (SImode, op1);
38988 op2 = copy_to_mode_reg (SImode, op2);
38990 emit_insn (fcode == IX86_BUILTIN_MONITOR
38991 ? ix86_gen_monitor (op0, op1, op2)
38992 : ix86_gen_monitorx (op0, op1, op2));
38995 case IX86_BUILTIN_MWAIT:
38996 arg0 = CALL_EXPR_ARG (exp, 0);
38997 arg1 = CALL_EXPR_ARG (exp, 1);
38998 op0 = expand_normal (arg0);
38999 op1 = expand_normal (arg1);
39001 op0 = copy_to_mode_reg (SImode, op0);
39003 op1 = copy_to_mode_reg (SImode, op1);
39004 emit_insn (gen_sse3_mwait (op0, op1));
39007 case IX86_BUILTIN_MWAITX:
39008 arg0 = CALL_EXPR_ARG (exp, 0);
39009 arg1 = CALL_EXPR_ARG (exp, 1);
39010 arg2 = CALL_EXPR_ARG (exp, 2);
39011 op0 = expand_normal (arg0);
39012 op1 = expand_normal (arg1);
39013 op2 = expand_normal (arg2);
39015 op0 = copy_to_mode_reg (SImode, op0);
39017 op1 = copy_to_mode_reg (SImode, op1);
39019 op2 = copy_to_mode_reg (SImode, op2);
39020 emit_insn (gen_mwaitx (op0, op1, op2));
39023 case IX86_BUILTIN_VEC_INIT_V2SI:
39024 case IX86_BUILTIN_VEC_INIT_V4HI:
39025 case IX86_BUILTIN_VEC_INIT_V8QI:
39026 return ix86_expand_vec_init_builtin (TREE_TYPE (exp), exp, target);
39028 case IX86_BUILTIN_VEC_EXT_V2DF:
39029 case IX86_BUILTIN_VEC_EXT_V2DI:
39030 case IX86_BUILTIN_VEC_EXT_V4SF:
39031 case IX86_BUILTIN_VEC_EXT_V4SI:
39032 case IX86_BUILTIN_VEC_EXT_V8HI:
39033 case IX86_BUILTIN_VEC_EXT_V2SI:
39034 case IX86_BUILTIN_VEC_EXT_V4HI:
39035 case IX86_BUILTIN_VEC_EXT_V16QI:
39036 return ix86_expand_vec_ext_builtin (exp, target);
39038 case IX86_BUILTIN_VEC_SET_V2DI:
39039 case IX86_BUILTIN_VEC_SET_V4SF:
39040 case IX86_BUILTIN_VEC_SET_V4SI:
39041 case IX86_BUILTIN_VEC_SET_V8HI:
39042 case IX86_BUILTIN_VEC_SET_V4HI:
39043 case IX86_BUILTIN_VEC_SET_V16QI:
39044 return ix86_expand_vec_set_builtin (exp);
39046 case IX86_BUILTIN_INFQ:
39047 case IX86_BUILTIN_HUGE_VALQ:
39049 REAL_VALUE_TYPE inf;
39053 tmp = CONST_DOUBLE_FROM_REAL_VALUE (inf, mode);
39055 tmp = validize_mem (force_const_mem (mode, tmp));
39058 target = gen_reg_rtx (mode);
39060 emit_move_insn (target, tmp);
39064 case IX86_BUILTIN_RDPMC:
39065 case IX86_BUILTIN_RDTSC:
39066 case IX86_BUILTIN_RDTSCP:
39068 op0 = gen_reg_rtx (DImode);
39069 op1 = gen_reg_rtx (DImode);
39071 if (fcode == IX86_BUILTIN_RDPMC)
39073 arg0 = CALL_EXPR_ARG (exp, 0);
39074 op2 = expand_normal (arg0);
39075 if (!register_operand (op2, SImode))
39076 op2 = copy_to_mode_reg (SImode, op2);
39078 insn = (TARGET_64BIT
39079 ? gen_rdpmc_rex64 (op0, op1, op2)
39080 : gen_rdpmc (op0, op2));
39083 else if (fcode == IX86_BUILTIN_RDTSC)
39085 insn = (TARGET_64BIT
39086 ? gen_rdtsc_rex64 (op0, op1)
39087 : gen_rdtsc (op0));
39092 op2 = gen_reg_rtx (SImode);
39094 insn = (TARGET_64BIT
39095 ? gen_rdtscp_rex64 (op0, op1, op2)
39096 : gen_rdtscp (op0, op2));
39099 arg0 = CALL_EXPR_ARG (exp, 0);
39100 op4 = expand_normal (arg0);
39101 if (!address_operand (op4, VOIDmode))
39103 op4 = convert_memory_address (Pmode, op4);
39104 op4 = copy_addr_to_reg (op4);
39106 emit_move_insn (gen_rtx_MEM (SImode, op4), op2);
39111 /* mode is VOIDmode if __builtin_rd* has been called
39113 if (mode == VOIDmode)
39115 target = gen_reg_rtx (mode);
39120 op1 = expand_simple_binop (DImode, ASHIFT, op1, GEN_INT (32),
39121 op1, 1, OPTAB_DIRECT);
39122 op0 = expand_simple_binop (DImode, IOR, op0, op1,
39123 op0, 1, OPTAB_DIRECT);
39126 emit_move_insn (target, op0);
39129 case IX86_BUILTIN_FXSAVE:
39130 case IX86_BUILTIN_FXRSTOR:
39131 case IX86_BUILTIN_FXSAVE64:
39132 case IX86_BUILTIN_FXRSTOR64:
39133 case IX86_BUILTIN_FNSTENV:
39134 case IX86_BUILTIN_FLDENV:
39138 case IX86_BUILTIN_FXSAVE:
39139 icode = CODE_FOR_fxsave;
39141 case IX86_BUILTIN_FXRSTOR:
39142 icode = CODE_FOR_fxrstor;
39144 case IX86_BUILTIN_FXSAVE64:
39145 icode = CODE_FOR_fxsave64;
39147 case IX86_BUILTIN_FXRSTOR64:
39148 icode = CODE_FOR_fxrstor64;
39150 case IX86_BUILTIN_FNSTENV:
39151 icode = CODE_FOR_fnstenv;
39153 case IX86_BUILTIN_FLDENV:
39154 icode = CODE_FOR_fldenv;
39157 gcc_unreachable ();
39160 arg0 = CALL_EXPR_ARG (exp, 0);
39161 op0 = expand_normal (arg0);
39163 if (!address_operand (op0, VOIDmode))
39165 op0 = convert_memory_address (Pmode, op0);
39166 op0 = copy_addr_to_reg (op0);
39168 op0 = gen_rtx_MEM (mode0, op0);
39170 pat = GEN_FCN (icode) (op0);
39175 case IX86_BUILTIN_XSAVE:
39176 case IX86_BUILTIN_XRSTOR:
39177 case IX86_BUILTIN_XSAVE64:
39178 case IX86_BUILTIN_XRSTOR64:
39179 case IX86_BUILTIN_XSAVEOPT:
39180 case IX86_BUILTIN_XSAVEOPT64:
39181 case IX86_BUILTIN_XSAVES:
39182 case IX86_BUILTIN_XRSTORS:
39183 case IX86_BUILTIN_XSAVES64:
39184 case IX86_BUILTIN_XRSTORS64:
39185 case IX86_BUILTIN_XSAVEC:
39186 case IX86_BUILTIN_XSAVEC64:
39187 arg0 = CALL_EXPR_ARG (exp, 0);
39188 arg1 = CALL_EXPR_ARG (exp, 1);
39189 op0 = expand_normal (arg0);
39190 op1 = expand_normal (arg1);
39192 if (!address_operand (op0, VOIDmode))
39194 op0 = convert_memory_address (Pmode, op0);
39195 op0 = copy_addr_to_reg (op0);
39197 op0 = gen_rtx_MEM (BLKmode, op0);
39199 op1 = force_reg (DImode, op1);
39203 op2 = expand_simple_binop (DImode, LSHIFTRT, op1, GEN_INT (32),
39204 NULL, 1, OPTAB_DIRECT);
39207 case IX86_BUILTIN_XSAVE:
39208 icode = CODE_FOR_xsave_rex64;
39210 case IX86_BUILTIN_XRSTOR:
39211 icode = CODE_FOR_xrstor_rex64;
39213 case IX86_BUILTIN_XSAVE64:
39214 icode = CODE_FOR_xsave64;
39216 case IX86_BUILTIN_XRSTOR64:
39217 icode = CODE_FOR_xrstor64;
39219 case IX86_BUILTIN_XSAVEOPT:
39220 icode = CODE_FOR_xsaveopt_rex64;
39222 case IX86_BUILTIN_XSAVEOPT64:
39223 icode = CODE_FOR_xsaveopt64;
39225 case IX86_BUILTIN_XSAVES:
39226 icode = CODE_FOR_xsaves_rex64;
39228 case IX86_BUILTIN_XRSTORS:
39229 icode = CODE_FOR_xrstors_rex64;
39231 case IX86_BUILTIN_XSAVES64:
39232 icode = CODE_FOR_xsaves64;
39234 case IX86_BUILTIN_XRSTORS64:
39235 icode = CODE_FOR_xrstors64;
39237 case IX86_BUILTIN_XSAVEC:
39238 icode = CODE_FOR_xsavec_rex64;
39240 case IX86_BUILTIN_XSAVEC64:
39241 icode = CODE_FOR_xsavec64;
39244 gcc_unreachable ();
39247 op2 = gen_lowpart (SImode, op2);
39248 op1 = gen_lowpart (SImode, op1);
39249 pat = GEN_FCN (icode) (op0, op1, op2);
39255 case IX86_BUILTIN_XSAVE:
39256 icode = CODE_FOR_xsave;
39258 case IX86_BUILTIN_XRSTOR:
39259 icode = CODE_FOR_xrstor;
39261 case IX86_BUILTIN_XSAVEOPT:
39262 icode = CODE_FOR_xsaveopt;
39264 case IX86_BUILTIN_XSAVES:
39265 icode = CODE_FOR_xsaves;
39267 case IX86_BUILTIN_XRSTORS:
39268 icode = CODE_FOR_xrstors;
39270 case IX86_BUILTIN_XSAVEC:
39271 icode = CODE_FOR_xsavec;
39274 gcc_unreachable ();
39276 pat = GEN_FCN (icode) (op0, op1);
39283 case IX86_BUILTIN_LLWPCB:
39284 arg0 = CALL_EXPR_ARG (exp, 0);
39285 op0 = expand_normal (arg0);
39286 icode = CODE_FOR_lwp_llwpcb;
39287 if (!insn_data[icode].operand[0].predicate (op0, Pmode))
39288 op0 = ix86_zero_extend_to_Pmode (op0);
39289 emit_insn (gen_lwp_llwpcb (op0));
39292 case IX86_BUILTIN_SLWPCB:
39293 icode = CODE_FOR_lwp_slwpcb;
39295 || !insn_data[icode].operand[0].predicate (target, Pmode))
39296 target = gen_reg_rtx (Pmode);
39297 emit_insn (gen_lwp_slwpcb (target));
39300 case IX86_BUILTIN_BEXTRI32:
39301 case IX86_BUILTIN_BEXTRI64:
39302 arg0 = CALL_EXPR_ARG (exp, 0);
39303 arg1 = CALL_EXPR_ARG (exp, 1);
39304 op0 = expand_normal (arg0);
39305 op1 = expand_normal (arg1);
39306 icode = (fcode == IX86_BUILTIN_BEXTRI32
39307 ? CODE_FOR_tbm_bextri_si
39308 : CODE_FOR_tbm_bextri_di);
39309 if (!CONST_INT_P (op1))
39311 error ("last argument must be an immediate");
39316 unsigned char length = (INTVAL (op1) >> 8) & 0xFF;
39317 unsigned char lsb_index = INTVAL (op1) & 0xFF;
39318 op1 = GEN_INT (length);
39319 op2 = GEN_INT (lsb_index);
39320 pat = GEN_FCN (icode) (target, op0, op1, op2);
39326 case IX86_BUILTIN_RDRAND16_STEP:
39327 icode = CODE_FOR_rdrandhi_1;
39331 case IX86_BUILTIN_RDRAND32_STEP:
39332 icode = CODE_FOR_rdrandsi_1;
39336 case IX86_BUILTIN_RDRAND64_STEP:
39337 icode = CODE_FOR_rdranddi_1;
39341 op0 = gen_reg_rtx (mode0);
39342 emit_insn (GEN_FCN (icode) (op0));
39344 arg0 = CALL_EXPR_ARG (exp, 0);
39345 op1 = expand_normal (arg0);
39346 if (!address_operand (op1, VOIDmode))
39348 op1 = convert_memory_address (Pmode, op1);
39349 op1 = copy_addr_to_reg (op1);
39351 emit_move_insn (gen_rtx_MEM (mode0, op1), op0);
39353 op1 = gen_reg_rtx (SImode);
39354 emit_move_insn (op1, CONST1_RTX (SImode));
39356 /* Emit SImode conditional move. */
39357 if (mode0 == HImode)
39359 op2 = gen_reg_rtx (SImode);
39360 emit_insn (gen_zero_extendhisi2 (op2, op0));
39362 else if (mode0 == SImode)
39365 op2 = gen_rtx_SUBREG (SImode, op0, 0);
39368 || !register_operand (target, SImode))
39369 target = gen_reg_rtx (SImode);
39371 pat = gen_rtx_GEU (VOIDmode, gen_rtx_REG (CCCmode, FLAGS_REG),
39373 emit_insn (gen_rtx_SET (target,
39374 gen_rtx_IF_THEN_ELSE (SImode, pat, op2, op1)));
39377 case IX86_BUILTIN_RDSEED16_STEP:
39378 icode = CODE_FOR_rdseedhi_1;
39382 case IX86_BUILTIN_RDSEED32_STEP:
39383 icode = CODE_FOR_rdseedsi_1;
39387 case IX86_BUILTIN_RDSEED64_STEP:
39388 icode = CODE_FOR_rdseeddi_1;
39392 op0 = gen_reg_rtx (mode0);
39393 emit_insn (GEN_FCN (icode) (op0));
39395 arg0 = CALL_EXPR_ARG (exp, 0);
39396 op1 = expand_normal (arg0);
39397 if (!address_operand (op1, VOIDmode))
39399 op1 = convert_memory_address (Pmode, op1);
39400 op1 = copy_addr_to_reg (op1);
39402 emit_move_insn (gen_rtx_MEM (mode0, op1), op0);
39404 op2 = gen_reg_rtx (QImode);
39406 pat = gen_rtx_LTU (QImode, gen_rtx_REG (CCCmode, FLAGS_REG),
39408 emit_insn (gen_rtx_SET (op2, pat));
39411 || !register_operand (target, SImode))
39412 target = gen_reg_rtx (SImode);
39414 emit_insn (gen_zero_extendqisi2 (target, op2));
39417 case IX86_BUILTIN_SBB32:
39418 icode = CODE_FOR_subsi3_carry;
39422 case IX86_BUILTIN_SBB64:
39423 icode = CODE_FOR_subdi3_carry;
39427 case IX86_BUILTIN_ADDCARRYX32:
39428 icode = TARGET_ADX ? CODE_FOR_adcxsi3 : CODE_FOR_addsi3_carry;
39432 case IX86_BUILTIN_ADDCARRYX64:
39433 icode = TARGET_ADX ? CODE_FOR_adcxdi3 : CODE_FOR_adddi3_carry;
39437 arg0 = CALL_EXPR_ARG (exp, 0); /* unsigned char c_in. */
39438 arg1 = CALL_EXPR_ARG (exp, 1); /* unsigned int src1. */
39439 arg2 = CALL_EXPR_ARG (exp, 2); /* unsigned int src2. */
39440 arg3 = CALL_EXPR_ARG (exp, 3); /* unsigned int *sum_out. */
39442 op0 = gen_reg_rtx (QImode);
39444 /* Generate CF from input operand. */
39445 op1 = expand_normal (arg0);
39446 op1 = copy_to_mode_reg (QImode, convert_to_mode (QImode, op1, 1));
39447 emit_insn (gen_addqi3_cc (op0, op1, constm1_rtx));
39449 /* Gen ADCX instruction to compute X+Y+CF. */
39450 op2 = expand_normal (arg1);
39451 op3 = expand_normal (arg2);
39454 op2 = copy_to_mode_reg (mode0, op2);
39456 op3 = copy_to_mode_reg (mode0, op3);
39458 op0 = gen_reg_rtx (mode0);
39460 op4 = gen_rtx_REG (CCCmode, FLAGS_REG);
39461 pat = gen_rtx_LTU (VOIDmode, op4, const0_rtx);
39462 emit_insn (GEN_FCN (icode) (op0, op2, op3, op4, pat));
39464 /* Store the result. */
39465 op4 = expand_normal (arg3);
39466 if (!address_operand (op4, VOIDmode))
39468 op4 = convert_memory_address (Pmode, op4);
39469 op4 = copy_addr_to_reg (op4);
39471 emit_move_insn (gen_rtx_MEM (mode0, op4), op0);
39473 /* Return current CF value. */
39475 target = gen_reg_rtx (QImode);
39477 PUT_MODE (pat, QImode);
39478 emit_insn (gen_rtx_SET (target, pat));
39481 case IX86_BUILTIN_READ_FLAGS:
39482 emit_insn (gen_push (gen_rtx_REG (word_mode, FLAGS_REG)));
39485 || target == NULL_RTX
39486 || !nonimmediate_operand (target, word_mode)
39487 || GET_MODE (target) != word_mode)
39488 target = gen_reg_rtx (word_mode);
39490 emit_insn (gen_pop (target));
39493 case IX86_BUILTIN_WRITE_FLAGS:
39495 arg0 = CALL_EXPR_ARG (exp, 0);
39496 op0 = expand_normal (arg0);
39497 if (!general_no_elim_operand (op0, word_mode))
39498 op0 = copy_to_mode_reg (word_mode, op0);
39500 emit_insn (gen_push (op0));
39501 emit_insn (gen_pop (gen_rtx_REG (word_mode, FLAGS_REG)));
39504 case IX86_BUILTIN_KORTESTC16:
39505 icode = CODE_FOR_kortestchi;
39510 case IX86_BUILTIN_KORTESTZ16:
39511 icode = CODE_FOR_kortestzhi;
39516 arg0 = CALL_EXPR_ARG (exp, 0); /* Mask reg src1. */
39517 arg1 = CALL_EXPR_ARG (exp, 1); /* Mask reg src2. */
39518 op0 = expand_normal (arg0);
39519 op1 = expand_normal (arg1);
39521 op0 = copy_to_reg (op0);
39522 op0 = simplify_gen_subreg (mode0, op0, GET_MODE (op0), 0);
39523 op1 = copy_to_reg (op1);
39524 op1 = simplify_gen_subreg (mode0, op1, GET_MODE (op1), 0);
39526 target = gen_reg_rtx (QImode);
39527 emit_insn (gen_rtx_SET (target, const0_rtx));
39529 /* Emit kortest. */
39530 emit_insn (GEN_FCN (icode) (op0, op1));
39531 /* And use setcc to return result from flags. */
39532 ix86_expand_setcc (target, EQ,
39533 gen_rtx_REG (mode1, FLAGS_REG), const0_rtx);
39536 case IX86_BUILTIN_GATHERSIV2DF:
39537 icode = CODE_FOR_avx2_gathersiv2df;
39539 case IX86_BUILTIN_GATHERSIV4DF:
39540 icode = CODE_FOR_avx2_gathersiv4df;
39542 case IX86_BUILTIN_GATHERDIV2DF:
39543 icode = CODE_FOR_avx2_gatherdiv2df;
39545 case IX86_BUILTIN_GATHERDIV4DF:
39546 icode = CODE_FOR_avx2_gatherdiv4df;
39548 case IX86_BUILTIN_GATHERSIV4SF:
39549 icode = CODE_FOR_avx2_gathersiv4sf;
39551 case IX86_BUILTIN_GATHERSIV8SF:
39552 icode = CODE_FOR_avx2_gathersiv8sf;
39554 case IX86_BUILTIN_GATHERDIV4SF:
39555 icode = CODE_FOR_avx2_gatherdiv4sf;
39557 case IX86_BUILTIN_GATHERDIV8SF:
39558 icode = CODE_FOR_avx2_gatherdiv8sf;
39560 case IX86_BUILTIN_GATHERSIV2DI:
39561 icode = CODE_FOR_avx2_gathersiv2di;
39563 case IX86_BUILTIN_GATHERSIV4DI:
39564 icode = CODE_FOR_avx2_gathersiv4di;
39566 case IX86_BUILTIN_GATHERDIV2DI:
39567 icode = CODE_FOR_avx2_gatherdiv2di;
39569 case IX86_BUILTIN_GATHERDIV4DI:
39570 icode = CODE_FOR_avx2_gatherdiv4di;
39572 case IX86_BUILTIN_GATHERSIV4SI:
39573 icode = CODE_FOR_avx2_gathersiv4si;
39575 case IX86_BUILTIN_GATHERSIV8SI:
39576 icode = CODE_FOR_avx2_gathersiv8si;
39578 case IX86_BUILTIN_GATHERDIV4SI:
39579 icode = CODE_FOR_avx2_gatherdiv4si;
39581 case IX86_BUILTIN_GATHERDIV8SI:
39582 icode = CODE_FOR_avx2_gatherdiv8si;
39584 case IX86_BUILTIN_GATHERALTSIV4DF:
39585 icode = CODE_FOR_avx2_gathersiv4df;
39587 case IX86_BUILTIN_GATHERALTDIV8SF:
39588 icode = CODE_FOR_avx2_gatherdiv8sf;
39590 case IX86_BUILTIN_GATHERALTSIV4DI:
39591 icode = CODE_FOR_avx2_gathersiv4di;
39593 case IX86_BUILTIN_GATHERALTDIV8SI:
39594 icode = CODE_FOR_avx2_gatherdiv8si;
39596 case IX86_BUILTIN_GATHER3SIV16SF:
39597 icode = CODE_FOR_avx512f_gathersiv16sf;
39599 case IX86_BUILTIN_GATHER3SIV8DF:
39600 icode = CODE_FOR_avx512f_gathersiv8df;
39602 case IX86_BUILTIN_GATHER3DIV16SF:
39603 icode = CODE_FOR_avx512f_gatherdiv16sf;
39605 case IX86_BUILTIN_GATHER3DIV8DF:
39606 icode = CODE_FOR_avx512f_gatherdiv8df;
39608 case IX86_BUILTIN_GATHER3SIV16SI:
39609 icode = CODE_FOR_avx512f_gathersiv16si;
39611 case IX86_BUILTIN_GATHER3SIV8DI:
39612 icode = CODE_FOR_avx512f_gathersiv8di;
39614 case IX86_BUILTIN_GATHER3DIV16SI:
39615 icode = CODE_FOR_avx512f_gatherdiv16si;
39617 case IX86_BUILTIN_GATHER3DIV8DI:
39618 icode = CODE_FOR_avx512f_gatherdiv8di;
39620 case IX86_BUILTIN_GATHER3ALTSIV8DF:
39621 icode = CODE_FOR_avx512f_gathersiv8df;
39623 case IX86_BUILTIN_GATHER3ALTDIV16SF:
39624 icode = CODE_FOR_avx512f_gatherdiv16sf;
39626 case IX86_BUILTIN_GATHER3ALTSIV8DI:
39627 icode = CODE_FOR_avx512f_gathersiv8di;
39629 case IX86_BUILTIN_GATHER3ALTDIV16SI:
39630 icode = CODE_FOR_avx512f_gatherdiv16si;
39632 case IX86_BUILTIN_GATHER3SIV2DF:
39633 icode = CODE_FOR_avx512vl_gathersiv2df;
39635 case IX86_BUILTIN_GATHER3SIV4DF:
39636 icode = CODE_FOR_avx512vl_gathersiv4df;
39638 case IX86_BUILTIN_GATHER3DIV2DF:
39639 icode = CODE_FOR_avx512vl_gatherdiv2df;
39641 case IX86_BUILTIN_GATHER3DIV4DF:
39642 icode = CODE_FOR_avx512vl_gatherdiv4df;
39644 case IX86_BUILTIN_GATHER3SIV4SF:
39645 icode = CODE_FOR_avx512vl_gathersiv4sf;
39647 case IX86_BUILTIN_GATHER3SIV8SF:
39648 icode = CODE_FOR_avx512vl_gathersiv8sf;
39650 case IX86_BUILTIN_GATHER3DIV4SF:
39651 icode = CODE_FOR_avx512vl_gatherdiv4sf;
39653 case IX86_BUILTIN_GATHER3DIV8SF:
39654 icode = CODE_FOR_avx512vl_gatherdiv8sf;
39656 case IX86_BUILTIN_GATHER3SIV2DI:
39657 icode = CODE_FOR_avx512vl_gathersiv2di;
39659 case IX86_BUILTIN_GATHER3SIV4DI:
39660 icode = CODE_FOR_avx512vl_gathersiv4di;
39662 case IX86_BUILTIN_GATHER3DIV2DI:
39663 icode = CODE_FOR_avx512vl_gatherdiv2di;
39665 case IX86_BUILTIN_GATHER3DIV4DI:
39666 icode = CODE_FOR_avx512vl_gatherdiv4di;
39668 case IX86_BUILTIN_GATHER3SIV4SI:
39669 icode = CODE_FOR_avx512vl_gathersiv4si;
39671 case IX86_BUILTIN_GATHER3SIV8SI:
39672 icode = CODE_FOR_avx512vl_gathersiv8si;
39674 case IX86_BUILTIN_GATHER3DIV4SI:
39675 icode = CODE_FOR_avx512vl_gatherdiv4si;
39677 case IX86_BUILTIN_GATHER3DIV8SI:
39678 icode = CODE_FOR_avx512vl_gatherdiv8si;
39680 case IX86_BUILTIN_GATHER3ALTSIV4DF:
39681 icode = CODE_FOR_avx512vl_gathersiv4df;
39683 case IX86_BUILTIN_GATHER3ALTDIV8SF:
39684 icode = CODE_FOR_avx512vl_gatherdiv8sf;
39686 case IX86_BUILTIN_GATHER3ALTSIV4DI:
39687 icode = CODE_FOR_avx512vl_gathersiv4di;
39689 case IX86_BUILTIN_GATHER3ALTDIV8SI:
39690 icode = CODE_FOR_avx512vl_gatherdiv8si;
39692 case IX86_BUILTIN_SCATTERSIV16SF:
39693 icode = CODE_FOR_avx512f_scattersiv16sf;
39695 case IX86_BUILTIN_SCATTERSIV8DF:
39696 icode = CODE_FOR_avx512f_scattersiv8df;
39698 case IX86_BUILTIN_SCATTERDIV16SF:
39699 icode = CODE_FOR_avx512f_scatterdiv16sf;
39701 case IX86_BUILTIN_SCATTERDIV8DF:
39702 icode = CODE_FOR_avx512f_scatterdiv8df;
39704 case IX86_BUILTIN_SCATTERSIV16SI:
39705 icode = CODE_FOR_avx512f_scattersiv16si;
39707 case IX86_BUILTIN_SCATTERSIV8DI:
39708 icode = CODE_FOR_avx512f_scattersiv8di;
39710 case IX86_BUILTIN_SCATTERDIV16SI:
39711 icode = CODE_FOR_avx512f_scatterdiv16si;
39713 case IX86_BUILTIN_SCATTERDIV8DI:
39714 icode = CODE_FOR_avx512f_scatterdiv8di;
39716 case IX86_BUILTIN_SCATTERSIV8SF:
39717 icode = CODE_FOR_avx512vl_scattersiv8sf;
39719 case IX86_BUILTIN_SCATTERSIV4SF:
39720 icode = CODE_FOR_avx512vl_scattersiv4sf;
39722 case IX86_BUILTIN_SCATTERSIV4DF:
39723 icode = CODE_FOR_avx512vl_scattersiv4df;
39725 case IX86_BUILTIN_SCATTERSIV2DF:
39726 icode = CODE_FOR_avx512vl_scattersiv2df;
39728 case IX86_BUILTIN_SCATTERDIV8SF:
39729 icode = CODE_FOR_avx512vl_scatterdiv8sf;
39731 case IX86_BUILTIN_SCATTERDIV4SF:
39732 icode = CODE_FOR_avx512vl_scatterdiv4sf;
39734 case IX86_BUILTIN_SCATTERDIV4DF:
39735 icode = CODE_FOR_avx512vl_scatterdiv4df;
39737 case IX86_BUILTIN_SCATTERDIV2DF:
39738 icode = CODE_FOR_avx512vl_scatterdiv2df;
39740 case IX86_BUILTIN_SCATTERSIV8SI:
39741 icode = CODE_FOR_avx512vl_scattersiv8si;
39743 case IX86_BUILTIN_SCATTERSIV4SI:
39744 icode = CODE_FOR_avx512vl_scattersiv4si;
39746 case IX86_BUILTIN_SCATTERSIV4DI:
39747 icode = CODE_FOR_avx512vl_scattersiv4di;
39749 case IX86_BUILTIN_SCATTERSIV2DI:
39750 icode = CODE_FOR_avx512vl_scattersiv2di;
39752 case IX86_BUILTIN_SCATTERDIV8SI:
39753 icode = CODE_FOR_avx512vl_scatterdiv8si;
39755 case IX86_BUILTIN_SCATTERDIV4SI:
39756 icode = CODE_FOR_avx512vl_scatterdiv4si;
39758 case IX86_BUILTIN_SCATTERDIV4DI:
39759 icode = CODE_FOR_avx512vl_scatterdiv4di;
39761 case IX86_BUILTIN_SCATTERDIV2DI:
39762 icode = CODE_FOR_avx512vl_scatterdiv2di;
39764 case IX86_BUILTIN_GATHERPFDPD:
39765 icode = CODE_FOR_avx512pf_gatherpfv8sidf;
39766 goto vec_prefetch_gen;
39767 case IX86_BUILTIN_GATHERPFDPS:
39768 icode = CODE_FOR_avx512pf_gatherpfv16sisf;
39769 goto vec_prefetch_gen;
39770 case IX86_BUILTIN_GATHERPFQPD:
39771 icode = CODE_FOR_avx512pf_gatherpfv8didf;
39772 goto vec_prefetch_gen;
39773 case IX86_BUILTIN_GATHERPFQPS:
39774 icode = CODE_FOR_avx512pf_gatherpfv8disf;
39775 goto vec_prefetch_gen;
39776 case IX86_BUILTIN_SCATTERPFDPD:
39777 icode = CODE_FOR_avx512pf_scatterpfv8sidf;
39778 goto vec_prefetch_gen;
39779 case IX86_BUILTIN_SCATTERPFDPS:
39780 icode = CODE_FOR_avx512pf_scatterpfv16sisf;
39781 goto vec_prefetch_gen;
39782 case IX86_BUILTIN_SCATTERPFQPD:
39783 icode = CODE_FOR_avx512pf_scatterpfv8didf;
39784 goto vec_prefetch_gen;
39785 case IX86_BUILTIN_SCATTERPFQPS:
39786 icode = CODE_FOR_avx512pf_scatterpfv8disf;
39787 goto vec_prefetch_gen;
39791 rtx (*gen) (rtx, rtx);
39793 arg0 = CALL_EXPR_ARG (exp, 0);
39794 arg1 = CALL_EXPR_ARG (exp, 1);
39795 arg2 = CALL_EXPR_ARG (exp, 2);
39796 arg3 = CALL_EXPR_ARG (exp, 3);
39797 arg4 = CALL_EXPR_ARG (exp, 4);
39798 op0 = expand_normal (arg0);
39799 op1 = expand_normal (arg1);
39800 op2 = expand_normal (arg2);
39801 op3 = expand_normal (arg3);
39802 op4 = expand_normal (arg4);
39803 /* Note the arg order is different from the operand order. */
39804 mode0 = insn_data[icode].operand[1].mode;
39805 mode2 = insn_data[icode].operand[3].mode;
39806 mode3 = insn_data[icode].operand[4].mode;
39807 mode4 = insn_data[icode].operand[5].mode;
39809 if (target == NULL_RTX
39810 || GET_MODE (target) != insn_data[icode].operand[0].mode
39811 || !insn_data[icode].operand[0].predicate (target,
39812 GET_MODE (target)))
39813 subtarget = gen_reg_rtx (insn_data[icode].operand[0].mode);
39815 subtarget = target;
39819 case IX86_BUILTIN_GATHER3ALTSIV8DF:
39820 case IX86_BUILTIN_GATHER3ALTSIV8DI:
39821 half = gen_reg_rtx (V8SImode);
39822 if (!nonimmediate_operand (op2, V16SImode))
39823 op2 = copy_to_mode_reg (V16SImode, op2);
39824 emit_insn (gen_vec_extract_lo_v16si (half, op2));
39827 case IX86_BUILTIN_GATHER3ALTSIV4DF:
39828 case IX86_BUILTIN_GATHER3ALTSIV4DI:
39829 case IX86_BUILTIN_GATHERALTSIV4DF:
39830 case IX86_BUILTIN_GATHERALTSIV4DI:
39831 half = gen_reg_rtx (V4SImode);
39832 if (!nonimmediate_operand (op2, V8SImode))
39833 op2 = copy_to_mode_reg (V8SImode, op2);
39834 emit_insn (gen_vec_extract_lo_v8si (half, op2));
39837 case IX86_BUILTIN_GATHER3ALTDIV16SF:
39838 case IX86_BUILTIN_GATHER3ALTDIV16SI:
39839 half = gen_reg_rtx (mode0);
39840 if (mode0 == V8SFmode)
39841 gen = gen_vec_extract_lo_v16sf;
39843 gen = gen_vec_extract_lo_v16si;
39844 if (!nonimmediate_operand (op0, GET_MODE (op0)))
39845 op0 = copy_to_mode_reg (GET_MODE (op0), op0);
39846 emit_insn (gen (half, op0));
39848 if (GET_MODE (op3) != VOIDmode)
39850 if (!nonimmediate_operand (op3, GET_MODE (op3)))
39851 op3 = copy_to_mode_reg (GET_MODE (op3), op3);
39852 emit_insn (gen (half, op3));
39856 case IX86_BUILTIN_GATHER3ALTDIV8SF:
39857 case IX86_BUILTIN_GATHER3ALTDIV8SI:
39858 case IX86_BUILTIN_GATHERALTDIV8SF:
39859 case IX86_BUILTIN_GATHERALTDIV8SI:
39860 half = gen_reg_rtx (mode0);
39861 if (mode0 == V4SFmode)
39862 gen = gen_vec_extract_lo_v8sf;
39864 gen = gen_vec_extract_lo_v8si;
39865 if (!nonimmediate_operand (op0, GET_MODE (op0)))
39866 op0 = copy_to_mode_reg (GET_MODE (op0), op0);
39867 emit_insn (gen (half, op0));
39869 if (GET_MODE (op3) != VOIDmode)
39871 if (!nonimmediate_operand (op3, GET_MODE (op3)))
39872 op3 = copy_to_mode_reg (GET_MODE (op3), op3);
39873 emit_insn (gen (half, op3));
39881 /* Force memory operand only with base register here. But we
39882 don't want to do it on memory operand for other builtin
39884 op1 = ix86_zero_extend_to_Pmode (op1);
39886 if (!insn_data[icode].operand[1].predicate (op0, mode0))
39887 op0 = copy_to_mode_reg (mode0, op0);
39888 if (!insn_data[icode].operand[2].predicate (op1, Pmode))
39889 op1 = copy_to_mode_reg (Pmode, op1);
39890 if (!insn_data[icode].operand[3].predicate (op2, mode2))
39891 op2 = copy_to_mode_reg (mode2, op2);
39893 op3 = fixup_modeless_constant (op3, mode3);
39895 if (GET_MODE (op3) == mode3 || GET_MODE (op3) == VOIDmode)
39897 if (!insn_data[icode].operand[4].predicate (op3, mode3))
39898 op3 = copy_to_mode_reg (mode3, op3);
39902 op3 = copy_to_reg (op3);
39903 op3 = simplify_gen_subreg (mode3, op3, GET_MODE (op3), 0);
39905 if (!insn_data[icode].operand[5].predicate (op4, mode4))
39907 error ("the last argument must be scale 1, 2, 4, 8");
39911 /* Optimize. If mask is known to have all high bits set,
39912 replace op0 with pc_rtx to signal that the instruction
39913 overwrites the whole destination and doesn't use its
39914 previous contents. */
39917 if (TREE_CODE (arg3) == INTEGER_CST)
39919 if (integer_all_onesp (arg3))
39922 else if (TREE_CODE (arg3) == VECTOR_CST)
39924 unsigned int negative = 0;
39925 for (i = 0; i < VECTOR_CST_NELTS (arg3); ++i)
39927 tree cst = VECTOR_CST_ELT (arg3, i);
39928 if (TREE_CODE (cst) == INTEGER_CST
39929 && tree_int_cst_sign_bit (cst))
39931 else if (TREE_CODE (cst) == REAL_CST
39932 && REAL_VALUE_NEGATIVE (TREE_REAL_CST (cst)))
39935 if (negative == TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg3)))
39938 else if (TREE_CODE (arg3) == SSA_NAME
39939 && TREE_CODE (TREE_TYPE (arg3)) == VECTOR_TYPE)
39941 /* Recognize also when mask is like:
39942 __v2df src = _mm_setzero_pd ();
39943 __v2df mask = _mm_cmpeq_pd (src, src);
39945 __v8sf src = _mm256_setzero_ps ();
39946 __v8sf mask = _mm256_cmp_ps (src, src, _CMP_EQ_OQ);
39947 as that is a cheaper way to load all ones into
39948 a register than having to load a constant from
39950 gimple def_stmt = SSA_NAME_DEF_STMT (arg3);
39951 if (is_gimple_call (def_stmt))
39953 tree fndecl = gimple_call_fndecl (def_stmt);
39955 && DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_MD)
39956 switch ((unsigned int) DECL_FUNCTION_CODE (fndecl))
39958 case IX86_BUILTIN_CMPPD:
39959 case IX86_BUILTIN_CMPPS:
39960 case IX86_BUILTIN_CMPPD256:
39961 case IX86_BUILTIN_CMPPS256:
39962 if (!integer_zerop (gimple_call_arg (def_stmt, 2)))
39965 case IX86_BUILTIN_CMPEQPD:
39966 case IX86_BUILTIN_CMPEQPS:
39967 if (initializer_zerop (gimple_call_arg (def_stmt, 0))
39968 && initializer_zerop (gimple_call_arg (def_stmt,
39979 pat = GEN_FCN (icode) (subtarget, op0, op1, op2, op3, op4);
39986 case IX86_BUILTIN_GATHER3DIV16SF:
39987 if (target == NULL_RTX)
39988 target = gen_reg_rtx (V8SFmode);
39989 emit_insn (gen_vec_extract_lo_v16sf (target, subtarget));
39991 case IX86_BUILTIN_GATHER3DIV16SI:
39992 if (target == NULL_RTX)
39993 target = gen_reg_rtx (V8SImode);
39994 emit_insn (gen_vec_extract_lo_v16si (target, subtarget));
39996 case IX86_BUILTIN_GATHER3DIV8SF:
39997 case IX86_BUILTIN_GATHERDIV8SF:
39998 if (target == NULL_RTX)
39999 target = gen_reg_rtx (V4SFmode);
40000 emit_insn (gen_vec_extract_lo_v8sf (target, subtarget));
40002 case IX86_BUILTIN_GATHER3DIV8SI:
40003 case IX86_BUILTIN_GATHERDIV8SI:
40004 if (target == NULL_RTX)
40005 target = gen_reg_rtx (V4SImode);
40006 emit_insn (gen_vec_extract_lo_v8si (target, subtarget));
40009 target = subtarget;
40015 arg0 = CALL_EXPR_ARG (exp, 0);
40016 arg1 = CALL_EXPR_ARG (exp, 1);
40017 arg2 = CALL_EXPR_ARG (exp, 2);
40018 arg3 = CALL_EXPR_ARG (exp, 3);
40019 arg4 = CALL_EXPR_ARG (exp, 4);
40020 op0 = expand_normal (arg0);
40021 op1 = expand_normal (arg1);
40022 op2 = expand_normal (arg2);
40023 op3 = expand_normal (arg3);
40024 op4 = expand_normal (arg4);
40025 mode1 = insn_data[icode].operand[1].mode;
40026 mode2 = insn_data[icode].operand[2].mode;
40027 mode3 = insn_data[icode].operand[3].mode;
40028 mode4 = insn_data[icode].operand[4].mode;
40030 /* Force memory operand only with base register here. But we
40031 don't want to do it on memory operand for other builtin
40033 op0 = force_reg (Pmode, convert_to_mode (Pmode, op0, 1));
40035 if (!insn_data[icode].operand[0].predicate (op0, Pmode))
40036 op0 = copy_to_mode_reg (Pmode, op0);
40038 op1 = fixup_modeless_constant (op1, mode1);
40040 if (GET_MODE (op1) == mode1 || GET_MODE (op1) == VOIDmode)
40042 if (!insn_data[icode].operand[1].predicate (op1, mode1))
40043 op1 = copy_to_mode_reg (mode1, op1);
40047 op1 = copy_to_reg (op1);
40048 op1 = simplify_gen_subreg (mode1, op1, GET_MODE (op1), 0);
40051 if (!insn_data[icode].operand[2].predicate (op2, mode2))
40052 op2 = copy_to_mode_reg (mode2, op2);
40054 if (!insn_data[icode].operand[3].predicate (op3, mode3))
40055 op3 = copy_to_mode_reg (mode3, op3);
40057 if (!insn_data[icode].operand[4].predicate (op4, mode4))
40059 error ("the last argument must be scale 1, 2, 4, 8");
40063 pat = GEN_FCN (icode) (op0, op1, op2, op3, op4);
40071 arg0 = CALL_EXPR_ARG (exp, 0);
40072 arg1 = CALL_EXPR_ARG (exp, 1);
40073 arg2 = CALL_EXPR_ARG (exp, 2);
40074 arg3 = CALL_EXPR_ARG (exp, 3);
40075 arg4 = CALL_EXPR_ARG (exp, 4);
40076 op0 = expand_normal (arg0);
40077 op1 = expand_normal (arg1);
40078 op2 = expand_normal (arg2);
40079 op3 = expand_normal (arg3);
40080 op4 = expand_normal (arg4);
40081 mode0 = insn_data[icode].operand[0].mode;
40082 mode1 = insn_data[icode].operand[1].mode;
40083 mode3 = insn_data[icode].operand[3].mode;
40084 mode4 = insn_data[icode].operand[4].mode;
40086 op0 = fixup_modeless_constant (op0, mode0);
40088 if (GET_MODE (op0) == mode0
40089 || (GET_MODE (op0) == VOIDmode && op0 != constm1_rtx))
40091 if (!insn_data[icode].operand[0].predicate (op0, mode0))
40092 op0 = copy_to_mode_reg (mode0, op0);
40094 else if (op0 != constm1_rtx)
40096 op0 = copy_to_reg (op0);
40097 op0 = simplify_gen_subreg (mode0, op0, GET_MODE (op0), 0);
40100 if (!insn_data[icode].operand[1].predicate (op1, mode1))
40101 op1 = copy_to_mode_reg (mode1, op1);
40103 /* Force memory operand only with base register here. But we
40104 don't want to do it on memory operand for other builtin
40106 op2 = force_reg (Pmode, convert_to_mode (Pmode, op2, 1));
40108 if (!insn_data[icode].operand[2].predicate (op2, Pmode))
40109 op2 = copy_to_mode_reg (Pmode, op2);
40111 if (!insn_data[icode].operand[3].predicate (op3, mode3))
40113 error ("the forth argument must be scale 1, 2, 4, 8");
40117 if (!insn_data[icode].operand[4].predicate (op4, mode4))
40119 error ("incorrect hint operand");
40123 pat = GEN_FCN (icode) (op0, op1, op2, op3, op4);
40131 case IX86_BUILTIN_XABORT:
40132 icode = CODE_FOR_xabort;
40133 arg0 = CALL_EXPR_ARG (exp, 0);
40134 op0 = expand_normal (arg0);
40135 mode0 = insn_data[icode].operand[0].mode;
40136 if (!insn_data[icode].operand[0].predicate (op0, mode0))
40138 error ("the xabort's argument must be an 8-bit immediate");
40141 emit_insn (gen_xabort (op0));
40148 for (i = 0, d = bdesc_special_args;
40149 i < ARRAY_SIZE (bdesc_special_args);
40151 if (d->code == fcode)
40152 return ix86_expand_special_args_builtin (d, exp, target);
40154 for (i = 0, d = bdesc_args;
40155 i < ARRAY_SIZE (bdesc_args);
40157 if (d->code == fcode)
40160 case IX86_BUILTIN_FABSQ:
40161 case IX86_BUILTIN_COPYSIGNQ:
40163 /* Emit a normal call if SSE isn't available. */
40164 return expand_call (exp, target, ignore);
40166 return ix86_expand_args_builtin (d, exp, target);
40169 for (i = 0, d = bdesc_comi; i < ARRAY_SIZE (bdesc_comi); i++, d++)
40170 if (d->code == fcode)
40171 return ix86_expand_sse_comi (d, exp, target);
40173 for (i = 0, d = bdesc_round_args; i < ARRAY_SIZE (bdesc_round_args); i++, d++)
40174 if (d->code == fcode)
40175 return ix86_expand_round_builtin (d, exp, target);
40177 for (i = 0, d = bdesc_pcmpestr;
40178 i < ARRAY_SIZE (bdesc_pcmpestr);
40180 if (d->code == fcode)
40181 return ix86_expand_sse_pcmpestr (d, exp, target);
40183 for (i = 0, d = bdesc_pcmpistr;
40184 i < ARRAY_SIZE (bdesc_pcmpistr);
40186 if (d->code == fcode)
40187 return ix86_expand_sse_pcmpistr (d, exp, target);
40189 for (i = 0, d = bdesc_multi_arg; i < ARRAY_SIZE (bdesc_multi_arg); i++, d++)
40190 if (d->code == fcode)
40191 return ix86_expand_multi_arg_builtin (d->icode, exp, target,
40192 (enum ix86_builtin_func_type)
40193 d->flag, d->comparison);
40195 gcc_unreachable ();
40198 /* This returns the target-specific builtin with code CODE if
40199 current_function_decl has visibility on this builtin, which is checked
40200 using isa flags. Returns NULL_TREE otherwise. */
40202 static tree ix86_get_builtin (enum ix86_builtins code)
40204 struct cl_target_option *opts;
40205 tree target_tree = NULL_TREE;
40207 /* Determine the isa flags of current_function_decl. */
40209 if (current_function_decl)
40210 target_tree = DECL_FUNCTION_SPECIFIC_TARGET (current_function_decl);
40212 if (target_tree == NULL)
40213 target_tree = target_option_default_node;
40215 opts = TREE_TARGET_OPTION (target_tree);
40217 if (ix86_builtins_isa[(int) code].isa & opts->x_ix86_isa_flags)
40218 return ix86_builtin_decl (code, true);
40223 /* Return function decl for target specific builtin
40224 for given MPX builtin passed i FCODE. */
40226 ix86_builtin_mpx_function (unsigned fcode)
40230 case BUILT_IN_CHKP_BNDMK:
40231 return ix86_builtins[IX86_BUILTIN_BNDMK];
40233 case BUILT_IN_CHKP_BNDSTX:
40234 return ix86_builtins[IX86_BUILTIN_BNDSTX];
40236 case BUILT_IN_CHKP_BNDLDX:
40237 return ix86_builtins[IX86_BUILTIN_BNDLDX];
40239 case BUILT_IN_CHKP_BNDCL:
40240 return ix86_builtins[IX86_BUILTIN_BNDCL];
40242 case BUILT_IN_CHKP_BNDCU:
40243 return ix86_builtins[IX86_BUILTIN_BNDCU];
40245 case BUILT_IN_CHKP_BNDRET:
40246 return ix86_builtins[IX86_BUILTIN_BNDRET];
40248 case BUILT_IN_CHKP_INTERSECT:
40249 return ix86_builtins[IX86_BUILTIN_BNDINT];
40251 case BUILT_IN_CHKP_NARROW:
40252 return ix86_builtins[IX86_BUILTIN_BNDNARROW];
40254 case BUILT_IN_CHKP_SIZEOF:
40255 return ix86_builtins[IX86_BUILTIN_SIZEOF];
40257 case BUILT_IN_CHKP_EXTRACT_LOWER:
40258 return ix86_builtins[IX86_BUILTIN_BNDLOWER];
40260 case BUILT_IN_CHKP_EXTRACT_UPPER:
40261 return ix86_builtins[IX86_BUILTIN_BNDUPPER];
40267 gcc_unreachable ();
40270 /* Helper function for ix86_load_bounds and ix86_store_bounds.
40272 Return an address to be used to load/store bounds for pointer
40275 SLOT_NO is an integer constant holding number of a target
40276 dependent special slot to be used in case SLOT is not a memory.
40278 SPECIAL_BASE is a pointer to be used as a base of fake address
40279 to access special slots in Bounds Table. SPECIAL_BASE[-1],
40280 SPECIAL_BASE[-2] etc. will be used as fake pointer locations. */
40283 ix86_get_arg_address_for_bt (rtx slot, rtx slot_no, rtx special_base)
40287 /* NULL slot means we pass bounds for pointer not passed to the
40288 function at all. Register slot means we pass pointer in a
40289 register. In both these cases bounds are passed via Bounds
40290 Table. Since we do not have actual pointer stored in memory,
40291 we have to use fake addresses to access Bounds Table. We
40292 start with (special_base - sizeof (void*)) and decrease this
40293 address by pointer size to get addresses for other slots. */
40294 if (!slot || REG_P (slot))
40296 gcc_assert (CONST_INT_P (slot_no));
40297 addr = plus_constant (Pmode, special_base,
40298 -(INTVAL (slot_no) + 1) * GET_MODE_SIZE (Pmode));
40300 /* If pointer is passed in a memory then its address is used to
40301 access Bounds Table. */
40302 else if (MEM_P (slot))
40304 addr = XEXP (slot, 0);
40305 if (!register_operand (addr, Pmode))
40306 addr = copy_addr_to_reg (addr);
40309 gcc_unreachable ();
40314 /* Expand pass uses this hook to load bounds for function parameter
40315 PTR passed in SLOT in case its bounds are not passed in a register.
40317 If SLOT is a memory, then bounds are loaded as for regular pointer
40318 loaded from memory. PTR may be NULL in case SLOT is a memory.
40319 In such case value of PTR (if required) may be loaded from SLOT.
40321 If SLOT is NULL or a register then SLOT_NO is an integer constant
40322 holding number of the target dependent special slot which should be
40323 used to obtain bounds.
40325 Return loaded bounds. */
40328 ix86_load_bounds (rtx slot, rtx ptr, rtx slot_no)
40330 rtx reg = gen_reg_rtx (BNDmode);
40333 /* Get address to be used to access Bounds Table. Special slots start
40334 at the location of return address of the current function. */
40335 addr = ix86_get_arg_address_for_bt (slot, slot_no, arg_pointer_rtx);
40337 /* Load pointer value from a memory if we don't have it. */
40340 gcc_assert (MEM_P (slot));
40341 ptr = copy_addr_to_reg (slot);
40344 emit_insn (BNDmode == BND64mode
40345 ? gen_bnd64_ldx (reg, addr, ptr)
40346 : gen_bnd32_ldx (reg, addr, ptr));
40351 /* Expand pass uses this hook to store BOUNDS for call argument PTR
40352 passed in SLOT in case BOUNDS are not passed in a register.
40354 If SLOT is a memory, then BOUNDS are stored as for regular pointer
40355 stored in memory. PTR may be NULL in case SLOT is a memory.
40356 In such case value of PTR (if required) may be loaded from SLOT.
40358 If SLOT is NULL or a register then SLOT_NO is an integer constant
40359 holding number of the target dependent special slot which should be
40360 used to store BOUNDS. */
40363 ix86_store_bounds (rtx ptr, rtx slot, rtx bounds, rtx slot_no)
40367 /* Get address to be used to access Bounds Table. Special slots start
40368 at the location of return address of a called function. */
40369 addr = ix86_get_arg_address_for_bt (slot, slot_no, stack_pointer_rtx);
40371 /* Load pointer value from a memory if we don't have it. */
40374 gcc_assert (MEM_P (slot));
40375 ptr = copy_addr_to_reg (slot);
40378 gcc_assert (POINTER_BOUNDS_MODE_P (GET_MODE (bounds)));
40379 if (!register_operand (bounds, BNDmode))
40380 bounds = copy_to_mode_reg (BNDmode, bounds);
40382 emit_insn (BNDmode == BND64mode
40383 ? gen_bnd64_stx (addr, ptr, bounds)
40384 : gen_bnd32_stx (addr, ptr, bounds));
40387 /* Load and return bounds returned by function in SLOT. */
40390 ix86_load_returned_bounds (rtx slot)
40394 gcc_assert (REG_P (slot));
40395 res = gen_reg_rtx (BNDmode);
40396 emit_move_insn (res, slot);
40401 /* Store BOUNDS returned by function into SLOT. */
40404 ix86_store_returned_bounds (rtx slot, rtx bounds)
40406 gcc_assert (REG_P (slot));
40407 emit_move_insn (slot, bounds);
40410 /* Returns a function decl for a vectorized version of the builtin function
40411 with builtin function code FN and the result vector type TYPE, or NULL_TREE
40412 if it is not available. */
40415 ix86_builtin_vectorized_function (tree fndecl, tree type_out,
40418 machine_mode in_mode, out_mode;
40420 enum built_in_function fn = DECL_FUNCTION_CODE (fndecl);
40422 if (TREE_CODE (type_out) != VECTOR_TYPE
40423 || TREE_CODE (type_in) != VECTOR_TYPE
40424 || DECL_BUILT_IN_CLASS (fndecl) != BUILT_IN_NORMAL)
40427 out_mode = TYPE_MODE (TREE_TYPE (type_out));
40428 out_n = TYPE_VECTOR_SUBPARTS (type_out);
40429 in_mode = TYPE_MODE (TREE_TYPE (type_in));
40430 in_n = TYPE_VECTOR_SUBPARTS (type_in);
40434 case BUILT_IN_SQRT:
40435 if (out_mode == DFmode && in_mode == DFmode)
40437 if (out_n == 2 && in_n == 2)
40438 return ix86_get_builtin (IX86_BUILTIN_SQRTPD);
40439 else if (out_n == 4 && in_n == 4)
40440 return ix86_get_builtin (IX86_BUILTIN_SQRTPD256);
40441 else if (out_n == 8 && in_n == 8)
40442 return ix86_get_builtin (IX86_BUILTIN_SQRTPD512);
40446 case BUILT_IN_EXP2F:
40447 if (out_mode == SFmode && in_mode == SFmode)
40449 if (out_n == 16 && in_n == 16)
40450 return ix86_get_builtin (IX86_BUILTIN_EXP2PS);
40454 case BUILT_IN_SQRTF:
40455 if (out_mode == SFmode && in_mode == SFmode)
40457 if (out_n == 4 && in_n == 4)
40458 return ix86_get_builtin (IX86_BUILTIN_SQRTPS_NR);
40459 else if (out_n == 8 && in_n == 8)
40460 return ix86_get_builtin (IX86_BUILTIN_SQRTPS_NR256);
40461 else if (out_n == 16 && in_n == 16)
40462 return ix86_get_builtin (IX86_BUILTIN_SQRTPS_NR512);
40466 case BUILT_IN_IFLOOR:
40467 case BUILT_IN_LFLOOR:
40468 case BUILT_IN_LLFLOOR:
40469 /* The round insn does not trap on denormals. */
40470 if (flag_trapping_math || !TARGET_ROUND)
40473 if (out_mode == SImode && in_mode == DFmode)
40475 if (out_n == 4 && in_n == 2)
40476 return ix86_get_builtin (IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX);
40477 else if (out_n == 8 && in_n == 4)
40478 return ix86_get_builtin (IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX256);
40479 else if (out_n == 16 && in_n == 8)
40480 return ix86_get_builtin (IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX512);
40484 case BUILT_IN_IFLOORF:
40485 case BUILT_IN_LFLOORF:
40486 case BUILT_IN_LLFLOORF:
40487 /* The round insn does not trap on denormals. */
40488 if (flag_trapping_math || !TARGET_ROUND)
40491 if (out_mode == SImode && in_mode == SFmode)
40493 if (out_n == 4 && in_n == 4)
40494 return ix86_get_builtin (IX86_BUILTIN_FLOORPS_SFIX);
40495 else if (out_n == 8 && in_n == 8)
40496 return ix86_get_builtin (IX86_BUILTIN_FLOORPS_SFIX256);
40500 case BUILT_IN_ICEIL:
40501 case BUILT_IN_LCEIL:
40502 case BUILT_IN_LLCEIL:
40503 /* The round insn does not trap on denormals. */
40504 if (flag_trapping_math || !TARGET_ROUND)
40507 if (out_mode == SImode && in_mode == DFmode)
40509 if (out_n == 4 && in_n == 2)
40510 return ix86_get_builtin (IX86_BUILTIN_CEILPD_VEC_PACK_SFIX);
40511 else if (out_n == 8 && in_n == 4)
40512 return ix86_get_builtin (IX86_BUILTIN_CEILPD_VEC_PACK_SFIX256);
40513 else if (out_n == 16 && in_n == 8)
40514 return ix86_get_builtin (IX86_BUILTIN_CEILPD_VEC_PACK_SFIX512);
40518 case BUILT_IN_ICEILF:
40519 case BUILT_IN_LCEILF:
40520 case BUILT_IN_LLCEILF:
40521 /* The round insn does not trap on denormals. */
40522 if (flag_trapping_math || !TARGET_ROUND)
40525 if (out_mode == SImode && in_mode == SFmode)
40527 if (out_n == 4 && in_n == 4)
40528 return ix86_get_builtin (IX86_BUILTIN_CEILPS_SFIX);
40529 else if (out_n == 8 && in_n == 8)
40530 return ix86_get_builtin (IX86_BUILTIN_CEILPS_SFIX256);
40534 case BUILT_IN_IRINT:
40535 case BUILT_IN_LRINT:
40536 case BUILT_IN_LLRINT:
40537 if (out_mode == SImode && in_mode == DFmode)
40539 if (out_n == 4 && in_n == 2)
40540 return ix86_get_builtin (IX86_BUILTIN_VEC_PACK_SFIX);
40541 else if (out_n == 8 && in_n == 4)
40542 return ix86_get_builtin (IX86_BUILTIN_VEC_PACK_SFIX256);
40546 case BUILT_IN_IRINTF:
40547 case BUILT_IN_LRINTF:
40548 case BUILT_IN_LLRINTF:
40549 if (out_mode == SImode && in_mode == SFmode)
40551 if (out_n == 4 && in_n == 4)
40552 return ix86_get_builtin (IX86_BUILTIN_CVTPS2DQ);
40553 else if (out_n == 8 && in_n == 8)
40554 return ix86_get_builtin (IX86_BUILTIN_CVTPS2DQ256);
40558 case BUILT_IN_IROUND:
40559 case BUILT_IN_LROUND:
40560 case BUILT_IN_LLROUND:
40561 /* The round insn does not trap on denormals. */
40562 if (flag_trapping_math || !TARGET_ROUND)
40565 if (out_mode == SImode && in_mode == DFmode)
40567 if (out_n == 4 && in_n == 2)
40568 return ix86_get_builtin (IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX);
40569 else if (out_n == 8 && in_n == 4)
40570 return ix86_get_builtin (IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX256);
40571 else if (out_n == 16 && in_n == 8)
40572 return ix86_get_builtin (IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX512);
40576 case BUILT_IN_IROUNDF:
40577 case BUILT_IN_LROUNDF:
40578 case BUILT_IN_LLROUNDF:
40579 /* The round insn does not trap on denormals. */
40580 if (flag_trapping_math || !TARGET_ROUND)
40583 if (out_mode == SImode && in_mode == SFmode)
40585 if (out_n == 4 && in_n == 4)
40586 return ix86_get_builtin (IX86_BUILTIN_ROUNDPS_AZ_SFIX);
40587 else if (out_n == 8 && in_n == 8)
40588 return ix86_get_builtin (IX86_BUILTIN_ROUNDPS_AZ_SFIX256);
40592 case BUILT_IN_COPYSIGN:
40593 if (out_mode == DFmode && in_mode == DFmode)
40595 if (out_n == 2 && in_n == 2)
40596 return ix86_get_builtin (IX86_BUILTIN_CPYSGNPD);
40597 else if (out_n == 4 && in_n == 4)
40598 return ix86_get_builtin (IX86_BUILTIN_CPYSGNPD256);
40599 else if (out_n == 8 && in_n == 8)
40600 return ix86_get_builtin (IX86_BUILTIN_CPYSGNPD512);
40604 case BUILT_IN_COPYSIGNF:
40605 if (out_mode == SFmode && in_mode == SFmode)
40607 if (out_n == 4 && in_n == 4)
40608 return ix86_get_builtin (IX86_BUILTIN_CPYSGNPS);
40609 else if (out_n == 8 && in_n == 8)
40610 return ix86_get_builtin (IX86_BUILTIN_CPYSGNPS256);
40611 else if (out_n == 16 && in_n == 16)
40612 return ix86_get_builtin (IX86_BUILTIN_CPYSGNPS512);
40616 case BUILT_IN_FLOOR:
40617 /* The round insn does not trap on denormals. */
40618 if (flag_trapping_math || !TARGET_ROUND)
40621 if (out_mode == DFmode && in_mode == DFmode)
40623 if (out_n == 2 && in_n == 2)
40624 return ix86_get_builtin (IX86_BUILTIN_FLOORPD);
40625 else if (out_n == 4 && in_n == 4)
40626 return ix86_get_builtin (IX86_BUILTIN_FLOORPD256);
40630 case BUILT_IN_FLOORF:
40631 /* The round insn does not trap on denormals. */
40632 if (flag_trapping_math || !TARGET_ROUND)
40635 if (out_mode == SFmode && in_mode == SFmode)
40637 if (out_n == 4 && in_n == 4)
40638 return ix86_get_builtin (IX86_BUILTIN_FLOORPS);
40639 else if (out_n == 8 && in_n == 8)
40640 return ix86_get_builtin (IX86_BUILTIN_FLOORPS256);
40644 case BUILT_IN_CEIL:
40645 /* The round insn does not trap on denormals. */
40646 if (flag_trapping_math || !TARGET_ROUND)
40649 if (out_mode == DFmode && in_mode == DFmode)
40651 if (out_n == 2 && in_n == 2)
40652 return ix86_get_builtin (IX86_BUILTIN_CEILPD);
40653 else if (out_n == 4 && in_n == 4)
40654 return ix86_get_builtin (IX86_BUILTIN_CEILPD256);
40658 case BUILT_IN_CEILF:
40659 /* The round insn does not trap on denormals. */
40660 if (flag_trapping_math || !TARGET_ROUND)
40663 if (out_mode == SFmode && in_mode == SFmode)
40665 if (out_n == 4 && in_n == 4)
40666 return ix86_get_builtin (IX86_BUILTIN_CEILPS);
40667 else if (out_n == 8 && in_n == 8)
40668 return ix86_get_builtin (IX86_BUILTIN_CEILPS256);
40672 case BUILT_IN_TRUNC:
40673 /* The round insn does not trap on denormals. */
40674 if (flag_trapping_math || !TARGET_ROUND)
40677 if (out_mode == DFmode && in_mode == DFmode)
40679 if (out_n == 2 && in_n == 2)
40680 return ix86_get_builtin (IX86_BUILTIN_TRUNCPD);
40681 else if (out_n == 4 && in_n == 4)
40682 return ix86_get_builtin (IX86_BUILTIN_TRUNCPD256);
40686 case BUILT_IN_TRUNCF:
40687 /* The round insn does not trap on denormals. */
40688 if (flag_trapping_math || !TARGET_ROUND)
40691 if (out_mode == SFmode && in_mode == SFmode)
40693 if (out_n == 4 && in_n == 4)
40694 return ix86_get_builtin (IX86_BUILTIN_TRUNCPS);
40695 else if (out_n == 8 && in_n == 8)
40696 return ix86_get_builtin (IX86_BUILTIN_TRUNCPS256);
40700 case BUILT_IN_RINT:
40701 /* The round insn does not trap on denormals. */
40702 if (flag_trapping_math || !TARGET_ROUND)
40705 if (out_mode == DFmode && in_mode == DFmode)
40707 if (out_n == 2 && in_n == 2)
40708 return ix86_get_builtin (IX86_BUILTIN_RINTPD);
40709 else if (out_n == 4 && in_n == 4)
40710 return ix86_get_builtin (IX86_BUILTIN_RINTPD256);
40714 case BUILT_IN_RINTF:
40715 /* The round insn does not trap on denormals. */
40716 if (flag_trapping_math || !TARGET_ROUND)
40719 if (out_mode == SFmode && in_mode == SFmode)
40721 if (out_n == 4 && in_n == 4)
40722 return ix86_get_builtin (IX86_BUILTIN_RINTPS);
40723 else if (out_n == 8 && in_n == 8)
40724 return ix86_get_builtin (IX86_BUILTIN_RINTPS256);
40728 case BUILT_IN_ROUND:
40729 /* The round insn does not trap on denormals. */
40730 if (flag_trapping_math || !TARGET_ROUND)
40733 if (out_mode == DFmode && in_mode == DFmode)
40735 if (out_n == 2 && in_n == 2)
40736 return ix86_get_builtin (IX86_BUILTIN_ROUNDPD_AZ);
40737 else if (out_n == 4 && in_n == 4)
40738 return ix86_get_builtin (IX86_BUILTIN_ROUNDPD_AZ256);
40742 case BUILT_IN_ROUNDF:
40743 /* The round insn does not trap on denormals. */
40744 if (flag_trapping_math || !TARGET_ROUND)
40747 if (out_mode == SFmode && in_mode == SFmode)
40749 if (out_n == 4 && in_n == 4)
40750 return ix86_get_builtin (IX86_BUILTIN_ROUNDPS_AZ);
40751 else if (out_n == 8 && in_n == 8)
40752 return ix86_get_builtin (IX86_BUILTIN_ROUNDPS_AZ256);
40757 if (out_mode == DFmode && in_mode == DFmode)
40759 if (out_n == 2 && in_n == 2)
40760 return ix86_get_builtin (IX86_BUILTIN_VFMADDPD);
40761 if (out_n == 4 && in_n == 4)
40762 return ix86_get_builtin (IX86_BUILTIN_VFMADDPD256);
40766 case BUILT_IN_FMAF:
40767 if (out_mode == SFmode && in_mode == SFmode)
40769 if (out_n == 4 && in_n == 4)
40770 return ix86_get_builtin (IX86_BUILTIN_VFMADDPS);
40771 if (out_n == 8 && in_n == 8)
40772 return ix86_get_builtin (IX86_BUILTIN_VFMADDPS256);
40780 /* Dispatch to a handler for a vectorization library. */
40781 if (ix86_veclib_handler)
40782 return ix86_veclib_handler ((enum built_in_function) fn, type_out,
40788 /* Handler for an SVML-style interface to
40789 a library with vectorized intrinsics. */
40792 ix86_veclibabi_svml (enum built_in_function fn, tree type_out, tree type_in)
40795 tree fntype, new_fndecl, args;
40798 machine_mode el_mode, in_mode;
40801 /* The SVML is suitable for unsafe math only. */
40802 if (!flag_unsafe_math_optimizations)
40805 el_mode = TYPE_MODE (TREE_TYPE (type_out));
40806 n = TYPE_VECTOR_SUBPARTS (type_out);
40807 in_mode = TYPE_MODE (TREE_TYPE (type_in));
40808 in_n = TYPE_VECTOR_SUBPARTS (type_in);
40809 if (el_mode != in_mode
40817 case BUILT_IN_LOG10:
40819 case BUILT_IN_TANH:
40821 case BUILT_IN_ATAN:
40822 case BUILT_IN_ATAN2:
40823 case BUILT_IN_ATANH:
40824 case BUILT_IN_CBRT:
40825 case BUILT_IN_SINH:
40827 case BUILT_IN_ASINH:
40828 case BUILT_IN_ASIN:
40829 case BUILT_IN_COSH:
40831 case BUILT_IN_ACOSH:
40832 case BUILT_IN_ACOS:
40833 if (el_mode != DFmode || n != 2)
40837 case BUILT_IN_EXPF:
40838 case BUILT_IN_LOGF:
40839 case BUILT_IN_LOG10F:
40840 case BUILT_IN_POWF:
40841 case BUILT_IN_TANHF:
40842 case BUILT_IN_TANF:
40843 case BUILT_IN_ATANF:
40844 case BUILT_IN_ATAN2F:
40845 case BUILT_IN_ATANHF:
40846 case BUILT_IN_CBRTF:
40847 case BUILT_IN_SINHF:
40848 case BUILT_IN_SINF:
40849 case BUILT_IN_ASINHF:
40850 case BUILT_IN_ASINF:
40851 case BUILT_IN_COSHF:
40852 case BUILT_IN_COSF:
40853 case BUILT_IN_ACOSHF:
40854 case BUILT_IN_ACOSF:
40855 if (el_mode != SFmode || n != 4)
40863 bname = IDENTIFIER_POINTER (DECL_NAME (builtin_decl_implicit (fn)));
40865 if (fn == BUILT_IN_LOGF)
40866 strcpy (name, "vmlsLn4");
40867 else if (fn == BUILT_IN_LOG)
40868 strcpy (name, "vmldLn2");
40871 sprintf (name, "vmls%s", bname+10);
40872 name[strlen (name)-1] = '4';
40875 sprintf (name, "vmld%s2", bname+10);
40877 /* Convert to uppercase. */
40881 for (args = DECL_ARGUMENTS (builtin_decl_implicit (fn));
40883 args = TREE_CHAIN (args))
40887 fntype = build_function_type_list (type_out, type_in, NULL);
40889 fntype = build_function_type_list (type_out, type_in, type_in, NULL);
40891 /* Build a function declaration for the vectorized function. */
40892 new_fndecl = build_decl (BUILTINS_LOCATION,
40893 FUNCTION_DECL, get_identifier (name), fntype);
40894 TREE_PUBLIC (new_fndecl) = 1;
40895 DECL_EXTERNAL (new_fndecl) = 1;
40896 DECL_IS_NOVOPS (new_fndecl) = 1;
40897 TREE_READONLY (new_fndecl) = 1;
40902 /* Handler for an ACML-style interface to
40903 a library with vectorized intrinsics. */
40906 ix86_veclibabi_acml (enum built_in_function fn, tree type_out, tree type_in)
40908 char name[20] = "__vr.._";
40909 tree fntype, new_fndecl, args;
40912 machine_mode el_mode, in_mode;
40915 /* The ACML is 64bits only and suitable for unsafe math only as
40916 it does not correctly support parts of IEEE with the required
40917 precision such as denormals. */
40919 || !flag_unsafe_math_optimizations)
40922 el_mode = TYPE_MODE (TREE_TYPE (type_out));
40923 n = TYPE_VECTOR_SUBPARTS (type_out);
40924 in_mode = TYPE_MODE (TREE_TYPE (type_in));
40925 in_n = TYPE_VECTOR_SUBPARTS (type_in);
40926 if (el_mode != in_mode
40936 case BUILT_IN_LOG2:
40937 case BUILT_IN_LOG10:
40940 if (el_mode != DFmode
40945 case BUILT_IN_SINF:
40946 case BUILT_IN_COSF:
40947 case BUILT_IN_EXPF:
40948 case BUILT_IN_POWF:
40949 case BUILT_IN_LOGF:
40950 case BUILT_IN_LOG2F:
40951 case BUILT_IN_LOG10F:
40954 if (el_mode != SFmode
40963 bname = IDENTIFIER_POINTER (DECL_NAME (builtin_decl_implicit (fn)));
40964 sprintf (name + 7, "%s", bname+10);
40967 for (args = DECL_ARGUMENTS (builtin_decl_implicit (fn));
40969 args = TREE_CHAIN (args))
40973 fntype = build_function_type_list (type_out, type_in, NULL);
40975 fntype = build_function_type_list (type_out, type_in, type_in, NULL);
40977 /* Build a function declaration for the vectorized function. */
40978 new_fndecl = build_decl (BUILTINS_LOCATION,
40979 FUNCTION_DECL, get_identifier (name), fntype);
40980 TREE_PUBLIC (new_fndecl) = 1;
40981 DECL_EXTERNAL (new_fndecl) = 1;
40982 DECL_IS_NOVOPS (new_fndecl) = 1;
40983 TREE_READONLY (new_fndecl) = 1;
40988 /* Returns a decl of a function that implements gather load with
40989 memory type MEM_VECTYPE and index type INDEX_VECTYPE and SCALE.
40990 Return NULL_TREE if it is not available. */
40993 ix86_vectorize_builtin_gather (const_tree mem_vectype,
40994 const_tree index_type, int scale)
40997 enum ix86_builtins code;
41002 if ((TREE_CODE (index_type) != INTEGER_TYPE
41003 && !POINTER_TYPE_P (index_type))
41004 || (TYPE_MODE (index_type) != SImode
41005 && TYPE_MODE (index_type) != DImode))
41008 if (TYPE_PRECISION (index_type) > POINTER_SIZE)
41011 /* v*gather* insn sign extends index to pointer mode. */
41012 if (TYPE_PRECISION (index_type) < POINTER_SIZE
41013 && TYPE_UNSIGNED (index_type))
41018 || (scale & (scale - 1)) != 0)
41021 si = TYPE_MODE (index_type) == SImode;
41022 switch (TYPE_MODE (mem_vectype))
41025 if (TARGET_AVX512VL)
41026 code = si ? IX86_BUILTIN_GATHER3SIV2DF : IX86_BUILTIN_GATHER3DIV2DF;
41028 code = si ? IX86_BUILTIN_GATHERSIV2DF : IX86_BUILTIN_GATHERDIV2DF;
41031 if (TARGET_AVX512VL)
41032 code = si ? IX86_BUILTIN_GATHER3ALTSIV4DF : IX86_BUILTIN_GATHER3DIV4DF;
41034 code = si ? IX86_BUILTIN_GATHERALTSIV4DF : IX86_BUILTIN_GATHERDIV4DF;
41037 if (TARGET_AVX512VL)
41038 code = si ? IX86_BUILTIN_GATHER3SIV2DI : IX86_BUILTIN_GATHER3DIV2DI;
41040 code = si ? IX86_BUILTIN_GATHERSIV2DI : IX86_BUILTIN_GATHERDIV2DI;
41043 if (TARGET_AVX512VL)
41044 code = si ? IX86_BUILTIN_GATHER3ALTSIV4DI : IX86_BUILTIN_GATHER3DIV4DI;
41046 code = si ? IX86_BUILTIN_GATHERALTSIV4DI : IX86_BUILTIN_GATHERDIV4DI;
41049 if (TARGET_AVX512VL)
41050 code = si ? IX86_BUILTIN_GATHER3SIV4SF : IX86_BUILTIN_GATHER3DIV4SF;
41052 code = si ? IX86_BUILTIN_GATHERSIV4SF : IX86_BUILTIN_GATHERDIV4SF;
41055 if (TARGET_AVX512VL)
41056 code = si ? IX86_BUILTIN_GATHER3SIV8SF : IX86_BUILTIN_GATHER3ALTDIV8SF;
41058 code = si ? IX86_BUILTIN_GATHERSIV8SF : IX86_BUILTIN_GATHERALTDIV8SF;
41061 if (TARGET_AVX512VL)
41062 code = si ? IX86_BUILTIN_GATHER3SIV4SI : IX86_BUILTIN_GATHER3DIV4SI;
41064 code = si ? IX86_BUILTIN_GATHERSIV4SI : IX86_BUILTIN_GATHERDIV4SI;
41067 if (TARGET_AVX512VL)
41068 code = si ? IX86_BUILTIN_GATHER3SIV8SI : IX86_BUILTIN_GATHER3ALTDIV8SI;
41070 code = si ? IX86_BUILTIN_GATHERSIV8SI : IX86_BUILTIN_GATHERALTDIV8SI;
41073 if (TARGET_AVX512F)
41074 code = si ? IX86_BUILTIN_GATHER3ALTSIV8DF : IX86_BUILTIN_GATHER3DIV8DF;
41079 if (TARGET_AVX512F)
41080 code = si ? IX86_BUILTIN_GATHER3ALTSIV8DI : IX86_BUILTIN_GATHER3DIV8DI;
41085 if (TARGET_AVX512F)
41086 code = si ? IX86_BUILTIN_GATHER3SIV16SF : IX86_BUILTIN_GATHER3ALTDIV16SF;
41091 if (TARGET_AVX512F)
41092 code = si ? IX86_BUILTIN_GATHER3SIV16SI : IX86_BUILTIN_GATHER3ALTDIV16SI;
41100 return ix86_get_builtin (code);
41103 /* Returns a code for a target-specific builtin that implements
41104 reciprocal of the function, or NULL_TREE if not available. */
41107 ix86_builtin_reciprocal (unsigned int fn, bool md_fn, bool)
41109 if (! (TARGET_SSE_MATH && !optimize_insn_for_size_p ()
41110 && flag_finite_math_only && !flag_trapping_math
41111 && flag_unsafe_math_optimizations))
41115 /* Machine dependent builtins. */
41118 /* Vectorized version of sqrt to rsqrt conversion. */
41119 case IX86_BUILTIN_SQRTPS_NR:
41120 return ix86_get_builtin (IX86_BUILTIN_RSQRTPS_NR);
41122 case IX86_BUILTIN_SQRTPS_NR256:
41123 return ix86_get_builtin (IX86_BUILTIN_RSQRTPS_NR256);
41129 /* Normal builtins. */
41132 /* Sqrt to rsqrt conversion. */
41133 case BUILT_IN_SQRTF:
41134 return ix86_get_builtin (IX86_BUILTIN_RSQRTF);
41141 /* Helper for avx_vpermilps256_operand et al. This is also used by
41142 the expansion functions to turn the parallel back into a mask.
41143 The return value is 0 for no match and the imm8+1 for a match. */
41146 avx_vpermilp_parallel (rtx par, machine_mode mode)
41148 unsigned i, nelt = GET_MODE_NUNITS (mode);
41150 unsigned char ipar[16] = {}; /* Silence -Wuninitialized warning. */
41152 if (XVECLEN (par, 0) != (int) nelt)
41155 /* Validate that all of the elements are constants, and not totally
41156 out of range. Copy the data into an integral array to make the
41157 subsequent checks easier. */
41158 for (i = 0; i < nelt; ++i)
41160 rtx er = XVECEXP (par, 0, i);
41161 unsigned HOST_WIDE_INT ei;
41163 if (!CONST_INT_P (er))
41174 /* In the 512-bit DFmode case, we can only move elements within
41175 a 128-bit lane. First fill the second part of the mask,
41177 for (i = 4; i < 6; ++i)
41179 if (ipar[i] < 4 || ipar[i] >= 6)
41181 mask |= (ipar[i] - 4) << i;
41183 for (i = 6; i < 8; ++i)
41187 mask |= (ipar[i] - 6) << i;
41192 /* In the 256-bit DFmode case, we can only move elements within
41194 for (i = 0; i < 2; ++i)
41198 mask |= ipar[i] << i;
41200 for (i = 2; i < 4; ++i)
41204 mask |= (ipar[i] - 2) << i;
41209 /* In 512 bit SFmode case, permutation in the upper 256 bits
41210 must mirror the permutation in the lower 256-bits. */
41211 for (i = 0; i < 8; ++i)
41212 if (ipar[i] + 8 != ipar[i + 8])
41217 /* In 256 bit SFmode case, we have full freedom of
41218 movement within the low 128-bit lane, but the high 128-bit
41219 lane must mirror the exact same pattern. */
41220 for (i = 0; i < 4; ++i)
41221 if (ipar[i] + 4 != ipar[i + 4])
41228 /* In the 128-bit case, we've full freedom in the placement of
41229 the elements from the source operand. */
41230 for (i = 0; i < nelt; ++i)
41231 mask |= ipar[i] << (i * (nelt / 2));
41235 gcc_unreachable ();
41238 /* Make sure success has a non-zero value by adding one. */
41242 /* Helper for avx_vperm2f128_v4df_operand et al. This is also used by
41243 the expansion functions to turn the parallel back into a mask.
41244 The return value is 0 for no match and the imm8+1 for a match. */
41247 avx_vperm2f128_parallel (rtx par, machine_mode mode)
41249 unsigned i, nelt = GET_MODE_NUNITS (mode), nelt2 = nelt / 2;
41251 unsigned char ipar[8] = {}; /* Silence -Wuninitialized warning. */
41253 if (XVECLEN (par, 0) != (int) nelt)
41256 /* Validate that all of the elements are constants, and not totally
41257 out of range. Copy the data into an integral array to make the
41258 subsequent checks easier. */
41259 for (i = 0; i < nelt; ++i)
41261 rtx er = XVECEXP (par, 0, i);
41262 unsigned HOST_WIDE_INT ei;
41264 if (!CONST_INT_P (er))
41267 if (ei >= 2 * nelt)
41272 /* Validate that the halves of the permute are halves. */
41273 for (i = 0; i < nelt2 - 1; ++i)
41274 if (ipar[i] + 1 != ipar[i + 1])
41276 for (i = nelt2; i < nelt - 1; ++i)
41277 if (ipar[i] + 1 != ipar[i + 1])
41280 /* Reconstruct the mask. */
41281 for (i = 0; i < 2; ++i)
41283 unsigned e = ipar[i * nelt2];
41287 mask |= e << (i * 4);
41290 /* Make sure success has a non-zero value by adding one. */
41294 /* Return a register priority for hard reg REGNO. */
41296 ix86_register_priority (int hard_regno)
41298 /* ebp and r13 as the base always wants a displacement, r12 as the
41299 base always wants an index. So discourage their usage in an
41301 if (hard_regno == R12_REG || hard_regno == R13_REG)
41303 if (hard_regno == BP_REG)
41305 /* New x86-64 int registers result in bigger code size. Discourage
41307 if (FIRST_REX_INT_REG <= hard_regno && hard_regno <= LAST_REX_INT_REG)
41309 /* New x86-64 SSE registers result in bigger code size. Discourage
41311 if (FIRST_REX_SSE_REG <= hard_regno && hard_regno <= LAST_REX_SSE_REG)
41313 /* Usage of AX register results in smaller code. Prefer it. */
41314 if (hard_regno == AX_REG)
41319 /* Implement TARGET_PREFERRED_RELOAD_CLASS.
41321 Put float CONST_DOUBLE in the constant pool instead of fp regs.
41322 QImode must go into class Q_REGS.
41323 Narrow ALL_REGS to GENERAL_REGS. This supports allowing movsf and
41324 movdf to do mem-to-mem moves through integer regs. */
41327 ix86_preferred_reload_class (rtx x, reg_class_t regclass)
41329 machine_mode mode = GET_MODE (x);
41331 /* We're only allowed to return a subclass of CLASS. Many of the
41332 following checks fail for NO_REGS, so eliminate that early. */
41333 if (regclass == NO_REGS)
41336 /* All classes can load zeros. */
41337 if (x == CONST0_RTX (mode))
41340 /* Force constants into memory if we are loading a (nonzero) constant into
41341 an MMX, SSE or MASK register. This is because there are no MMX/SSE/MASK
41342 instructions to load from a constant. */
41344 && (MAYBE_MMX_CLASS_P (regclass)
41345 || MAYBE_SSE_CLASS_P (regclass)
41346 || MAYBE_MASK_CLASS_P (regclass)))
41349 /* Prefer SSE regs only, if we can use them for math. */
41350 if (TARGET_SSE_MATH && !TARGET_MIX_SSE_I387 && SSE_FLOAT_MODE_P (mode))
41351 return SSE_CLASS_P (regclass) ? regclass : NO_REGS;
41353 /* Floating-point constants need more complex checks. */
41354 if (CONST_DOUBLE_P (x))
41356 /* General regs can load everything. */
41357 if (reg_class_subset_p (regclass, GENERAL_REGS))
41360 /* Floats can load 0 and 1 plus some others. Note that we eliminated
41361 zero above. We only want to wind up preferring 80387 registers if
41362 we plan on doing computation with them. */
41364 && standard_80387_constant_p (x) > 0)
41366 /* Limit class to non-sse. */
41367 if (regclass == FLOAT_SSE_REGS)
41369 if (regclass == FP_TOP_SSE_REGS)
41371 if (regclass == FP_SECOND_SSE_REGS)
41372 return FP_SECOND_REG;
41373 if (regclass == FLOAT_INT_REGS || regclass == FLOAT_REGS)
41380 /* Generally when we see PLUS here, it's the function invariant
41381 (plus soft-fp const_int). Which can only be computed into general
41383 if (GET_CODE (x) == PLUS)
41384 return reg_class_subset_p (regclass, GENERAL_REGS) ? regclass : NO_REGS;
41386 /* QImode constants are easy to load, but non-constant QImode data
41387 must go into Q_REGS. */
41388 if (GET_MODE (x) == QImode && !CONSTANT_P (x))
41390 if (reg_class_subset_p (regclass, Q_REGS))
41392 if (reg_class_subset_p (Q_REGS, regclass))
41400 /* Discourage putting floating-point values in SSE registers unless
41401 SSE math is being used, and likewise for the 387 registers. */
41403 ix86_preferred_output_reload_class (rtx x, reg_class_t regclass)
41405 machine_mode mode = GET_MODE (x);
41407 /* Restrict the output reload class to the register bank that we are doing
41408 math on. If we would like not to return a subset of CLASS, reject this
41409 alternative: if reload cannot do this, it will still use its choice. */
41410 mode = GET_MODE (x);
41411 if (TARGET_SSE_MATH && SSE_FLOAT_MODE_P (mode))
41412 return MAYBE_SSE_CLASS_P (regclass) ? ALL_SSE_REGS : NO_REGS;
41414 if (X87_FLOAT_MODE_P (mode))
41416 if (regclass == FP_TOP_SSE_REGS)
41418 else if (regclass == FP_SECOND_SSE_REGS)
41419 return FP_SECOND_REG;
41421 return FLOAT_CLASS_P (regclass) ? regclass : NO_REGS;
41428 ix86_secondary_reload (bool in_p, rtx x, reg_class_t rclass,
41429 machine_mode mode, secondary_reload_info *sri)
41431 /* Double-word spills from general registers to non-offsettable memory
41432 references (zero-extended addresses) require special handling. */
41435 && GET_MODE_SIZE (mode) > UNITS_PER_WORD
41436 && INTEGER_CLASS_P (rclass)
41437 && !offsettable_memref_p (x))
41440 ? CODE_FOR_reload_noff_load
41441 : CODE_FOR_reload_noff_store);
41442 /* Add the cost of moving address to a temporary. */
41443 sri->extra_cost = 1;
41448 /* QImode spills from non-QI registers require
41449 intermediate register on 32bit targets. */
41451 && (MAYBE_MASK_CLASS_P (rclass)
41452 || (!TARGET_64BIT && !in_p
41453 && INTEGER_CLASS_P (rclass)
41454 && MAYBE_NON_Q_CLASS_P (rclass))))
41463 if (regno >= FIRST_PSEUDO_REGISTER || GET_CODE (x) == SUBREG)
41464 regno = true_regnum (x);
41466 /* Return Q_REGS if the operand is in memory. */
41471 /* This condition handles corner case where an expression involving
41472 pointers gets vectorized. We're trying to use the address of a
41473 stack slot as a vector initializer.
41475 (set (reg:V2DI 74 [ vect_cst_.2 ])
41476 (vec_duplicate:V2DI (reg/f:DI 20 frame)))
41478 Eventually frame gets turned into sp+offset like this:
41480 (set (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
41481 (vec_duplicate:V2DI (plus:DI (reg/f:DI 7 sp)
41482 (const_int 392 [0x188]))))
41484 That later gets turned into:
41486 (set (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
41487 (vec_duplicate:V2DI (plus:DI (reg/f:DI 7 sp)
41488 (mem/u/c/i:DI (symbol_ref/u:DI ("*.LC0") [flags 0x2]) [0 S8 A64]))))
41490 We'll have the following reload recorded:
41492 Reload 0: reload_in (DI) =
41493 (plus:DI (reg/f:DI 7 sp)
41494 (mem/u/c/i:DI (symbol_ref/u:DI ("*.LC0") [flags 0x2]) [0 S8 A64]))
41495 reload_out (V2DI) = (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
41496 SSE_REGS, RELOAD_OTHER (opnum = 0), can't combine
41497 reload_in_reg: (plus:DI (reg/f:DI 7 sp) (const_int 392 [0x188]))
41498 reload_out_reg: (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
41499 reload_reg_rtx: (reg:V2DI 22 xmm1)
41501 Which isn't going to work since SSE instructions can't handle scalar
41502 additions. Returning GENERAL_REGS forces the addition into integer
41503 register and reload can handle subsequent reloads without problems. */
41505 if (in_p && GET_CODE (x) == PLUS
41506 && SSE_CLASS_P (rclass)
41507 && SCALAR_INT_MODE_P (mode))
41508 return GENERAL_REGS;
41513 /* Implement TARGET_CLASS_LIKELY_SPILLED_P. */
41516 ix86_class_likely_spilled_p (reg_class_t rclass)
41527 case SSE_FIRST_REG:
41529 case FP_SECOND_REG:
41540 /* If we are copying between general and FP registers, we need a memory
41541 location. The same is true for SSE and MMX registers.
41543 To optimize register_move_cost performance, allow inline variant.
41545 The macro can't work reliably when one of the CLASSES is class containing
41546 registers from multiple units (SSE, MMX, integer). We avoid this by never
41547 combining those units in single alternative in the machine description.
41548 Ensure that this constraint holds to avoid unexpected surprises.
41550 When STRICT is false, we are being called from REGISTER_MOVE_COST, so do not
41551 enforce these sanity checks. */
41554 inline_secondary_memory_needed (enum reg_class class1, enum reg_class class2,
41555 machine_mode mode, int strict)
41557 if (lra_in_progress && (class1 == NO_REGS || class2 == NO_REGS))
41559 if (MAYBE_FLOAT_CLASS_P (class1) != FLOAT_CLASS_P (class1)
41560 || MAYBE_FLOAT_CLASS_P (class2) != FLOAT_CLASS_P (class2)
41561 || MAYBE_SSE_CLASS_P (class1) != SSE_CLASS_P (class1)
41562 || MAYBE_SSE_CLASS_P (class2) != SSE_CLASS_P (class2)
41563 || MAYBE_MMX_CLASS_P (class1) != MMX_CLASS_P (class1)
41564 || MAYBE_MMX_CLASS_P (class2) != MMX_CLASS_P (class2))
41566 gcc_assert (!strict || lra_in_progress);
41570 if (FLOAT_CLASS_P (class1) != FLOAT_CLASS_P (class2))
41573 /* Between mask and general, we have moves no larger than word size. */
41574 if ((MAYBE_MASK_CLASS_P (class1) != MAYBE_MASK_CLASS_P (class2))
41575 && (GET_MODE_SIZE (mode) > UNITS_PER_WORD))
41578 /* ??? This is a lie. We do have moves between mmx/general, and for
41579 mmx/sse2. But by saying we need secondary memory we discourage the
41580 register allocator from using the mmx registers unless needed. */
41581 if (MMX_CLASS_P (class1) != MMX_CLASS_P (class2))
41584 if (SSE_CLASS_P (class1) != SSE_CLASS_P (class2))
41586 /* SSE1 doesn't have any direct moves from other classes. */
41590 /* If the target says that inter-unit moves are more expensive
41591 than moving through memory, then don't generate them. */
41592 if ((SSE_CLASS_P (class1) && !TARGET_INTER_UNIT_MOVES_FROM_VEC)
41593 || (SSE_CLASS_P (class2) && !TARGET_INTER_UNIT_MOVES_TO_VEC))
41596 /* Between SSE and general, we have moves no larger than word size. */
41597 if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
41605 ix86_secondary_memory_needed (enum reg_class class1, enum reg_class class2,
41606 machine_mode mode, int strict)
41608 return inline_secondary_memory_needed (class1, class2, mode, strict);
41611 /* Implement the TARGET_CLASS_MAX_NREGS hook.
41613 On the 80386, this is the size of MODE in words,
41614 except in the FP regs, where a single reg is always enough. */
41616 static unsigned char
41617 ix86_class_max_nregs (reg_class_t rclass, machine_mode mode)
41619 if (MAYBE_INTEGER_CLASS_P (rclass))
41621 if (mode == XFmode)
41622 return (TARGET_64BIT ? 2 : 3);
41623 else if (mode == XCmode)
41624 return (TARGET_64BIT ? 4 : 6);
41626 return ((GET_MODE_SIZE (mode) + UNITS_PER_WORD - 1) / UNITS_PER_WORD);
41630 if (COMPLEX_MODE_P (mode))
41637 /* Return true if the registers in CLASS cannot represent the change from
41638 modes FROM to TO. */
41641 ix86_cannot_change_mode_class (machine_mode from, machine_mode to,
41642 enum reg_class regclass)
41647 /* x87 registers can't do subreg at all, as all values are reformatted
41648 to extended precision. */
41649 if (MAYBE_FLOAT_CLASS_P (regclass))
41652 if (MAYBE_SSE_CLASS_P (regclass) || MAYBE_MMX_CLASS_P (regclass))
41654 /* Vector registers do not support QI or HImode loads. If we don't
41655 disallow a change to these modes, reload will assume it's ok to
41656 drop the subreg from (subreg:SI (reg:HI 100) 0). This affects
41657 the vec_dupv4hi pattern. */
41658 if (GET_MODE_SIZE (from) < 4)
41665 /* Return the cost of moving data of mode M between a
41666 register and memory. A value of 2 is the default; this cost is
41667 relative to those in `REGISTER_MOVE_COST'.
41669 This function is used extensively by register_move_cost that is used to
41670 build tables at startup. Make it inline in this case.
41671 When IN is 2, return maximum of in and out move cost.
41673 If moving between registers and memory is more expensive than
41674 between two registers, you should define this macro to express the
41677 Model also increased moving costs of QImode registers in non
41681 inline_memory_move_cost (machine_mode mode, enum reg_class regclass,
41685 if (FLOAT_CLASS_P (regclass))
41703 return MAX (ix86_cost->fp_load [index], ix86_cost->fp_store [index]);
41704 return in ? ix86_cost->fp_load [index] : ix86_cost->fp_store [index];
41706 if (SSE_CLASS_P (regclass))
41709 switch (GET_MODE_SIZE (mode))
41724 return MAX (ix86_cost->sse_load [index], ix86_cost->sse_store [index]);
41725 return in ? ix86_cost->sse_load [index] : ix86_cost->sse_store [index];
41727 if (MMX_CLASS_P (regclass))
41730 switch (GET_MODE_SIZE (mode))
41742 return MAX (ix86_cost->mmx_load [index], ix86_cost->mmx_store [index]);
41743 return in ? ix86_cost->mmx_load [index] : ix86_cost->mmx_store [index];
41745 switch (GET_MODE_SIZE (mode))
41748 if (Q_CLASS_P (regclass) || TARGET_64BIT)
41751 return ix86_cost->int_store[0];
41752 if (TARGET_PARTIAL_REG_DEPENDENCY
41753 && optimize_function_for_speed_p (cfun))
41754 cost = ix86_cost->movzbl_load;
41756 cost = ix86_cost->int_load[0];
41758 return MAX (cost, ix86_cost->int_store[0]);
41764 return MAX (ix86_cost->movzbl_load, ix86_cost->int_store[0] + 4);
41766 return ix86_cost->movzbl_load;
41768 return ix86_cost->int_store[0] + 4;
41773 return MAX (ix86_cost->int_load[1], ix86_cost->int_store[1]);
41774 return in ? ix86_cost->int_load[1] : ix86_cost->int_store[1];
41776 /* Compute number of 32bit moves needed. TFmode is moved as XFmode. */
41777 if (mode == TFmode)
41780 cost = MAX (ix86_cost->int_load[2] , ix86_cost->int_store[2]);
41782 cost = ix86_cost->int_load[2];
41784 cost = ix86_cost->int_store[2];
41785 return (cost * (((int) GET_MODE_SIZE (mode)
41786 + UNITS_PER_WORD - 1) / UNITS_PER_WORD));
41791 ix86_memory_move_cost (machine_mode mode, reg_class_t regclass,
41794 return inline_memory_move_cost (mode, (enum reg_class) regclass, in ? 1 : 0);
41798 /* Return the cost of moving data from a register in class CLASS1 to
41799 one in class CLASS2.
41801 It is not required that the cost always equal 2 when FROM is the same as TO;
41802 on some machines it is expensive to move between registers if they are not
41803 general registers. */
41806 ix86_register_move_cost (machine_mode mode, reg_class_t class1_i,
41807 reg_class_t class2_i)
41809 enum reg_class class1 = (enum reg_class) class1_i;
41810 enum reg_class class2 = (enum reg_class) class2_i;
41812 /* In case we require secondary memory, compute cost of the store followed
41813 by load. In order to avoid bad register allocation choices, we need
41814 for this to be *at least* as high as the symmetric MEMORY_MOVE_COST. */
41816 if (inline_secondary_memory_needed (class1, class2, mode, 0))
41820 cost += inline_memory_move_cost (mode, class1, 2);
41821 cost += inline_memory_move_cost (mode, class2, 2);
41823 /* In case of copying from general_purpose_register we may emit multiple
41824 stores followed by single load causing memory size mismatch stall.
41825 Count this as arbitrarily high cost of 20. */
41826 if (targetm.class_max_nregs (class1, mode)
41827 > targetm.class_max_nregs (class2, mode))
41830 /* In the case of FP/MMX moves, the registers actually overlap, and we
41831 have to switch modes in order to treat them differently. */
41832 if ((MMX_CLASS_P (class1) && MAYBE_FLOAT_CLASS_P (class2))
41833 || (MMX_CLASS_P (class2) && MAYBE_FLOAT_CLASS_P (class1)))
41839 /* Moves between SSE/MMX and integer unit are expensive. */
41840 if (MMX_CLASS_P (class1) != MMX_CLASS_P (class2)
41841 || SSE_CLASS_P (class1) != SSE_CLASS_P (class2))
41843 /* ??? By keeping returned value relatively high, we limit the number
41844 of moves between integer and MMX/SSE registers for all targets.
41845 Additionally, high value prevents problem with x86_modes_tieable_p(),
41846 where integer modes in MMX/SSE registers are not tieable
41847 because of missing QImode and HImode moves to, from or between
41848 MMX/SSE registers. */
41849 return MAX (8, ix86_cost->mmxsse_to_integer);
41851 if (MAYBE_FLOAT_CLASS_P (class1))
41852 return ix86_cost->fp_move;
41853 if (MAYBE_SSE_CLASS_P (class1))
41854 return ix86_cost->sse_move;
41855 if (MAYBE_MMX_CLASS_P (class1))
41856 return ix86_cost->mmx_move;
41860 /* Return TRUE if hard register REGNO can hold a value of machine-mode
41864 ix86_hard_regno_mode_ok (int regno, machine_mode mode)
41866 /* Flags and only flags can only hold CCmode values. */
41867 if (CC_REGNO_P (regno))
41868 return GET_MODE_CLASS (mode) == MODE_CC;
41869 if (GET_MODE_CLASS (mode) == MODE_CC
41870 || GET_MODE_CLASS (mode) == MODE_RANDOM
41871 || GET_MODE_CLASS (mode) == MODE_PARTIAL_INT)
41873 if (STACK_REGNO_P (regno))
41874 return VALID_FP_MODE_P (mode);
41875 if (MASK_REGNO_P (regno))
41876 return (VALID_MASK_REG_MODE (mode)
41877 || (TARGET_AVX512BW
41878 && VALID_MASK_AVX512BW_MODE (mode)));
41879 if (BND_REGNO_P (regno))
41880 return VALID_BND_REG_MODE (mode);
41881 if (SSE_REGNO_P (regno))
41883 /* We implement the move patterns for all vector modes into and
41884 out of SSE registers, even when no operation instructions
41887 /* For AVX-512 we allow, regardless of regno:
41889 - any of 512-bit wide vector mode
41890 - any scalar mode. */
41893 || VALID_AVX512F_REG_MODE (mode)
41894 || VALID_AVX512F_SCALAR_MODE (mode)))
41897 /* TODO check for QI/HI scalars. */
41898 /* AVX512VL allows sse regs16+ for 128/256 bit modes. */
41899 if (TARGET_AVX512VL
41902 || VALID_AVX256_REG_MODE (mode)
41903 || VALID_AVX512VL_128_REG_MODE (mode)))
41906 /* xmm16-xmm31 are only available for AVX-512. */
41907 if (EXT_REX_SSE_REGNO_P (regno))
41910 /* OImode and AVX modes are available only when AVX is enabled. */
41911 return ((TARGET_AVX
41912 && VALID_AVX256_REG_OR_OI_MODE (mode))
41913 || VALID_SSE_REG_MODE (mode)
41914 || VALID_SSE2_REG_MODE (mode)
41915 || VALID_MMX_REG_MODE (mode)
41916 || VALID_MMX_REG_MODE_3DNOW (mode));
41918 if (MMX_REGNO_P (regno))
41920 /* We implement the move patterns for 3DNOW modes even in MMX mode,
41921 so if the register is available at all, then we can move data of
41922 the given mode into or out of it. */
41923 return (VALID_MMX_REG_MODE (mode)
41924 || VALID_MMX_REG_MODE_3DNOW (mode));
41927 if (mode == QImode)
41929 /* Take care for QImode values - they can be in non-QI regs,
41930 but then they do cause partial register stalls. */
41931 if (ANY_QI_REGNO_P (regno))
41933 if (!TARGET_PARTIAL_REG_STALL)
41935 /* LRA checks if the hard register is OK for the given mode.
41936 QImode values can live in non-QI regs, so we allow all
41938 if (lra_in_progress)
41940 return !can_create_pseudo_p ();
41942 /* We handle both integer and floats in the general purpose registers. */
41943 else if (VALID_INT_MODE_P (mode))
41945 else if (VALID_FP_MODE_P (mode))
41947 else if (VALID_DFP_MODE_P (mode))
41949 /* Lots of MMX code casts 8 byte vector modes to DImode. If we then go
41950 on to use that value in smaller contexts, this can easily force a
41951 pseudo to be allocated to GENERAL_REGS. Since this is no worse than
41952 supporting DImode, allow it. */
41953 else if (VALID_MMX_REG_MODE_3DNOW (mode) || VALID_MMX_REG_MODE (mode))
41959 /* A subroutine of ix86_modes_tieable_p. Return true if MODE is a
41960 tieable integer mode. */
41963 ix86_tieable_integer_mode_p (machine_mode mode)
41972 return TARGET_64BIT || !TARGET_PARTIAL_REG_STALL;
41975 return TARGET_64BIT;
41982 /* Return true if MODE1 is accessible in a register that can hold MODE2
41983 without copying. That is, all register classes that can hold MODE2
41984 can also hold MODE1. */
41987 ix86_modes_tieable_p (machine_mode mode1, machine_mode mode2)
41989 if (mode1 == mode2)
41992 if (ix86_tieable_integer_mode_p (mode1)
41993 && ix86_tieable_integer_mode_p (mode2))
41996 /* MODE2 being XFmode implies fp stack or general regs, which means we
41997 can tie any smaller floating point modes to it. Note that we do not
41998 tie this with TFmode. */
41999 if (mode2 == XFmode)
42000 return mode1 == SFmode || mode1 == DFmode;
42002 /* MODE2 being DFmode implies fp stack, general or sse regs, which means
42003 that we can tie it with SFmode. */
42004 if (mode2 == DFmode)
42005 return mode1 == SFmode;
42007 /* If MODE2 is only appropriate for an SSE register, then tie with
42008 any other mode acceptable to SSE registers. */
42009 if (GET_MODE_SIZE (mode2) == 32
42010 && ix86_hard_regno_mode_ok (FIRST_SSE_REG, mode2))
42011 return (GET_MODE_SIZE (mode1) == 32
42012 && ix86_hard_regno_mode_ok (FIRST_SSE_REG, mode1));
42013 if (GET_MODE_SIZE (mode2) == 16
42014 && ix86_hard_regno_mode_ok (FIRST_SSE_REG, mode2))
42015 return (GET_MODE_SIZE (mode1) == 16
42016 && ix86_hard_regno_mode_ok (FIRST_SSE_REG, mode1));
42018 /* If MODE2 is appropriate for an MMX register, then tie
42019 with any other mode acceptable to MMX registers. */
42020 if (GET_MODE_SIZE (mode2) == 8
42021 && ix86_hard_regno_mode_ok (FIRST_MMX_REG, mode2))
42022 return (GET_MODE_SIZE (mode1) == 8
42023 && ix86_hard_regno_mode_ok (FIRST_MMX_REG, mode1));
42028 /* Return the cost of moving between two registers of mode MODE. */
42031 ix86_set_reg_reg_cost (machine_mode mode)
42033 unsigned int units = UNITS_PER_WORD;
42035 switch (GET_MODE_CLASS (mode))
42041 units = GET_MODE_SIZE (CCmode);
42045 if ((TARGET_SSE && mode == TFmode)
42046 || (TARGET_80387 && mode == XFmode)
42047 || ((TARGET_80387 || TARGET_SSE2) && mode == DFmode)
42048 || ((TARGET_80387 || TARGET_SSE) && mode == SFmode))
42049 units = GET_MODE_SIZE (mode);
42052 case MODE_COMPLEX_FLOAT:
42053 if ((TARGET_SSE && mode == TCmode)
42054 || (TARGET_80387 && mode == XCmode)
42055 || ((TARGET_80387 || TARGET_SSE2) && mode == DCmode)
42056 || ((TARGET_80387 || TARGET_SSE) && mode == SCmode))
42057 units = GET_MODE_SIZE (mode);
42060 case MODE_VECTOR_INT:
42061 case MODE_VECTOR_FLOAT:
42062 if ((TARGET_AVX512F && VALID_AVX512F_REG_MODE (mode))
42063 || (TARGET_AVX && VALID_AVX256_REG_MODE (mode))
42064 || (TARGET_SSE2 && VALID_SSE2_REG_MODE (mode))
42065 || (TARGET_SSE && VALID_SSE_REG_MODE (mode))
42066 || (TARGET_MMX && VALID_MMX_REG_MODE (mode)))
42067 units = GET_MODE_SIZE (mode);
42070 /* Return the cost of moving between two registers of mode MODE,
42071 assuming that the move will be in pieces of at most UNITS bytes. */
42072 return COSTS_N_INSNS ((GET_MODE_SIZE (mode) + units - 1) / units);
42075 /* Compute a (partial) cost for rtx X. Return true if the complete
42076 cost has been computed, and false if subexpressions should be
42077 scanned. In either case, *TOTAL contains the cost result. */
42080 ix86_rtx_costs (rtx x, int code_i, int outer_code_i, int opno, int *total,
42084 enum rtx_code code = (enum rtx_code) code_i;
42085 enum rtx_code outer_code = (enum rtx_code) outer_code_i;
42086 machine_mode mode = GET_MODE (x);
42087 const struct processor_costs *cost = speed ? ix86_cost : &ix86_size_cost;
42092 if (register_operand (SET_DEST (x), VOIDmode)
42093 && reg_or_0_operand (SET_SRC (x), VOIDmode))
42095 *total = ix86_set_reg_reg_cost (GET_MODE (SET_DEST (x)));
42104 if (TARGET_64BIT && !x86_64_immediate_operand (x, VOIDmode))
42106 else if (TARGET_64BIT && !x86_64_zext_immediate_operand (x, VOIDmode))
42108 else if (flag_pic && SYMBOLIC_CONST (x)
42110 && (GET_CODE (x) == LABEL_REF
42111 || (GET_CODE (x) == SYMBOL_REF
42112 && SYMBOL_REF_LOCAL_P (x))))
42113 /* Use 0 cost for CONST to improve its propagation. */
42114 && (TARGET_64BIT || GET_CODE (x) != CONST))
42120 case CONST_WIDE_INT:
42125 switch (standard_80387_constant_p (x))
42130 default: /* Other constants */
42137 if (SSE_FLOAT_MODE_P (mode))
42140 switch (standard_sse_constant_p (x))
42144 case 1: /* 0: xor eliminates false dependency */
42147 default: /* -1: cmp contains false dependency */
42152 /* Fall back to (MEM (SYMBOL_REF)), since that's where
42153 it'll probably end up. Add a penalty for size. */
42154 *total = (COSTS_N_INSNS (1)
42155 + (flag_pic != 0 && !TARGET_64BIT)
42156 + (mode == SFmode ? 0 : mode == DFmode ? 1 : 2));
42160 /* The zero extensions is often completely free on x86_64, so make
42161 it as cheap as possible. */
42162 if (TARGET_64BIT && mode == DImode
42163 && GET_MODE (XEXP (x, 0)) == SImode)
42165 else if (TARGET_ZERO_EXTEND_WITH_AND)
42166 *total = cost->add;
42168 *total = cost->movzx;
42172 *total = cost->movsx;
42176 if (SCALAR_INT_MODE_P (mode)
42177 && GET_MODE_SIZE (mode) < UNITS_PER_WORD
42178 && CONST_INT_P (XEXP (x, 1)))
42180 HOST_WIDE_INT value = INTVAL (XEXP (x, 1));
42183 *total = cost->add;
42186 if ((value == 2 || value == 3)
42187 && cost->lea <= cost->shift_const)
42189 *total = cost->lea;
42199 if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
42201 /* ??? Should be SSE vector operation cost. */
42202 /* At least for published AMD latencies, this really is the same
42203 as the latency for a simple fpu operation like fabs. */
42204 /* V*QImode is emulated with 1-11 insns. */
42205 if (mode == V16QImode || mode == V32QImode)
42208 if (TARGET_XOP && mode == V16QImode)
42210 /* For XOP we use vpshab, which requires a broadcast of the
42211 value to the variable shift insn. For constants this
42212 means a V16Q const in mem; even when we can perform the
42213 shift with one insn set the cost to prefer paddb. */
42214 if (CONSTANT_P (XEXP (x, 1)))
42216 *total = (cost->fabs
42217 + rtx_cost (XEXP (x, 0), code, 0, speed)
42218 + (speed ? 2 : COSTS_N_BYTES (16)));
42223 else if (TARGET_SSSE3)
42225 *total = cost->fabs * count;
42228 *total = cost->fabs;
42230 else if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
42232 if (CONST_INT_P (XEXP (x, 1)))
42234 if (INTVAL (XEXP (x, 1)) > 32)
42235 *total = cost->shift_const + COSTS_N_INSNS (2);
42237 *total = cost->shift_const * 2;
42241 if (GET_CODE (XEXP (x, 1)) == AND)
42242 *total = cost->shift_var * 2;
42244 *total = cost->shift_var * 6 + COSTS_N_INSNS (2);
42249 if (CONST_INT_P (XEXP (x, 1)))
42250 *total = cost->shift_const;
42251 else if (GET_CODE (XEXP (x, 1)) == SUBREG
42252 && GET_CODE (XEXP (XEXP (x, 1), 0)) == AND)
42254 /* Return the cost after shift-and truncation. */
42255 *total = cost->shift_var;
42259 *total = cost->shift_var;
42267 gcc_assert (FLOAT_MODE_P (mode));
42268 gcc_assert (TARGET_FMA || TARGET_FMA4 || TARGET_AVX512F);
42270 /* ??? SSE scalar/vector cost should be used here. */
42271 /* ??? Bald assumption that fma has the same cost as fmul. */
42272 *total = cost->fmul;
42273 *total += rtx_cost (XEXP (x, 1), FMA, 1, speed);
42275 /* Negate in op0 or op2 is free: FMS, FNMA, FNMS. */
42277 if (GET_CODE (sub) == NEG)
42278 sub = XEXP (sub, 0);
42279 *total += rtx_cost (sub, FMA, 0, speed);
42282 if (GET_CODE (sub) == NEG)
42283 sub = XEXP (sub, 0);
42284 *total += rtx_cost (sub, FMA, 2, speed);
42289 if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
42291 /* ??? SSE scalar cost should be used here. */
42292 *total = cost->fmul;
42295 else if (X87_FLOAT_MODE_P (mode))
42297 *total = cost->fmul;
42300 else if (FLOAT_MODE_P (mode))
42302 /* ??? SSE vector cost should be used here. */
42303 *total = cost->fmul;
42306 else if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
42308 /* V*QImode is emulated with 7-13 insns. */
42309 if (mode == V16QImode || mode == V32QImode)
42312 if (TARGET_XOP && mode == V16QImode)
42314 else if (TARGET_SSSE3)
42316 *total = cost->fmul * 2 + cost->fabs * extra;
42318 /* V*DImode is emulated with 5-8 insns. */
42319 else if (mode == V2DImode || mode == V4DImode)
42321 if (TARGET_XOP && mode == V2DImode)
42322 *total = cost->fmul * 2 + cost->fabs * 3;
42324 *total = cost->fmul * 3 + cost->fabs * 5;
42326 /* Without sse4.1, we don't have PMULLD; it's emulated with 7
42327 insns, including two PMULUDQ. */
42328 else if (mode == V4SImode && !(TARGET_SSE4_1 || TARGET_AVX))
42329 *total = cost->fmul * 2 + cost->fabs * 5;
42331 *total = cost->fmul;
42336 rtx op0 = XEXP (x, 0);
42337 rtx op1 = XEXP (x, 1);
42339 if (CONST_INT_P (XEXP (x, 1)))
42341 unsigned HOST_WIDE_INT value = INTVAL (XEXP (x, 1));
42342 for (nbits = 0; value != 0; value &= value - 1)
42346 /* This is arbitrary. */
42349 /* Compute costs correctly for widening multiplication. */
42350 if ((GET_CODE (op0) == SIGN_EXTEND || GET_CODE (op0) == ZERO_EXTEND)
42351 && GET_MODE_SIZE (GET_MODE (XEXP (op0, 0))) * 2
42352 == GET_MODE_SIZE (mode))
42354 int is_mulwiden = 0;
42355 machine_mode inner_mode = GET_MODE (op0);
42357 if (GET_CODE (op0) == GET_CODE (op1))
42358 is_mulwiden = 1, op1 = XEXP (op1, 0);
42359 else if (CONST_INT_P (op1))
42361 if (GET_CODE (op0) == SIGN_EXTEND)
42362 is_mulwiden = trunc_int_for_mode (INTVAL (op1), inner_mode)
42365 is_mulwiden = !(INTVAL (op1) & ~GET_MODE_MASK (inner_mode));
42369 op0 = XEXP (op0, 0), mode = GET_MODE (op0);
42372 *total = (cost->mult_init[MODE_INDEX (mode)]
42373 + nbits * cost->mult_bit
42374 + rtx_cost (op0, outer_code, opno, speed)
42375 + rtx_cost (op1, outer_code, opno, speed));
42384 if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
42385 /* ??? SSE cost should be used here. */
42386 *total = cost->fdiv;
42387 else if (X87_FLOAT_MODE_P (mode))
42388 *total = cost->fdiv;
42389 else if (FLOAT_MODE_P (mode))
42390 /* ??? SSE vector cost should be used here. */
42391 *total = cost->fdiv;
42393 *total = cost->divide[MODE_INDEX (mode)];
42397 if (GET_MODE_CLASS (mode) == MODE_INT
42398 && GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
42400 if (GET_CODE (XEXP (x, 0)) == PLUS
42401 && GET_CODE (XEXP (XEXP (x, 0), 0)) == MULT
42402 && CONST_INT_P (XEXP (XEXP (XEXP (x, 0), 0), 1))
42403 && CONSTANT_P (XEXP (x, 1)))
42405 HOST_WIDE_INT val = INTVAL (XEXP (XEXP (XEXP (x, 0), 0), 1));
42406 if (val == 2 || val == 4 || val == 8)
42408 *total = cost->lea;
42409 *total += rtx_cost (XEXP (XEXP (x, 0), 1),
42410 outer_code, opno, speed);
42411 *total += rtx_cost (XEXP (XEXP (XEXP (x, 0), 0), 0),
42412 outer_code, opno, speed);
42413 *total += rtx_cost (XEXP (x, 1), outer_code, opno, speed);
42417 else if (GET_CODE (XEXP (x, 0)) == MULT
42418 && CONST_INT_P (XEXP (XEXP (x, 0), 1)))
42420 HOST_WIDE_INT val = INTVAL (XEXP (XEXP (x, 0), 1));
42421 if (val == 2 || val == 4 || val == 8)
42423 *total = cost->lea;
42424 *total += rtx_cost (XEXP (XEXP (x, 0), 0),
42425 outer_code, opno, speed);
42426 *total += rtx_cost (XEXP (x, 1), outer_code, opno, speed);
42430 else if (GET_CODE (XEXP (x, 0)) == PLUS)
42432 *total = cost->lea;
42433 *total += rtx_cost (XEXP (XEXP (x, 0), 0),
42434 outer_code, opno, speed);
42435 *total += rtx_cost (XEXP (XEXP (x, 0), 1),
42436 outer_code, opno, speed);
42437 *total += rtx_cost (XEXP (x, 1), outer_code, opno, speed);
42444 if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
42446 /* ??? SSE cost should be used here. */
42447 *total = cost->fadd;
42450 else if (X87_FLOAT_MODE_P (mode))
42452 *total = cost->fadd;
42455 else if (FLOAT_MODE_P (mode))
42457 /* ??? SSE vector cost should be used here. */
42458 *total = cost->fadd;
42466 if (GET_MODE_CLASS (mode) == MODE_INT
42467 && GET_MODE_SIZE (mode) > UNITS_PER_WORD)
42469 *total = (cost->add * 2
42470 + (rtx_cost (XEXP (x, 0), outer_code, opno, speed)
42471 << (GET_MODE (XEXP (x, 0)) != DImode))
42472 + (rtx_cost (XEXP (x, 1), outer_code, opno, speed)
42473 << (GET_MODE (XEXP (x, 1)) != DImode)));
42479 if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
42481 /* ??? SSE cost should be used here. */
42482 *total = cost->fchs;
42485 else if (X87_FLOAT_MODE_P (mode))
42487 *total = cost->fchs;
42490 else if (FLOAT_MODE_P (mode))
42492 /* ??? SSE vector cost should be used here. */
42493 *total = cost->fchs;
42499 if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
42501 /* ??? Should be SSE vector operation cost. */
42502 /* At least for published AMD latencies, this really is the same
42503 as the latency for a simple fpu operation like fabs. */
42504 *total = cost->fabs;
42506 else if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
42507 *total = cost->add * 2;
42509 *total = cost->add;
42513 if (GET_CODE (XEXP (x, 0)) == ZERO_EXTRACT
42514 && XEXP (XEXP (x, 0), 1) == const1_rtx
42515 && CONST_INT_P (XEXP (XEXP (x, 0), 2))
42516 && XEXP (x, 1) == const0_rtx)
42518 /* This kind of construct is implemented using test[bwl].
42519 Treat it as if we had an AND. */
42520 *total = (cost->add
42521 + rtx_cost (XEXP (XEXP (x, 0), 0), outer_code, opno, speed)
42522 + rtx_cost (const1_rtx, outer_code, opno, speed));
42528 if (!(SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH))
42533 if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
42534 /* ??? SSE cost should be used here. */
42535 *total = cost->fabs;
42536 else if (X87_FLOAT_MODE_P (mode))
42537 *total = cost->fabs;
42538 else if (FLOAT_MODE_P (mode))
42539 /* ??? SSE vector cost should be used here. */
42540 *total = cost->fabs;
42544 if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
42545 /* ??? SSE cost should be used here. */
42546 *total = cost->fsqrt;
42547 else if (X87_FLOAT_MODE_P (mode))
42548 *total = cost->fsqrt;
42549 else if (FLOAT_MODE_P (mode))
42550 /* ??? SSE vector cost should be used here. */
42551 *total = cost->fsqrt;
42555 if (XINT (x, 1) == UNSPEC_TP)
42561 case VEC_DUPLICATE:
42562 /* ??? Assume all of these vector manipulation patterns are
42563 recognizable. In which case they all pretty much have the
42565 *total = cost->fabs;
42568 mask = XEXP (x, 2);
42569 /* This is masked instruction, assume the same cost,
42570 as nonmasked variant. */
42571 if (TARGET_AVX512F && register_operand (mask, GET_MODE (mask)))
42572 *total = rtx_cost (XEXP (x, 0), outer_code, opno, speed);
42574 *total = cost->fabs;
42584 static int current_machopic_label_num;
42586 /* Given a symbol name and its associated stub, write out the
42587 definition of the stub. */
42590 machopic_output_stub (FILE *file, const char *symb, const char *stub)
42592 unsigned int length;
42593 char *binder_name, *symbol_name, lazy_ptr_name[32];
42594 int label = ++current_machopic_label_num;
42596 /* For 64-bit we shouldn't get here. */
42597 gcc_assert (!TARGET_64BIT);
42599 /* Lose our funky encoding stuff so it doesn't contaminate the stub. */
42600 symb = targetm.strip_name_encoding (symb);
42602 length = strlen (stub);
42603 binder_name = XALLOCAVEC (char, length + 32);
42604 GEN_BINDER_NAME_FOR_STUB (binder_name, stub, length);
42606 length = strlen (symb);
42607 symbol_name = XALLOCAVEC (char, length + 32);
42608 GEN_SYMBOL_NAME_FOR_SYMBOL (symbol_name, symb, length);
42610 sprintf (lazy_ptr_name, "L%d$lz", label);
42612 if (MACHOPIC_ATT_STUB)
42613 switch_to_section (darwin_sections[machopic_picsymbol_stub3_section]);
42614 else if (MACHOPIC_PURE)
42615 switch_to_section (darwin_sections[machopic_picsymbol_stub2_section]);
42617 switch_to_section (darwin_sections[machopic_symbol_stub_section]);
42619 fprintf (file, "%s:\n", stub);
42620 fprintf (file, "\t.indirect_symbol %s\n", symbol_name);
42622 if (MACHOPIC_ATT_STUB)
42624 fprintf (file, "\thlt ; hlt ; hlt ; hlt ; hlt\n");
42626 else if (MACHOPIC_PURE)
42629 /* 25-byte PIC stub using "CALL get_pc_thunk". */
42630 rtx tmp = gen_rtx_REG (SImode, 2 /* ECX */);
42631 output_set_got (tmp, NULL_RTX); /* "CALL ___<cpu>.get_pc_thunk.cx". */
42632 fprintf (file, "LPC$%d:\tmovl\t%s-LPC$%d(%%ecx),%%ecx\n",
42633 label, lazy_ptr_name, label);
42634 fprintf (file, "\tjmp\t*%%ecx\n");
42637 fprintf (file, "\tjmp\t*%s\n", lazy_ptr_name);
42639 /* The AT&T-style ("self-modifying") stub is not lazily bound, thus
42640 it needs no stub-binding-helper. */
42641 if (MACHOPIC_ATT_STUB)
42644 fprintf (file, "%s:\n", binder_name);
42648 fprintf (file, "\tlea\t%s-%s(%%ecx),%%ecx\n", lazy_ptr_name, binder_name);
42649 fprintf (file, "\tpushl\t%%ecx\n");
42652 fprintf (file, "\tpushl\t$%s\n", lazy_ptr_name);
42654 fputs ("\tjmp\tdyld_stub_binding_helper\n", file);
42656 /* N.B. Keep the correspondence of these
42657 'symbol_ptr/symbol_ptr2/symbol_ptr3' sections consistent with the
42658 old-pic/new-pic/non-pic stubs; altering this will break
42659 compatibility with existing dylibs. */
42662 /* 25-byte PIC stub using "CALL get_pc_thunk". */
42663 switch_to_section (darwin_sections[machopic_lazy_symbol_ptr2_section]);
42666 /* 16-byte -mdynamic-no-pic stub. */
42667 switch_to_section(darwin_sections[machopic_lazy_symbol_ptr3_section]);
42669 fprintf (file, "%s:\n", lazy_ptr_name);
42670 fprintf (file, "\t.indirect_symbol %s\n", symbol_name);
42671 fprintf (file, ASM_LONG "%s\n", binder_name);
42673 #endif /* TARGET_MACHO */
42675 /* Order the registers for register allocator. */
42678 x86_order_regs_for_local_alloc (void)
42683 /* First allocate the local general purpose registers. */
42684 for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
42685 if (GENERAL_REGNO_P (i) && call_used_regs[i])
42686 reg_alloc_order [pos++] = i;
42688 /* Global general purpose registers. */
42689 for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
42690 if (GENERAL_REGNO_P (i) && !call_used_regs[i])
42691 reg_alloc_order [pos++] = i;
42693 /* x87 registers come first in case we are doing FP math
42695 if (!TARGET_SSE_MATH)
42696 for (i = FIRST_STACK_REG; i <= LAST_STACK_REG; i++)
42697 reg_alloc_order [pos++] = i;
42699 /* SSE registers. */
42700 for (i = FIRST_SSE_REG; i <= LAST_SSE_REG; i++)
42701 reg_alloc_order [pos++] = i;
42702 for (i = FIRST_REX_SSE_REG; i <= LAST_REX_SSE_REG; i++)
42703 reg_alloc_order [pos++] = i;
42705 /* Extended REX SSE registers. */
42706 for (i = FIRST_EXT_REX_SSE_REG; i <= LAST_EXT_REX_SSE_REG; i++)
42707 reg_alloc_order [pos++] = i;
42709 /* Mask register. */
42710 for (i = FIRST_MASK_REG; i <= LAST_MASK_REG; i++)
42711 reg_alloc_order [pos++] = i;
42713 /* MPX bound registers. */
42714 for (i = FIRST_BND_REG; i <= LAST_BND_REG; i++)
42715 reg_alloc_order [pos++] = i;
42717 /* x87 registers. */
42718 if (TARGET_SSE_MATH)
42719 for (i = FIRST_STACK_REG; i <= LAST_STACK_REG; i++)
42720 reg_alloc_order [pos++] = i;
42722 for (i = FIRST_MMX_REG; i <= LAST_MMX_REG; i++)
42723 reg_alloc_order [pos++] = i;
42725 /* Initialize the rest of array as we do not allocate some registers
42727 while (pos < FIRST_PSEUDO_REGISTER)
42728 reg_alloc_order [pos++] = 0;
42731 /* Handle a "callee_pop_aggregate_return" attribute; arguments as
42732 in struct attribute_spec handler. */
42734 ix86_handle_callee_pop_aggregate_return (tree *node, tree name,
42737 bool *no_add_attrs)
42739 if (TREE_CODE (*node) != FUNCTION_TYPE
42740 && TREE_CODE (*node) != METHOD_TYPE
42741 && TREE_CODE (*node) != FIELD_DECL
42742 && TREE_CODE (*node) != TYPE_DECL)
42744 warning (OPT_Wattributes, "%qE attribute only applies to functions",
42746 *no_add_attrs = true;
42751 warning (OPT_Wattributes, "%qE attribute only available for 32-bit",
42753 *no_add_attrs = true;
42756 if (is_attribute_p ("callee_pop_aggregate_return", name))
42760 cst = TREE_VALUE (args);
42761 if (TREE_CODE (cst) != INTEGER_CST)
42763 warning (OPT_Wattributes,
42764 "%qE attribute requires an integer constant argument",
42766 *no_add_attrs = true;
42768 else if (compare_tree_int (cst, 0) != 0
42769 && compare_tree_int (cst, 1) != 0)
42771 warning (OPT_Wattributes,
42772 "argument to %qE attribute is neither zero, nor one",
42774 *no_add_attrs = true;
42783 /* Handle a "ms_abi" or "sysv" attribute; arguments as in
42784 struct attribute_spec.handler. */
42786 ix86_handle_abi_attribute (tree *node, tree name, tree, int,
42787 bool *no_add_attrs)
42789 if (TREE_CODE (*node) != FUNCTION_TYPE
42790 && TREE_CODE (*node) != METHOD_TYPE
42791 && TREE_CODE (*node) != FIELD_DECL
42792 && TREE_CODE (*node) != TYPE_DECL)
42794 warning (OPT_Wattributes, "%qE attribute only applies to functions",
42796 *no_add_attrs = true;
42800 /* Can combine regparm with all attributes but fastcall. */
42801 if (is_attribute_p ("ms_abi", name))
42803 if (lookup_attribute ("sysv_abi", TYPE_ATTRIBUTES (*node)))
42805 error ("ms_abi and sysv_abi attributes are not compatible");
42810 else if (is_attribute_p ("sysv_abi", name))
42812 if (lookup_attribute ("ms_abi", TYPE_ATTRIBUTES (*node)))
42814 error ("ms_abi and sysv_abi attributes are not compatible");
42823 /* Handle a "ms_struct" or "gcc_struct" attribute; arguments as in
42824 struct attribute_spec.handler. */
42826 ix86_handle_struct_attribute (tree *node, tree name, tree, int,
42827 bool *no_add_attrs)
42830 if (DECL_P (*node))
42832 if (TREE_CODE (*node) == TYPE_DECL)
42833 type = &TREE_TYPE (*node);
42838 if (!(type && RECORD_OR_UNION_TYPE_P (*type)))
42840 warning (OPT_Wattributes, "%qE attribute ignored",
42842 *no_add_attrs = true;
42845 else if ((is_attribute_p ("ms_struct", name)
42846 && lookup_attribute ("gcc_struct", TYPE_ATTRIBUTES (*type)))
42847 || ((is_attribute_p ("gcc_struct", name)
42848 && lookup_attribute ("ms_struct", TYPE_ATTRIBUTES (*type)))))
42850 warning (OPT_Wattributes, "%qE incompatible attribute ignored",
42852 *no_add_attrs = true;
42859 ix86_handle_fndecl_attribute (tree *node, tree name, tree, int,
42860 bool *no_add_attrs)
42862 if (TREE_CODE (*node) != FUNCTION_DECL)
42864 warning (OPT_Wattributes, "%qE attribute only applies to functions",
42866 *no_add_attrs = true;
42872 ix86_ms_bitfield_layout_p (const_tree record_type)
42874 return ((TARGET_MS_BITFIELD_LAYOUT
42875 && !lookup_attribute ("gcc_struct", TYPE_ATTRIBUTES (record_type)))
42876 || lookup_attribute ("ms_struct", TYPE_ATTRIBUTES (record_type)));
42879 /* Returns an expression indicating where the this parameter is
42880 located on entry to the FUNCTION. */
42883 x86_this_parameter (tree function)
42885 tree type = TREE_TYPE (function);
42886 bool aggr = aggregate_value_p (TREE_TYPE (type), type) != 0;
42891 const int *parm_regs;
42893 if (ix86_function_type_abi (type) == MS_ABI)
42894 parm_regs = x86_64_ms_abi_int_parameter_registers;
42896 parm_regs = x86_64_int_parameter_registers;
42897 return gen_rtx_REG (Pmode, parm_regs[aggr]);
42900 nregs = ix86_function_regparm (type, function);
42902 if (nregs > 0 && !stdarg_p (type))
42905 unsigned int ccvt = ix86_get_callcvt (type);
42907 if ((ccvt & IX86_CALLCVT_FASTCALL) != 0)
42908 regno = aggr ? DX_REG : CX_REG;
42909 else if ((ccvt & IX86_CALLCVT_THISCALL) != 0)
42913 return gen_rtx_MEM (SImode,
42914 plus_constant (Pmode, stack_pointer_rtx, 4));
42923 return gen_rtx_MEM (SImode,
42924 plus_constant (Pmode,
42925 stack_pointer_rtx, 4));
42928 return gen_rtx_REG (SImode, regno);
42931 return gen_rtx_MEM (SImode, plus_constant (Pmode, stack_pointer_rtx,
42935 /* Determine whether x86_output_mi_thunk can succeed. */
42938 x86_can_output_mi_thunk (const_tree, HOST_WIDE_INT, HOST_WIDE_INT vcall_offset,
42939 const_tree function)
42941 /* 64-bit can handle anything. */
42945 /* For 32-bit, everything's fine if we have one free register. */
42946 if (ix86_function_regparm (TREE_TYPE (function), function) < 3)
42949 /* Need a free register for vcall_offset. */
42953 /* Need a free register for GOT references. */
42954 if (flag_pic && !targetm.binds_local_p (function))
42957 /* Otherwise ok. */
42961 /* Output the assembler code for a thunk function. THUNK_DECL is the
42962 declaration for the thunk function itself, FUNCTION is the decl for
42963 the target function. DELTA is an immediate constant offset to be
42964 added to THIS. If VCALL_OFFSET is nonzero, the word at
42965 *(*this + vcall_offset) should be added to THIS. */
42968 x86_output_mi_thunk (FILE *file, tree, HOST_WIDE_INT delta,
42969 HOST_WIDE_INT vcall_offset, tree function)
42971 rtx this_param = x86_this_parameter (function);
42972 rtx this_reg, tmp, fnaddr;
42973 unsigned int tmp_regno;
42977 tmp_regno = R10_REG;
42980 unsigned int ccvt = ix86_get_callcvt (TREE_TYPE (function));
42981 if ((ccvt & IX86_CALLCVT_FASTCALL) != 0)
42982 tmp_regno = AX_REG;
42983 else if ((ccvt & IX86_CALLCVT_THISCALL) != 0)
42984 tmp_regno = DX_REG;
42986 tmp_regno = CX_REG;
42989 emit_note (NOTE_INSN_PROLOGUE_END);
42991 /* If VCALL_OFFSET, we'll need THIS in a register. Might as well
42992 pull it in now and let DELTA benefit. */
42993 if (REG_P (this_param))
42994 this_reg = this_param;
42995 else if (vcall_offset)
42997 /* Put the this parameter into %eax. */
42998 this_reg = gen_rtx_REG (Pmode, AX_REG);
42999 emit_move_insn (this_reg, this_param);
43002 this_reg = NULL_RTX;
43004 /* Adjust the this parameter by a fixed constant. */
43007 rtx delta_rtx = GEN_INT (delta);
43008 rtx delta_dst = this_reg ? this_reg : this_param;
43012 if (!x86_64_general_operand (delta_rtx, Pmode))
43014 tmp = gen_rtx_REG (Pmode, tmp_regno);
43015 emit_move_insn (tmp, delta_rtx);
43020 ix86_emit_binop (PLUS, Pmode, delta_dst, delta_rtx);
43023 /* Adjust the this parameter by a value stored in the vtable. */
43026 rtx vcall_addr, vcall_mem, this_mem;
43028 tmp = gen_rtx_REG (Pmode, tmp_regno);
43030 this_mem = gen_rtx_MEM (ptr_mode, this_reg);
43031 if (Pmode != ptr_mode)
43032 this_mem = gen_rtx_ZERO_EXTEND (Pmode, this_mem);
43033 emit_move_insn (tmp, this_mem);
43035 /* Adjust the this parameter. */
43036 vcall_addr = plus_constant (Pmode, tmp, vcall_offset);
43038 && !ix86_legitimate_address_p (ptr_mode, vcall_addr, true))
43040 rtx tmp2 = gen_rtx_REG (Pmode, R11_REG);
43041 emit_move_insn (tmp2, GEN_INT (vcall_offset));
43042 vcall_addr = gen_rtx_PLUS (Pmode, tmp, tmp2);
43045 vcall_mem = gen_rtx_MEM (ptr_mode, vcall_addr);
43046 if (Pmode != ptr_mode)
43047 emit_insn (gen_addsi_1_zext (this_reg,
43048 gen_rtx_REG (ptr_mode,
43052 ix86_emit_binop (PLUS, Pmode, this_reg, vcall_mem);
43055 /* If necessary, drop THIS back to its stack slot. */
43056 if (this_reg && this_reg != this_param)
43057 emit_move_insn (this_param, this_reg);
43059 fnaddr = XEXP (DECL_RTL (function), 0);
43062 if (!flag_pic || targetm.binds_local_p (function)
43067 tmp = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, fnaddr), UNSPEC_GOTPCREL);
43068 tmp = gen_rtx_CONST (Pmode, tmp);
43069 fnaddr = gen_const_mem (Pmode, tmp);
43074 if (!flag_pic || targetm.binds_local_p (function))
43077 else if (TARGET_MACHO)
43079 fnaddr = machopic_indirect_call_target (DECL_RTL (function));
43080 fnaddr = XEXP (fnaddr, 0);
43082 #endif /* TARGET_MACHO */
43085 tmp = gen_rtx_REG (Pmode, CX_REG);
43086 output_set_got (tmp, NULL_RTX);
43088 fnaddr = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, fnaddr), UNSPEC_GOT);
43089 fnaddr = gen_rtx_CONST (Pmode, fnaddr);
43090 fnaddr = gen_rtx_PLUS (Pmode, tmp, fnaddr);
43091 fnaddr = gen_const_mem (Pmode, fnaddr);
43095 /* Our sibling call patterns do not allow memories, because we have no
43096 predicate that can distinguish between frame and non-frame memory.
43097 For our purposes here, we can get away with (ab)using a jump pattern,
43098 because we're going to do no optimization. */
43099 if (MEM_P (fnaddr))
43101 if (sibcall_insn_operand (fnaddr, word_mode))
43103 fnaddr = XEXP (DECL_RTL (function), 0);
43104 tmp = gen_rtx_MEM (QImode, fnaddr);
43105 tmp = gen_rtx_CALL (VOIDmode, tmp, const0_rtx);
43106 tmp = emit_call_insn (tmp);
43107 SIBLING_CALL_P (tmp) = 1;
43110 emit_jump_insn (gen_indirect_jump (fnaddr));
43114 if (ix86_cmodel == CM_LARGE_PIC && SYMBOLIC_CONST (fnaddr))
43116 // CM_LARGE_PIC always uses pseudo PIC register which is
43117 // uninitialized. Since FUNCTION is local and calling it
43118 // doesn't go through PLT, we use scratch register %r11 as
43119 // PIC register and initialize it here.
43120 pic_offset_table_rtx = gen_rtx_REG (Pmode, R11_REG);
43121 ix86_init_large_pic_reg (tmp_regno);
43122 fnaddr = legitimize_pic_address (fnaddr,
43123 gen_rtx_REG (Pmode, tmp_regno));
43126 if (!sibcall_insn_operand (fnaddr, word_mode))
43128 tmp = gen_rtx_REG (word_mode, tmp_regno);
43129 if (GET_MODE (fnaddr) != word_mode)
43130 fnaddr = gen_rtx_ZERO_EXTEND (word_mode, fnaddr);
43131 emit_move_insn (tmp, fnaddr);
43135 tmp = gen_rtx_MEM (QImode, fnaddr);
43136 tmp = gen_rtx_CALL (VOIDmode, tmp, const0_rtx);
43137 tmp = emit_call_insn (tmp);
43138 SIBLING_CALL_P (tmp) = 1;
43142 /* Emit just enough of rest_of_compilation to get the insns emitted.
43143 Note that use_thunk calls assemble_start_function et al. */
43144 insn = get_insns ();
43145 shorten_branches (insn);
43146 final_start_function (insn, file, 1);
43147 final (insn, file, 1);
43148 final_end_function ();
43152 x86_file_start (void)
43154 default_file_start ();
43156 fputs ("\t.code16gcc\n", asm_out_file);
43158 darwin_file_start ();
43160 if (X86_FILE_START_VERSION_DIRECTIVE)
43161 fputs ("\t.version\t\"01.01\"\n", asm_out_file);
43162 if (X86_FILE_START_FLTUSED)
43163 fputs ("\t.global\t__fltused\n", asm_out_file);
43164 if (ix86_asm_dialect == ASM_INTEL)
43165 fputs ("\t.intel_syntax noprefix\n", asm_out_file);
43169 x86_field_alignment (tree field, int computed)
43172 tree type = TREE_TYPE (field);
43174 if (TARGET_64BIT || TARGET_ALIGN_DOUBLE)
43176 mode = TYPE_MODE (strip_array_types (type));
43177 if (mode == DFmode || mode == DCmode
43178 || GET_MODE_CLASS (mode) == MODE_INT
43179 || GET_MODE_CLASS (mode) == MODE_COMPLEX_INT)
43180 return MIN (32, computed);
43184 /* Print call to TARGET to FILE. */
43187 x86_print_call_or_nop (FILE *file, const char *target)
43189 if (flag_nop_mcount)
43190 fprintf (file, "1:\tnopl 0x00(%%eax,%%eax,1)\n"); /* 5 byte nop. */
43192 fprintf (file, "1:\tcall\t%s\n", target);
43195 /* Output assembler code to FILE to increment profiler label # LABELNO
43196 for profiling a function entry. */
43198 x86_function_profiler (FILE *file, int labelno ATTRIBUTE_UNUSED)
43200 const char *mcount_name = (flag_fentry ? MCOUNT_NAME_BEFORE_PROLOGUE
43204 #ifndef NO_PROFILE_COUNTERS
43205 fprintf (file, "\tleaq\t%sP%d(%%rip),%%r11\n", LPREFIX, labelno);
43208 if (!TARGET_PECOFF && flag_pic)
43209 fprintf (file, "1:\tcall\t*%s@GOTPCREL(%%rip)\n", mcount_name);
43211 x86_print_call_or_nop (file, mcount_name);
43215 #ifndef NO_PROFILE_COUNTERS
43216 fprintf (file, "\tleal\t%sP%d@GOTOFF(%%ebx),%%" PROFILE_COUNT_REGISTER "\n",
43219 fprintf (file, "1:\tcall\t*%s@GOT(%%ebx)\n", mcount_name);
43223 #ifndef NO_PROFILE_COUNTERS
43224 fprintf (file, "\tmovl\t$%sP%d,%%" PROFILE_COUNT_REGISTER "\n",
43227 x86_print_call_or_nop (file, mcount_name);
43230 if (flag_record_mcount)
43232 fprintf (file, "\t.section __mcount_loc, \"a\",@progbits\n");
43233 fprintf (file, "\t.%s 1b\n", TARGET_64BIT ? "quad" : "long");
43234 fprintf (file, "\t.previous\n");
43238 /* We don't have exact information about the insn sizes, but we may assume
43239 quite safely that we are informed about all 1 byte insns and memory
43240 address sizes. This is enough to eliminate unnecessary padding in
43244 min_insn_size (rtx_insn *insn)
43248 if (!INSN_P (insn) || !active_insn_p (insn))
43251 /* Discard alignments we've emit and jump instructions. */
43252 if (GET_CODE (PATTERN (insn)) == UNSPEC_VOLATILE
43253 && XINT (PATTERN (insn), 1) == UNSPECV_ALIGN)
43256 /* Important case - calls are always 5 bytes.
43257 It is common to have many calls in the row. */
43259 && symbolic_reference_mentioned_p (PATTERN (insn))
43260 && !SIBLING_CALL_P (insn))
43262 len = get_attr_length (insn);
43266 /* For normal instructions we rely on get_attr_length being exact,
43267 with a few exceptions. */
43268 if (!JUMP_P (insn))
43270 enum attr_type type = get_attr_type (insn);
43275 if (GET_CODE (PATTERN (insn)) == ASM_INPUT
43276 || asm_noperands (PATTERN (insn)) >= 0)
43283 /* Otherwise trust get_attr_length. */
43287 l = get_attr_length_address (insn);
43288 if (l < 4 && symbolic_reference_mentioned_p (PATTERN (insn)))
43297 #ifdef ASM_OUTPUT_MAX_SKIP_PAD
43299 /* AMD K8 core mispredicts jumps when there are more than 3 jumps in 16 byte
43303 ix86_avoid_jump_mispredicts (void)
43305 rtx_insn *insn, *start = get_insns ();
43306 int nbytes = 0, njumps = 0;
43307 bool isjump = false;
43309 /* Look for all minimal intervals of instructions containing 4 jumps.
43310 The intervals are bounded by START and INSN. NBYTES is the total
43311 size of instructions in the interval including INSN and not including
43312 START. When the NBYTES is smaller than 16 bytes, it is possible
43313 that the end of START and INSN ends up in the same 16byte page.
43315 The smallest offset in the page INSN can start is the case where START
43316 ends on the offset 0. Offset of INSN is then NBYTES - sizeof (INSN).
43317 We add p2align to 16byte window with maxskip 15 - NBYTES + sizeof (INSN).
43319 Don't consider asm goto as jump, while it can contain a jump, it doesn't
43320 have to, control transfer to label(s) can be performed through other
43321 means, and also we estimate minimum length of all asm stmts as 0. */
43322 for (insn = start; insn; insn = NEXT_INSN (insn))
43326 if (LABEL_P (insn))
43328 int align = label_to_alignment (insn);
43329 int max_skip = label_to_max_skip (insn);
43333 /* If align > 3, only up to 16 - max_skip - 1 bytes can be
43334 already in the current 16 byte page, because otherwise
43335 ASM_OUTPUT_MAX_SKIP_ALIGN could skip max_skip or fewer
43336 bytes to reach 16 byte boundary. */
43338 || (align <= 3 && max_skip != (1 << align) - 1))
43341 fprintf (dump_file, "Label %i with max_skip %i\n",
43342 INSN_UID (insn), max_skip);
43345 while (nbytes + max_skip >= 16)
43347 start = NEXT_INSN (start);
43348 if ((JUMP_P (start) && asm_noperands (PATTERN (start)) < 0)
43350 njumps--, isjump = true;
43353 nbytes -= min_insn_size (start);
43359 min_size = min_insn_size (insn);
43360 nbytes += min_size;
43362 fprintf (dump_file, "Insn %i estimated to %i bytes\n",
43363 INSN_UID (insn), min_size);
43364 if ((JUMP_P (insn) && asm_noperands (PATTERN (insn)) < 0)
43372 start = NEXT_INSN (start);
43373 if ((JUMP_P (start) && asm_noperands (PATTERN (start)) < 0)
43375 njumps--, isjump = true;
43378 nbytes -= min_insn_size (start);
43380 gcc_assert (njumps >= 0);
43382 fprintf (dump_file, "Interval %i to %i has %i bytes\n",
43383 INSN_UID (start), INSN_UID (insn), nbytes);
43385 if (njumps == 3 && isjump && nbytes < 16)
43387 int padsize = 15 - nbytes + min_insn_size (insn);
43390 fprintf (dump_file, "Padding insn %i by %i bytes!\n",
43391 INSN_UID (insn), padsize);
43392 emit_insn_before (gen_pad (GEN_INT (padsize)), insn);
43398 /* AMD Athlon works faster
43399 when RET is not destination of conditional jump or directly preceded
43400 by other jump instruction. We avoid the penalty by inserting NOP just
43401 before the RET instructions in such cases. */
43403 ix86_pad_returns (void)
43408 FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
43410 basic_block bb = e->src;
43411 rtx_insn *ret = BB_END (bb);
43413 bool replace = false;
43415 if (!JUMP_P (ret) || !ANY_RETURN_P (PATTERN (ret))
43416 || optimize_bb_for_size_p (bb))
43418 for (prev = PREV_INSN (ret); prev; prev = PREV_INSN (prev))
43419 if (active_insn_p (prev) || LABEL_P (prev))
43421 if (prev && LABEL_P (prev))
43426 FOR_EACH_EDGE (e, ei, bb->preds)
43427 if (EDGE_FREQUENCY (e) && e->src->index >= 0
43428 && !(e->flags & EDGE_FALLTHRU))
43436 prev = prev_active_insn (ret);
43438 && ((JUMP_P (prev) && any_condjump_p (prev))
43441 /* Empty functions get branch mispredict even when
43442 the jump destination is not visible to us. */
43443 if (!prev && !optimize_function_for_size_p (cfun))
43448 emit_jump_insn_before (gen_simple_return_internal_long (), ret);
43454 /* Count the minimum number of instructions in BB. Return 4 if the
43455 number of instructions >= 4. */
43458 ix86_count_insn_bb (basic_block bb)
43461 int insn_count = 0;
43463 /* Count number of instructions in this block. Return 4 if the number
43464 of instructions >= 4. */
43465 FOR_BB_INSNS (bb, insn)
43467 /* Only happen in exit blocks. */
43469 && ANY_RETURN_P (PATTERN (insn)))
43472 if (NONDEBUG_INSN_P (insn)
43473 && GET_CODE (PATTERN (insn)) != USE
43474 && GET_CODE (PATTERN (insn)) != CLOBBER)
43477 if (insn_count >= 4)
43486 /* Count the minimum number of instructions in code path in BB.
43487 Return 4 if the number of instructions >= 4. */
43490 ix86_count_insn (basic_block bb)
43494 int min_prev_count;
43496 /* Only bother counting instructions along paths with no
43497 more than 2 basic blocks between entry and exit. Given
43498 that BB has an edge to exit, determine if a predecessor
43499 of BB has an edge from entry. If so, compute the number
43500 of instructions in the predecessor block. If there
43501 happen to be multiple such blocks, compute the minimum. */
43502 min_prev_count = 4;
43503 FOR_EACH_EDGE (e, ei, bb->preds)
43506 edge_iterator prev_ei;
43508 if (e->src == ENTRY_BLOCK_PTR_FOR_FN (cfun))
43510 min_prev_count = 0;
43513 FOR_EACH_EDGE (prev_e, prev_ei, e->src->preds)
43515 if (prev_e->src == ENTRY_BLOCK_PTR_FOR_FN (cfun))
43517 int count = ix86_count_insn_bb (e->src);
43518 if (count < min_prev_count)
43519 min_prev_count = count;
43525 if (min_prev_count < 4)
43526 min_prev_count += ix86_count_insn_bb (bb);
43528 return min_prev_count;
43531 /* Pad short function to 4 instructions. */
43534 ix86_pad_short_function (void)
43539 FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
43541 rtx_insn *ret = BB_END (e->src);
43542 if (JUMP_P (ret) && ANY_RETURN_P (PATTERN (ret)))
43544 int insn_count = ix86_count_insn (e->src);
43546 /* Pad short function. */
43547 if (insn_count < 4)
43549 rtx_insn *insn = ret;
43551 /* Find epilogue. */
43554 || NOTE_KIND (insn) != NOTE_INSN_EPILOGUE_BEG))
43555 insn = PREV_INSN (insn);
43560 /* Two NOPs count as one instruction. */
43561 insn_count = 2 * (4 - insn_count);
43562 emit_insn_before (gen_nops (GEN_INT (insn_count)), insn);
43568 /* Fix up a Windows system unwinder issue. If an EH region falls through into
43569 the epilogue, the Windows system unwinder will apply epilogue logic and
43570 produce incorrect offsets. This can be avoided by adding a nop between
43571 the last insn that can throw and the first insn of the epilogue. */
43574 ix86_seh_fixup_eh_fallthru (void)
43579 FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
43581 rtx_insn *insn, *next;
43583 /* Find the beginning of the epilogue. */
43584 for (insn = BB_END (e->src); insn != NULL; insn = PREV_INSN (insn))
43585 if (NOTE_P (insn) && NOTE_KIND (insn) == NOTE_INSN_EPILOGUE_BEG)
43590 /* We only care about preceding insns that can throw. */
43591 insn = prev_active_insn (insn);
43592 if (insn == NULL || !can_throw_internal (insn))
43595 /* Do not separate calls from their debug information. */
43596 for (next = NEXT_INSN (insn); next != NULL; next = NEXT_INSN (next))
43598 && (NOTE_KIND (next) == NOTE_INSN_VAR_LOCATION
43599 || NOTE_KIND (next) == NOTE_INSN_CALL_ARG_LOCATION))
43604 emit_insn_after (gen_nops (const1_rtx), insn);
43608 /* Implement machine specific optimizations. We implement padding of returns
43609 for K8 CPUs and pass to avoid 4 jumps in the single 16 byte window. */
43613 /* We are freeing block_for_insn in the toplev to keep compatibility
43614 with old MDEP_REORGS that are not CFG based. Recompute it now. */
43615 compute_bb_for_insn ();
43617 if (TARGET_SEH && current_function_has_exception_handlers ())
43618 ix86_seh_fixup_eh_fallthru ();
43620 if (optimize && optimize_function_for_speed_p (cfun))
43622 if (TARGET_PAD_SHORT_FUNCTION)
43623 ix86_pad_short_function ();
43624 else if (TARGET_PAD_RETURNS)
43625 ix86_pad_returns ();
43626 #ifdef ASM_OUTPUT_MAX_SKIP_PAD
43627 if (TARGET_FOUR_JUMP_LIMIT)
43628 ix86_avoid_jump_mispredicts ();
43633 /* Return nonzero when QImode register that must be represented via REX prefix
43636 x86_extended_QIreg_mentioned_p (rtx_insn *insn)
43639 extract_insn_cached (insn);
43640 for (i = 0; i < recog_data.n_operands; i++)
43641 if (GENERAL_REG_P (recog_data.operand[i])
43642 && !QI_REGNO_P (REGNO (recog_data.operand[i])))
43647 /* Return true when INSN mentions register that must be encoded using REX
43650 x86_extended_reg_mentioned_p (rtx insn)
43652 subrtx_iterator::array_type array;
43653 FOR_EACH_SUBRTX (iter, array, INSN_P (insn) ? PATTERN (insn) : insn, NONCONST)
43655 const_rtx x = *iter;
43657 && (REX_INT_REGNO_P (REGNO (x)) || REX_SSE_REGNO_P (REGNO (x))))
43663 /* If profitable, negate (without causing overflow) integer constant
43664 of mode MODE at location LOC. Return true in this case. */
43666 x86_maybe_negate_const_int (rtx *loc, machine_mode mode)
43670 if (!CONST_INT_P (*loc))
43676 /* DImode x86_64 constants must fit in 32 bits. */
43677 gcc_assert (x86_64_immediate_operand (*loc, mode));
43688 gcc_unreachable ();
43691 /* Avoid overflows. */
43692 if (mode_signbit_p (mode, *loc))
43695 val = INTVAL (*loc);
43697 /* Make things pretty and `subl $4,%eax' rather than `addl $-4,%eax'.
43698 Exceptions: -128 encodes smaller than 128, so swap sign and op. */
43699 if ((val < 0 && val != -128)
43702 *loc = GEN_INT (-val);
43709 /* Generate an unsigned DImode/SImode to FP conversion. This is the same code
43710 optabs would emit if we didn't have TFmode patterns. */
43713 x86_emit_floatuns (rtx operands[2])
43715 rtx_code_label *neglab, *donelab;
43716 rtx i0, i1, f0, in, out;
43717 machine_mode mode, inmode;
43719 inmode = GET_MODE (operands[1]);
43720 gcc_assert (inmode == SImode || inmode == DImode);
43723 in = force_reg (inmode, operands[1]);
43724 mode = GET_MODE (out);
43725 neglab = gen_label_rtx ();
43726 donelab = gen_label_rtx ();
43727 f0 = gen_reg_rtx (mode);
43729 emit_cmp_and_jump_insns (in, const0_rtx, LT, const0_rtx, inmode, 0, neglab);
43731 expand_float (out, in, 0);
43733 emit_jump_insn (gen_jump (donelab));
43736 emit_label (neglab);
43738 i0 = expand_simple_binop (inmode, LSHIFTRT, in, const1_rtx, NULL,
43740 i1 = expand_simple_binop (inmode, AND, in, const1_rtx, NULL,
43742 i0 = expand_simple_binop (inmode, IOR, i0, i1, i0, 1, OPTAB_DIRECT);
43744 expand_float (f0, i0, 0);
43746 emit_insn (gen_rtx_SET (out, gen_rtx_PLUS (mode, f0, f0)));
43748 emit_label (donelab);
43751 static bool canonicalize_perm (struct expand_vec_perm_d *d);
43752 static bool expand_vec_perm_1 (struct expand_vec_perm_d *d);
43753 static bool expand_vec_perm_broadcast_1 (struct expand_vec_perm_d *d);
43754 static bool expand_vec_perm_palignr (struct expand_vec_perm_d *d, bool);
43756 /* Get a vector mode of the same size as the original but with elements
43757 twice as wide. This is only guaranteed to apply to integral vectors. */
43759 static inline machine_mode
43760 get_mode_wider_vector (machine_mode o)
43762 /* ??? Rely on the ordering that genmodes.c gives to vectors. */
43763 machine_mode n = GET_MODE_WIDER_MODE (o);
43764 gcc_assert (GET_MODE_NUNITS (o) == GET_MODE_NUNITS (n) * 2);
43765 gcc_assert (GET_MODE_SIZE (o) == GET_MODE_SIZE (n));
43769 /* A subroutine of ix86_expand_vector_init_duplicate. Tries to
43770 fill target with val via vec_duplicate. */
43773 ix86_vector_duplicate_value (machine_mode mode, rtx target, rtx val)
43779 /* First attempt to recognize VAL as-is. */
43780 dup = gen_rtx_VEC_DUPLICATE (mode, val);
43781 insn = emit_insn (gen_rtx_SET (target, dup));
43782 if (recog_memoized (insn) < 0)
43785 /* If that fails, force VAL into a register. */
43788 XEXP (dup, 0) = force_reg (GET_MODE_INNER (mode), val);
43789 seq = get_insns ();
43792 emit_insn_before (seq, insn);
43794 ok = recog_memoized (insn) >= 0;
43800 /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
43801 with all elements equal to VAR. Return true if successful. */
43804 ix86_expand_vector_init_duplicate (bool mmx_ok, machine_mode mode,
43805 rtx target, rtx val)
43829 return ix86_vector_duplicate_value (mode, target, val);
43834 if (TARGET_SSE || TARGET_3DNOW_A)
43838 val = gen_lowpart (SImode, val);
43839 x = gen_rtx_TRUNCATE (HImode, val);
43840 x = gen_rtx_VEC_DUPLICATE (mode, x);
43841 emit_insn (gen_rtx_SET (target, x));
43853 return ix86_vector_duplicate_value (mode, target, val);
43857 struct expand_vec_perm_d dperm;
43861 memset (&dperm, 0, sizeof (dperm));
43862 dperm.target = target;
43863 dperm.vmode = mode;
43864 dperm.nelt = GET_MODE_NUNITS (mode);
43865 dperm.op0 = dperm.op1 = gen_reg_rtx (mode);
43866 dperm.one_operand_p = true;
43868 /* Extend to SImode using a paradoxical SUBREG. */
43869 tmp1 = gen_reg_rtx (SImode);
43870 emit_move_insn (tmp1, gen_lowpart (SImode, val));
43872 /* Insert the SImode value as low element of a V4SImode vector. */
43873 tmp2 = gen_reg_rtx (V4SImode);
43874 emit_insn (gen_vec_setv4si_0 (tmp2, CONST0_RTX (V4SImode), tmp1));
43875 emit_move_insn (dperm.op0, gen_lowpart (mode, tmp2));
43877 ok = (expand_vec_perm_1 (&dperm)
43878 || expand_vec_perm_broadcast_1 (&dperm));
43886 return ix86_vector_duplicate_value (mode, target, val);
43893 /* Replicate the value once into the next wider mode and recurse. */
43895 machine_mode smode, wsmode, wvmode;
43898 smode = GET_MODE_INNER (mode);
43899 wvmode = get_mode_wider_vector (mode);
43900 wsmode = GET_MODE_INNER (wvmode);
43902 val = convert_modes (wsmode, smode, val, true);
43903 x = expand_simple_binop (wsmode, ASHIFT, val,
43904 GEN_INT (GET_MODE_BITSIZE (smode)),
43905 NULL_RTX, 1, OPTAB_LIB_WIDEN);
43906 val = expand_simple_binop (wsmode, IOR, val, x, x, 1, OPTAB_LIB_WIDEN);
43908 x = gen_reg_rtx (wvmode);
43909 ok = ix86_expand_vector_init_duplicate (mmx_ok, wvmode, x, val);
43911 emit_move_insn (target, gen_lowpart (GET_MODE (target), x));
43918 return ix86_vector_duplicate_value (mode, target, val);
43921 machine_mode hvmode = (mode == V16HImode ? V8HImode : V16QImode);
43922 rtx x = gen_reg_rtx (hvmode);
43924 ok = ix86_expand_vector_init_duplicate (false, hvmode, x, val);
43927 x = gen_rtx_VEC_CONCAT (mode, x, x);
43928 emit_insn (gen_rtx_SET (target, x));
43934 if (TARGET_AVX512BW)
43935 return ix86_vector_duplicate_value (mode, target, val);
43938 machine_mode hvmode = (mode == V32HImode ? V16HImode : V32QImode);
43939 rtx x = gen_reg_rtx (hvmode);
43941 ok = ix86_expand_vector_init_duplicate (false, hvmode, x, val);
43944 x = gen_rtx_VEC_CONCAT (mode, x, x);
43945 emit_insn (gen_rtx_SET (target, x));
43954 /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
43955 whose ONE_VAR element is VAR, and other elements are zero. Return true
43959 ix86_expand_vector_init_one_nonzero (bool mmx_ok, machine_mode mode,
43960 rtx target, rtx var, int one_var)
43962 machine_mode vsimode;
43965 bool use_vector_set = false;
43970 /* For SSE4.1, we normally use vector set. But if the second
43971 element is zero and inter-unit moves are OK, we use movq
43973 use_vector_set = (TARGET_64BIT && TARGET_SSE4_1
43974 && !(TARGET_INTER_UNIT_MOVES_TO_VEC
43980 use_vector_set = TARGET_SSE4_1;
43983 use_vector_set = TARGET_SSE2;
43986 use_vector_set = TARGET_SSE || TARGET_3DNOW_A;
43993 use_vector_set = TARGET_AVX;
43996 /* Use ix86_expand_vector_set in 64bit mode only. */
43997 use_vector_set = TARGET_AVX && TARGET_64BIT;
44003 if (use_vector_set)
44005 emit_insn (gen_rtx_SET (target, CONST0_RTX (mode)));
44006 var = force_reg (GET_MODE_INNER (mode), var);
44007 ix86_expand_vector_set (mmx_ok, target, var, one_var);
44023 var = force_reg (GET_MODE_INNER (mode), var);
44024 x = gen_rtx_VEC_CONCAT (mode, var, CONST0_RTX (GET_MODE_INNER (mode)));
44025 emit_insn (gen_rtx_SET (target, x));
44030 if (!REG_P (target) || REGNO (target) < FIRST_PSEUDO_REGISTER)
44031 new_target = gen_reg_rtx (mode);
44033 new_target = target;
44034 var = force_reg (GET_MODE_INNER (mode), var);
44035 x = gen_rtx_VEC_DUPLICATE (mode, var);
44036 x = gen_rtx_VEC_MERGE (mode, x, CONST0_RTX (mode), const1_rtx);
44037 emit_insn (gen_rtx_SET (new_target, x));
44040 /* We need to shuffle the value to the correct position, so
44041 create a new pseudo to store the intermediate result. */
44043 /* With SSE2, we can use the integer shuffle insns. */
44044 if (mode != V4SFmode && TARGET_SSE2)
44046 emit_insn (gen_sse2_pshufd_1 (new_target, new_target,
44048 GEN_INT (one_var == 1 ? 0 : 1),
44049 GEN_INT (one_var == 2 ? 0 : 1),
44050 GEN_INT (one_var == 3 ? 0 : 1)));
44051 if (target != new_target)
44052 emit_move_insn (target, new_target);
44056 /* Otherwise convert the intermediate result to V4SFmode and
44057 use the SSE1 shuffle instructions. */
44058 if (mode != V4SFmode)
44060 tmp = gen_reg_rtx (V4SFmode);
44061 emit_move_insn (tmp, gen_lowpart (V4SFmode, new_target));
44066 emit_insn (gen_sse_shufps_v4sf (tmp, tmp, tmp,
44068 GEN_INT (one_var == 1 ? 0 : 1),
44069 GEN_INT (one_var == 2 ? 0+4 : 1+4),
44070 GEN_INT (one_var == 3 ? 0+4 : 1+4)));
44072 if (mode != V4SFmode)
44073 emit_move_insn (target, gen_lowpart (V4SImode, tmp));
44074 else if (tmp != target)
44075 emit_move_insn (target, tmp);
44077 else if (target != new_target)
44078 emit_move_insn (target, new_target);
44083 vsimode = V4SImode;
44089 vsimode = V2SImode;
44095 /* Zero extend the variable element to SImode and recurse. */
44096 var = convert_modes (SImode, GET_MODE_INNER (mode), var, true);
44098 x = gen_reg_rtx (vsimode);
44099 if (!ix86_expand_vector_init_one_nonzero (mmx_ok, vsimode, x,
44101 gcc_unreachable ();
44103 emit_move_insn (target, gen_lowpart (mode, x));
44111 /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
44112 consisting of the values in VALS. It is known that all elements
44113 except ONE_VAR are constants. Return true if successful. */
44116 ix86_expand_vector_init_one_var (bool mmx_ok, machine_mode mode,
44117 rtx target, rtx vals, int one_var)
44119 rtx var = XVECEXP (vals, 0, one_var);
44120 machine_mode wmode;
44123 const_vec = copy_rtx (vals);
44124 XVECEXP (const_vec, 0, one_var) = CONST0_RTX (GET_MODE_INNER (mode));
44125 const_vec = gen_rtx_CONST_VECTOR (mode, XVEC (const_vec, 0));
44133 /* For the two element vectors, it's just as easy to use
44134 the general case. */
44138 /* Use ix86_expand_vector_set in 64bit mode only. */
44161 /* There's no way to set one QImode entry easily. Combine
44162 the variable value with its adjacent constant value, and
44163 promote to an HImode set. */
44164 x = XVECEXP (vals, 0, one_var ^ 1);
44167 var = convert_modes (HImode, QImode, var, true);
44168 var = expand_simple_binop (HImode, ASHIFT, var, GEN_INT (8),
44169 NULL_RTX, 1, OPTAB_LIB_WIDEN);
44170 x = GEN_INT (INTVAL (x) & 0xff);
44174 var = convert_modes (HImode, QImode, var, true);
44175 x = gen_int_mode (INTVAL (x) << 8, HImode);
44177 if (x != const0_rtx)
44178 var = expand_simple_binop (HImode, IOR, var, x, var,
44179 1, OPTAB_LIB_WIDEN);
44181 x = gen_reg_rtx (wmode);
44182 emit_move_insn (x, gen_lowpart (wmode, const_vec));
44183 ix86_expand_vector_set (mmx_ok, x, var, one_var >> 1);
44185 emit_move_insn (target, gen_lowpart (mode, x));
44192 emit_move_insn (target, const_vec);
44193 ix86_expand_vector_set (mmx_ok, target, var, one_var);
44197 /* A subroutine of ix86_expand_vector_init_general. Use vector
44198 concatenate to handle the most general case: all values variable,
44199 and none identical. */
44202 ix86_expand_vector_init_concat (machine_mode mode,
44203 rtx target, rtx *ops, int n)
44205 machine_mode cmode, hmode = VOIDmode, gmode = VOIDmode;
44206 rtx first[16], second[8], third[4];
44258 gcc_unreachable ();
44261 if (!register_operand (ops[1], cmode))
44262 ops[1] = force_reg (cmode, ops[1]);
44263 if (!register_operand (ops[0], cmode))
44264 ops[0] = force_reg (cmode, ops[0]);
44265 emit_insn (gen_rtx_SET (target, gen_rtx_VEC_CONCAT (mode, ops[0],
44285 gcc_unreachable ();
44309 gcc_unreachable ();
44327 gcc_unreachable ();
44332 /* FIXME: We process inputs backward to help RA. PR 36222. */
44335 for (; i > 0; i -= 2, j--)
44337 first[j] = gen_reg_rtx (cmode);
44338 v = gen_rtvec (2, ops[i - 1], ops[i]);
44339 ix86_expand_vector_init (false, first[j],
44340 gen_rtx_PARALLEL (cmode, v));
44346 gcc_assert (hmode != VOIDmode);
44347 gcc_assert (gmode != VOIDmode);
44348 for (i = j = 0; i < n; i += 2, j++)
44350 second[j] = gen_reg_rtx (hmode);
44351 ix86_expand_vector_init_concat (hmode, second [j],
44355 for (i = j = 0; i < n; i += 2, j++)
44357 third[j] = gen_reg_rtx (gmode);
44358 ix86_expand_vector_init_concat (gmode, third[j],
44362 ix86_expand_vector_init_concat (mode, target, third, n);
44366 gcc_assert (hmode != VOIDmode);
44367 for (i = j = 0; i < n; i += 2, j++)
44369 second[j] = gen_reg_rtx (hmode);
44370 ix86_expand_vector_init_concat (hmode, second [j],
44374 ix86_expand_vector_init_concat (mode, target, second, n);
44377 ix86_expand_vector_init_concat (mode, target, first, n);
44381 gcc_unreachable ();
44385 /* A subroutine of ix86_expand_vector_init_general. Use vector
44386 interleave to handle the most general case: all values variable,
44387 and none identical. */
44390 ix86_expand_vector_init_interleave (machine_mode mode,
44391 rtx target, rtx *ops, int n)
44393 machine_mode first_imode, second_imode, third_imode, inner_mode;
44396 rtx (*gen_load_even) (rtx, rtx, rtx);
44397 rtx (*gen_interleave_first_low) (rtx, rtx, rtx);
44398 rtx (*gen_interleave_second_low) (rtx, rtx, rtx);
44403 gen_load_even = gen_vec_setv8hi;
44404 gen_interleave_first_low = gen_vec_interleave_lowv4si;
44405 gen_interleave_second_low = gen_vec_interleave_lowv2di;
44406 inner_mode = HImode;
44407 first_imode = V4SImode;
44408 second_imode = V2DImode;
44409 third_imode = VOIDmode;
44412 gen_load_even = gen_vec_setv16qi;
44413 gen_interleave_first_low = gen_vec_interleave_lowv8hi;
44414 gen_interleave_second_low = gen_vec_interleave_lowv4si;
44415 inner_mode = QImode;
44416 first_imode = V8HImode;
44417 second_imode = V4SImode;
44418 third_imode = V2DImode;
44421 gcc_unreachable ();
44424 for (i = 0; i < n; i++)
44426 /* Extend the odd elment to SImode using a paradoxical SUBREG. */
44427 op0 = gen_reg_rtx (SImode);
44428 emit_move_insn (op0, gen_lowpart (SImode, ops [i + i]));
44430 /* Insert the SImode value as low element of V4SImode vector. */
44431 op1 = gen_reg_rtx (V4SImode);
44432 op0 = gen_rtx_VEC_MERGE (V4SImode,
44433 gen_rtx_VEC_DUPLICATE (V4SImode,
44435 CONST0_RTX (V4SImode),
44437 emit_insn (gen_rtx_SET (op1, op0));
44439 /* Cast the V4SImode vector back to a vector in orignal mode. */
44440 op0 = gen_reg_rtx (mode);
44441 emit_move_insn (op0, gen_lowpart (mode, op1));
44443 /* Load even elements into the second position. */
44444 emit_insn (gen_load_even (op0,
44445 force_reg (inner_mode,
44449 /* Cast vector to FIRST_IMODE vector. */
44450 ops[i] = gen_reg_rtx (first_imode);
44451 emit_move_insn (ops[i], gen_lowpart (first_imode, op0));
44454 /* Interleave low FIRST_IMODE vectors. */
44455 for (i = j = 0; i < n; i += 2, j++)
44457 op0 = gen_reg_rtx (first_imode);
44458 emit_insn (gen_interleave_first_low (op0, ops[i], ops[i + 1]));
44460 /* Cast FIRST_IMODE vector to SECOND_IMODE vector. */
44461 ops[j] = gen_reg_rtx (second_imode);
44462 emit_move_insn (ops[j], gen_lowpart (second_imode, op0));
44465 /* Interleave low SECOND_IMODE vectors. */
44466 switch (second_imode)
44469 for (i = j = 0; i < n / 2; i += 2, j++)
44471 op0 = gen_reg_rtx (second_imode);
44472 emit_insn (gen_interleave_second_low (op0, ops[i],
44475 /* Cast the SECOND_IMODE vector to the THIRD_IMODE
44477 ops[j] = gen_reg_rtx (third_imode);
44478 emit_move_insn (ops[j], gen_lowpart (third_imode, op0));
44480 second_imode = V2DImode;
44481 gen_interleave_second_low = gen_vec_interleave_lowv2di;
44485 op0 = gen_reg_rtx (second_imode);
44486 emit_insn (gen_interleave_second_low (op0, ops[0],
44489 /* Cast the SECOND_IMODE vector back to a vector on original
44491 emit_insn (gen_rtx_SET (target, gen_lowpart (mode, op0)));
44495 gcc_unreachable ();
44499 /* A subroutine of ix86_expand_vector_init. Handle the most general case:
44500 all values variable, and none identical. */
44503 ix86_expand_vector_init_general (bool mmx_ok, machine_mode mode,
44504 rtx target, rtx vals)
44506 rtx ops[64], op0, op1, op2, op3, op4, op5;
44507 machine_mode half_mode = VOIDmode;
44508 machine_mode quarter_mode = VOIDmode;
44515 if (!mmx_ok && !TARGET_SSE)
44531 n = GET_MODE_NUNITS (mode);
44532 for (i = 0; i < n; i++)
44533 ops[i] = XVECEXP (vals, 0, i);
44534 ix86_expand_vector_init_concat (mode, target, ops, n);
44538 half_mode = V16QImode;
44542 half_mode = V8HImode;
44546 n = GET_MODE_NUNITS (mode);
44547 for (i = 0; i < n; i++)
44548 ops[i] = XVECEXP (vals, 0, i);
44549 op0 = gen_reg_rtx (half_mode);
44550 op1 = gen_reg_rtx (half_mode);
44551 ix86_expand_vector_init_interleave (half_mode, op0, ops,
44553 ix86_expand_vector_init_interleave (half_mode, op1,
44554 &ops [n >> 1], n >> 2);
44555 emit_insn (gen_rtx_SET (target, gen_rtx_VEC_CONCAT (mode, op0, op1)));
44559 quarter_mode = V16QImode;
44560 half_mode = V32QImode;
44564 quarter_mode = V8HImode;
44565 half_mode = V16HImode;
44569 n = GET_MODE_NUNITS (mode);
44570 for (i = 0; i < n; i++)
44571 ops[i] = XVECEXP (vals, 0, i);
44572 op0 = gen_reg_rtx (quarter_mode);
44573 op1 = gen_reg_rtx (quarter_mode);
44574 op2 = gen_reg_rtx (quarter_mode);
44575 op3 = gen_reg_rtx (quarter_mode);
44576 op4 = gen_reg_rtx (half_mode);
44577 op5 = gen_reg_rtx (half_mode);
44578 ix86_expand_vector_init_interleave (quarter_mode, op0, ops,
44580 ix86_expand_vector_init_interleave (quarter_mode, op1,
44581 &ops [n >> 2], n >> 3);
44582 ix86_expand_vector_init_interleave (quarter_mode, op2,
44583 &ops [n >> 1], n >> 3);
44584 ix86_expand_vector_init_interleave (quarter_mode, op3,
44585 &ops [(n >> 1) | (n >> 2)], n >> 3);
44586 emit_insn (gen_rtx_SET (op4, gen_rtx_VEC_CONCAT (half_mode, op0, op1)));
44587 emit_insn (gen_rtx_SET (op5, gen_rtx_VEC_CONCAT (half_mode, op2, op3)));
44588 emit_insn (gen_rtx_SET (target, gen_rtx_VEC_CONCAT (mode, op4, op5)));
44592 if (!TARGET_SSE4_1)
44600 /* Don't use ix86_expand_vector_init_interleave if we can't
44601 move from GPR to SSE register directly. */
44602 if (!TARGET_INTER_UNIT_MOVES_TO_VEC)
44605 n = GET_MODE_NUNITS (mode);
44606 for (i = 0; i < n; i++)
44607 ops[i] = XVECEXP (vals, 0, i);
44608 ix86_expand_vector_init_interleave (mode, target, ops, n >> 1);
44616 gcc_unreachable ();
44620 int i, j, n_elts, n_words, n_elt_per_word;
44621 machine_mode inner_mode;
44622 rtx words[4], shift;
44624 inner_mode = GET_MODE_INNER (mode);
44625 n_elts = GET_MODE_NUNITS (mode);
44626 n_words = GET_MODE_SIZE (mode) / UNITS_PER_WORD;
44627 n_elt_per_word = n_elts / n_words;
44628 shift = GEN_INT (GET_MODE_BITSIZE (inner_mode));
44630 for (i = 0; i < n_words; ++i)
44632 rtx word = NULL_RTX;
44634 for (j = 0; j < n_elt_per_word; ++j)
44636 rtx elt = XVECEXP (vals, 0, (i+1)*n_elt_per_word - j - 1);
44637 elt = convert_modes (word_mode, inner_mode, elt, true);
44643 word = expand_simple_binop (word_mode, ASHIFT, word, shift,
44644 word, 1, OPTAB_LIB_WIDEN);
44645 word = expand_simple_binop (word_mode, IOR, word, elt,
44646 word, 1, OPTAB_LIB_WIDEN);
44654 emit_move_insn (target, gen_lowpart (mode, words[0]));
44655 else if (n_words == 2)
44657 rtx tmp = gen_reg_rtx (mode);
44658 emit_clobber (tmp);
44659 emit_move_insn (gen_lowpart (word_mode, tmp), words[0]);
44660 emit_move_insn (gen_highpart (word_mode, tmp), words[1]);
44661 emit_move_insn (target, tmp);
44663 else if (n_words == 4)
44665 rtx tmp = gen_reg_rtx (V4SImode);
44666 gcc_assert (word_mode == SImode);
44667 vals = gen_rtx_PARALLEL (V4SImode, gen_rtvec_v (4, words));
44668 ix86_expand_vector_init_general (false, V4SImode, tmp, vals);
44669 emit_move_insn (target, gen_lowpart (mode, tmp));
44672 gcc_unreachable ();
44676 /* Initialize vector TARGET via VALS. Suppress the use of MMX
44677 instructions unless MMX_OK is true. */
44680 ix86_expand_vector_init (bool mmx_ok, rtx target, rtx vals)
44682 machine_mode mode = GET_MODE (target);
44683 machine_mode inner_mode = GET_MODE_INNER (mode);
44684 int n_elts = GET_MODE_NUNITS (mode);
44685 int n_var = 0, one_var = -1;
44686 bool all_same = true, all_const_zero = true;
44690 for (i = 0; i < n_elts; ++i)
44692 x = XVECEXP (vals, 0, i);
44693 if (!(CONST_SCALAR_INT_P (x)
44694 || CONST_DOUBLE_P (x)
44695 || CONST_FIXED_P (x)))
44696 n_var++, one_var = i;
44697 else if (x != CONST0_RTX (inner_mode))
44698 all_const_zero = false;
44699 if (i > 0 && !rtx_equal_p (x, XVECEXP (vals, 0, 0)))
44703 /* Constants are best loaded from the constant pool. */
44706 emit_move_insn (target, gen_rtx_CONST_VECTOR (mode, XVEC (vals, 0)));
44710 /* If all values are identical, broadcast the value. */
44712 && ix86_expand_vector_init_duplicate (mmx_ok, mode, target,
44713 XVECEXP (vals, 0, 0)))
44716 /* Values where only one field is non-constant are best loaded from
44717 the pool and overwritten via move later. */
44721 && ix86_expand_vector_init_one_nonzero (mmx_ok, mode, target,
44722 XVECEXP (vals, 0, one_var),
44726 if (ix86_expand_vector_init_one_var (mmx_ok, mode, target, vals, one_var))
44730 ix86_expand_vector_init_general (mmx_ok, mode, target, vals);
44734 ix86_expand_vector_set (bool mmx_ok, rtx target, rtx val, int elt)
44736 machine_mode mode = GET_MODE (target);
44737 machine_mode inner_mode = GET_MODE_INNER (mode);
44738 machine_mode half_mode;
44739 bool use_vec_merge = false;
44741 static rtx (*gen_extract[6][2]) (rtx, rtx)
44743 { gen_vec_extract_lo_v32qi, gen_vec_extract_hi_v32qi },
44744 { gen_vec_extract_lo_v16hi, gen_vec_extract_hi_v16hi },
44745 { gen_vec_extract_lo_v8si, gen_vec_extract_hi_v8si },
44746 { gen_vec_extract_lo_v4di, gen_vec_extract_hi_v4di },
44747 { gen_vec_extract_lo_v8sf, gen_vec_extract_hi_v8sf },
44748 { gen_vec_extract_lo_v4df, gen_vec_extract_hi_v4df }
44750 static rtx (*gen_insert[6][2]) (rtx, rtx, rtx)
44752 { gen_vec_set_lo_v32qi, gen_vec_set_hi_v32qi },
44753 { gen_vec_set_lo_v16hi, gen_vec_set_hi_v16hi },
44754 { gen_vec_set_lo_v8si, gen_vec_set_hi_v8si },
44755 { gen_vec_set_lo_v4di, gen_vec_set_hi_v4di },
44756 { gen_vec_set_lo_v8sf, gen_vec_set_hi_v8sf },
44757 { gen_vec_set_lo_v4df, gen_vec_set_hi_v4df }
44760 machine_mode mmode = VOIDmode;
44761 rtx (*gen_blendm) (rtx, rtx, rtx, rtx);
44769 tmp = gen_reg_rtx (GET_MODE_INNER (mode));
44770 ix86_expand_vector_extract (true, tmp, target, 1 - elt);
44772 tmp = gen_rtx_VEC_CONCAT (mode, val, tmp);
44774 tmp = gen_rtx_VEC_CONCAT (mode, tmp, val);
44775 emit_insn (gen_rtx_SET (target, tmp));
44781 use_vec_merge = TARGET_SSE4_1 && TARGET_64BIT;
44785 tmp = gen_reg_rtx (GET_MODE_INNER (mode));
44786 ix86_expand_vector_extract (false, tmp, target, 1 - elt);
44788 tmp = gen_rtx_VEC_CONCAT (mode, val, tmp);
44790 tmp = gen_rtx_VEC_CONCAT (mode, tmp, val);
44791 emit_insn (gen_rtx_SET (target, tmp));
44798 /* For the two element vectors, we implement a VEC_CONCAT with
44799 the extraction of the other element. */
44801 tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, GEN_INT (1 - elt)));
44802 tmp = gen_rtx_VEC_SELECT (inner_mode, target, tmp);
44805 op0 = val, op1 = tmp;
44807 op0 = tmp, op1 = val;
44809 tmp = gen_rtx_VEC_CONCAT (mode, op0, op1);
44810 emit_insn (gen_rtx_SET (target, tmp));
44815 use_vec_merge = TARGET_SSE4_1;
44822 use_vec_merge = true;
44826 /* tmp = target = A B C D */
44827 tmp = copy_to_reg (target);
44828 /* target = A A B B */
44829 emit_insn (gen_vec_interleave_lowv4sf (target, target, target));
44830 /* target = X A B B */
44831 ix86_expand_vector_set (false, target, val, 0);
44832 /* target = A X C D */
44833 emit_insn (gen_sse_shufps_v4sf (target, target, tmp,
44834 const1_rtx, const0_rtx,
44835 GEN_INT (2+4), GEN_INT (3+4)));
44839 /* tmp = target = A B C D */
44840 tmp = copy_to_reg (target);
44841 /* tmp = X B C D */
44842 ix86_expand_vector_set (false, tmp, val, 0);
44843 /* target = A B X D */
44844 emit_insn (gen_sse_shufps_v4sf (target, target, tmp,
44845 const0_rtx, const1_rtx,
44846 GEN_INT (0+4), GEN_INT (3+4)));
44850 /* tmp = target = A B C D */
44851 tmp = copy_to_reg (target);
44852 /* tmp = X B C D */
44853 ix86_expand_vector_set (false, tmp, val, 0);
44854 /* target = A B X D */
44855 emit_insn (gen_sse_shufps_v4sf (target, target, tmp,
44856 const0_rtx, const1_rtx,
44857 GEN_INT (2+4), GEN_INT (0+4)));
44861 gcc_unreachable ();
44866 use_vec_merge = TARGET_SSE4_1;
44870 /* Element 0 handled by vec_merge below. */
44873 use_vec_merge = true;
44879 /* With SSE2, use integer shuffles to swap element 0 and ELT,
44880 store into element 0, then shuffle them back. */
44884 order[0] = GEN_INT (elt);
44885 order[1] = const1_rtx;
44886 order[2] = const2_rtx;
44887 order[3] = GEN_INT (3);
44888 order[elt] = const0_rtx;
44890 emit_insn (gen_sse2_pshufd_1 (target, target, order[0],
44891 order[1], order[2], order[3]));
44893 ix86_expand_vector_set (false, target, val, 0);
44895 emit_insn (gen_sse2_pshufd_1 (target, target, order[0],
44896 order[1], order[2], order[3]));
44900 /* For SSE1, we have to reuse the V4SF code. */
44901 rtx t = gen_reg_rtx (V4SFmode);
44902 ix86_expand_vector_set (false, t, gen_lowpart (SFmode, val), elt);
44903 emit_move_insn (target, gen_lowpart (mode, t));
44908 use_vec_merge = TARGET_SSE2;
44911 use_vec_merge = mmx_ok && (TARGET_SSE || TARGET_3DNOW_A);
44915 use_vec_merge = TARGET_SSE4_1;
44922 half_mode = V16QImode;
44928 half_mode = V8HImode;
44934 half_mode = V4SImode;
44940 half_mode = V2DImode;
44946 half_mode = V4SFmode;
44952 half_mode = V2DFmode;
44958 /* Compute offset. */
44962 gcc_assert (i <= 1);
44964 /* Extract the half. */
44965 tmp = gen_reg_rtx (half_mode);
44966 emit_insn (gen_extract[j][i] (tmp, target));
44968 /* Put val in tmp at elt. */
44969 ix86_expand_vector_set (false, tmp, val, elt);
44972 emit_insn (gen_insert[j][i] (target, target, tmp));
44976 if (TARGET_AVX512F)
44979 gen_blendm = gen_avx512f_blendmv8df;
44984 if (TARGET_AVX512F)
44987 gen_blendm = gen_avx512f_blendmv8di;
44992 if (TARGET_AVX512F)
44995 gen_blendm = gen_avx512f_blendmv16sf;
45000 if (TARGET_AVX512F)
45003 gen_blendm = gen_avx512f_blendmv16si;
45008 if (TARGET_AVX512F && TARGET_AVX512BW)
45011 gen_blendm = gen_avx512bw_blendmv32hi;
45016 if (TARGET_AVX512F && TARGET_AVX512BW)
45019 gen_blendm = gen_avx512bw_blendmv64qi;
45027 if (mmode != VOIDmode)
45029 tmp = gen_reg_rtx (mode);
45030 emit_insn (gen_rtx_SET (tmp, gen_rtx_VEC_DUPLICATE (mode, val)));
45031 emit_insn (gen_blendm (target, tmp, target,
45033 gen_int_mode (1 << elt, mmode))));
45035 else if (use_vec_merge)
45037 tmp = gen_rtx_VEC_DUPLICATE (mode, val);
45038 tmp = gen_rtx_VEC_MERGE (mode, tmp, target, GEN_INT (1 << elt));
45039 emit_insn (gen_rtx_SET (target, tmp));
45043 rtx mem = assign_stack_temp (mode, GET_MODE_SIZE (mode));
45045 emit_move_insn (mem, target);
45047 tmp = adjust_address (mem, inner_mode, elt*GET_MODE_SIZE (inner_mode));
45048 emit_move_insn (tmp, val);
45050 emit_move_insn (target, mem);
45055 ix86_expand_vector_extract (bool mmx_ok, rtx target, rtx vec, int elt)
45057 machine_mode mode = GET_MODE (vec);
45058 machine_mode inner_mode = GET_MODE_INNER (mode);
45059 bool use_vec_extr = false;
45072 use_vec_extr = true;
45076 use_vec_extr = TARGET_SSE4_1;
45088 tmp = gen_reg_rtx (mode);
45089 emit_insn (gen_sse_shufps_v4sf (tmp, vec, vec,
45090 GEN_INT (elt), GEN_INT (elt),
45091 GEN_INT (elt+4), GEN_INT (elt+4)));
45095 tmp = gen_reg_rtx (mode);
45096 emit_insn (gen_vec_interleave_highv4sf (tmp, vec, vec));
45100 gcc_unreachable ();
45103 use_vec_extr = true;
45108 use_vec_extr = TARGET_SSE4_1;
45122 tmp = gen_reg_rtx (mode);
45123 emit_insn (gen_sse2_pshufd_1 (tmp, vec,
45124 GEN_INT (elt), GEN_INT (elt),
45125 GEN_INT (elt), GEN_INT (elt)));
45129 tmp = gen_reg_rtx (mode);
45130 emit_insn (gen_vec_interleave_highv4si (tmp, vec, vec));
45134 gcc_unreachable ();
45137 use_vec_extr = true;
45142 /* For SSE1, we have to reuse the V4SF code. */
45143 ix86_expand_vector_extract (false, gen_lowpart (SFmode, target),
45144 gen_lowpart (V4SFmode, vec), elt);
45150 use_vec_extr = TARGET_SSE2;
45153 use_vec_extr = mmx_ok && (TARGET_SSE || TARGET_3DNOW_A);
45157 use_vec_extr = TARGET_SSE4_1;
45163 tmp = gen_reg_rtx (V4SFmode);
45165 emit_insn (gen_vec_extract_lo_v8sf (tmp, vec));
45167 emit_insn (gen_vec_extract_hi_v8sf (tmp, vec));
45168 ix86_expand_vector_extract (false, target, tmp, elt & 3);
45176 tmp = gen_reg_rtx (V2DFmode);
45178 emit_insn (gen_vec_extract_lo_v4df (tmp, vec));
45180 emit_insn (gen_vec_extract_hi_v4df (tmp, vec));
45181 ix86_expand_vector_extract (false, target, tmp, elt & 1);
45189 tmp = gen_reg_rtx (V16QImode);
45191 emit_insn (gen_vec_extract_lo_v32qi (tmp, vec));
45193 emit_insn (gen_vec_extract_hi_v32qi (tmp, vec));
45194 ix86_expand_vector_extract (false, target, tmp, elt & 15);
45202 tmp = gen_reg_rtx (V8HImode);
45204 emit_insn (gen_vec_extract_lo_v16hi (tmp, vec));
45206 emit_insn (gen_vec_extract_hi_v16hi (tmp, vec));
45207 ix86_expand_vector_extract (false, target, tmp, elt & 7);
45215 tmp = gen_reg_rtx (V4SImode);
45217 emit_insn (gen_vec_extract_lo_v8si (tmp, vec));
45219 emit_insn (gen_vec_extract_hi_v8si (tmp, vec));
45220 ix86_expand_vector_extract (false, target, tmp, elt & 3);
45228 tmp = gen_reg_rtx (V2DImode);
45230 emit_insn (gen_vec_extract_lo_v4di (tmp, vec));
45232 emit_insn (gen_vec_extract_hi_v4di (tmp, vec));
45233 ix86_expand_vector_extract (false, target, tmp, elt & 1);
45239 if (TARGET_AVX512BW)
45241 tmp = gen_reg_rtx (V16HImode);
45243 emit_insn (gen_vec_extract_lo_v32hi (tmp, vec));
45245 emit_insn (gen_vec_extract_hi_v32hi (tmp, vec));
45246 ix86_expand_vector_extract (false, target, tmp, elt & 15);
45252 if (TARGET_AVX512BW)
45254 tmp = gen_reg_rtx (V32QImode);
45256 emit_insn (gen_vec_extract_lo_v64qi (tmp, vec));
45258 emit_insn (gen_vec_extract_hi_v64qi (tmp, vec));
45259 ix86_expand_vector_extract (false, target, tmp, elt & 31);
45265 tmp = gen_reg_rtx (V8SFmode);
45267 emit_insn (gen_vec_extract_lo_v16sf (tmp, vec));
45269 emit_insn (gen_vec_extract_hi_v16sf (tmp, vec));
45270 ix86_expand_vector_extract (false, target, tmp, elt & 7);
45274 tmp = gen_reg_rtx (V4DFmode);
45276 emit_insn (gen_vec_extract_lo_v8df (tmp, vec));
45278 emit_insn (gen_vec_extract_hi_v8df (tmp, vec));
45279 ix86_expand_vector_extract (false, target, tmp, elt & 3);
45283 tmp = gen_reg_rtx (V8SImode);
45285 emit_insn (gen_vec_extract_lo_v16si (tmp, vec));
45287 emit_insn (gen_vec_extract_hi_v16si (tmp, vec));
45288 ix86_expand_vector_extract (false, target, tmp, elt & 7);
45292 tmp = gen_reg_rtx (V4DImode);
45294 emit_insn (gen_vec_extract_lo_v8di (tmp, vec));
45296 emit_insn (gen_vec_extract_hi_v8di (tmp, vec));
45297 ix86_expand_vector_extract (false, target, tmp, elt & 3);
45301 /* ??? Could extract the appropriate HImode element and shift. */
45308 tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, GEN_INT (elt)));
45309 tmp = gen_rtx_VEC_SELECT (inner_mode, vec, tmp);
45311 /* Let the rtl optimizers know about the zero extension performed. */
45312 if (inner_mode == QImode || inner_mode == HImode)
45314 tmp = gen_rtx_ZERO_EXTEND (SImode, tmp);
45315 target = gen_lowpart (SImode, target);
45318 emit_insn (gen_rtx_SET (target, tmp));
45322 rtx mem = assign_stack_temp (mode, GET_MODE_SIZE (mode));
45324 emit_move_insn (mem, vec);
45326 tmp = adjust_address (mem, inner_mode, elt*GET_MODE_SIZE (inner_mode));
45327 emit_move_insn (target, tmp);
45331 /* Generate code to copy vector bits i / 2 ... i - 1 from vector SRC
45332 to bits 0 ... i / 2 - 1 of vector DEST, which has the same mode.
45333 The upper bits of DEST are undefined, though they shouldn't cause
45334 exceptions (some bits from src or all zeros are ok). */
45337 emit_reduc_half (rtx dest, rtx src, int i)
45340 switch (GET_MODE (src))
45344 tem = gen_sse_movhlps (dest, src, src);
45346 tem = gen_sse_shufps_v4sf (dest, src, src, const1_rtx, const1_rtx,
45347 GEN_INT (1 + 4), GEN_INT (1 + 4));
45350 tem = gen_vec_interleave_highv2df (dest, src, src);
45356 d = gen_reg_rtx (V1TImode);
45357 tem = gen_sse2_lshrv1ti3 (d, gen_lowpart (V1TImode, src),
45362 tem = gen_avx_vperm2f128v8sf3 (dest, src, src, const1_rtx);
45364 tem = gen_avx_shufps256 (dest, src, src,
45365 GEN_INT (i == 128 ? 2 + (3 << 2) : 1));
45369 tem = gen_avx_vperm2f128v4df3 (dest, src, src, const1_rtx);
45371 tem = gen_avx_shufpd256 (dest, src, src, const1_rtx);
45379 if (GET_MODE (dest) != V4DImode)
45380 d = gen_reg_rtx (V4DImode);
45381 tem = gen_avx2_permv2ti (d, gen_lowpart (V4DImode, src),
45382 gen_lowpart (V4DImode, src),
45387 d = gen_reg_rtx (V2TImode);
45388 tem = gen_avx2_lshrv2ti3 (d, gen_lowpart (V2TImode, src),
45399 tem = gen_avx512f_shuf_i32x4_1 (gen_lowpart (V16SImode, dest),
45400 gen_lowpart (V16SImode, src),
45401 gen_lowpart (V16SImode, src),
45402 GEN_INT (0x4 + (i == 512 ? 4 : 0)),
45403 GEN_INT (0x5 + (i == 512 ? 4 : 0)),
45404 GEN_INT (0x6 + (i == 512 ? 4 : 0)),
45405 GEN_INT (0x7 + (i == 512 ? 4 : 0)),
45406 GEN_INT (0xC), GEN_INT (0xD),
45407 GEN_INT (0xE), GEN_INT (0xF),
45408 GEN_INT (0x10), GEN_INT (0x11),
45409 GEN_INT (0x12), GEN_INT (0x13),
45410 GEN_INT (0x14), GEN_INT (0x15),
45411 GEN_INT (0x16), GEN_INT (0x17));
45413 tem = gen_avx512f_pshufd_1 (gen_lowpart (V16SImode, dest),
45414 gen_lowpart (V16SImode, src),
45415 GEN_INT (i == 128 ? 0x2 : 0x1),
45419 GEN_INT (i == 128 ? 0x6 : 0x5),
45423 GEN_INT (i == 128 ? 0xA : 0x9),
45427 GEN_INT (i == 128 ? 0xE : 0xD),
45433 gcc_unreachable ();
45437 emit_move_insn (dest, gen_lowpart (GET_MODE (dest), d));
45440 /* Expand a vector reduction. FN is the binary pattern to reduce;
45441 DEST is the destination; IN is the input vector. */
45444 ix86_expand_reduc (rtx (*fn) (rtx, rtx, rtx), rtx dest, rtx in)
45446 rtx half, dst, vec = in;
45447 machine_mode mode = GET_MODE (in);
45450 /* SSE4 has a special instruction for V8HImode UMIN reduction. */
45452 && mode == V8HImode
45453 && fn == gen_uminv8hi3)
45455 emit_insn (gen_sse4_1_phminposuw (dest, in));
45459 for (i = GET_MODE_BITSIZE (mode);
45460 i > GET_MODE_BITSIZE (GET_MODE_INNER (mode));
45463 half = gen_reg_rtx (mode);
45464 emit_reduc_half (half, vec, i);
45465 if (i == GET_MODE_BITSIZE (GET_MODE_INNER (mode)) * 2)
45468 dst = gen_reg_rtx (mode);
45469 emit_insn (fn (dst, half, vec));
45474 /* Target hook for scalar_mode_supported_p. */
45476 ix86_scalar_mode_supported_p (machine_mode mode)
45478 if (DECIMAL_FLOAT_MODE_P (mode))
45479 return default_decimal_float_supported_p ();
45480 else if (mode == TFmode)
45483 return default_scalar_mode_supported_p (mode);
45486 /* Implements target hook vector_mode_supported_p. */
45488 ix86_vector_mode_supported_p (machine_mode mode)
45490 if (TARGET_SSE && VALID_SSE_REG_MODE (mode))
45492 if (TARGET_SSE2 && VALID_SSE2_REG_MODE (mode))
45494 if (TARGET_AVX && VALID_AVX256_REG_MODE (mode))
45496 if (TARGET_AVX512F && VALID_AVX512F_REG_MODE (mode))
45498 if (TARGET_MMX && VALID_MMX_REG_MODE (mode))
45500 if (TARGET_3DNOW && VALID_MMX_REG_MODE_3DNOW (mode))
45505 /* Implement target hook libgcc_floating_mode_supported_p. */
45507 ix86_libgcc_floating_mode_supported_p (machine_mode mode)
45517 #ifdef IX86_NO_LIBGCC_TFMODE
45519 #elif defined IX86_MAYBE_NO_LIBGCC_TFMODE
45520 return TARGET_LONG_DOUBLE_128;
45530 /* Target hook for c_mode_for_suffix. */
45531 static machine_mode
45532 ix86_c_mode_for_suffix (char suffix)
45542 /* Worker function for TARGET_MD_ASM_ADJUST.
45544 We do this in the new i386 backend to maintain source compatibility
45545 with the old cc0-based compiler. */
45548 ix86_md_asm_adjust (vec<rtx> &/*outputs*/, vec<rtx> &/*inputs*/,
45549 vec<const char *> &/*constraints*/,
45550 vec<rtx> &clobbers, HARD_REG_SET &clobbered_regs)
45552 clobbers.safe_push (gen_rtx_REG (CCmode, FLAGS_REG));
45553 clobbers.safe_push (gen_rtx_REG (CCFPmode, FPSR_REG));
45555 SET_HARD_REG_BIT (clobbered_regs, FLAGS_REG);
45556 SET_HARD_REG_BIT (clobbered_regs, FPSR_REG);
45561 /* Implements target vector targetm.asm.encode_section_info. */
45563 static void ATTRIBUTE_UNUSED
45564 ix86_encode_section_info (tree decl, rtx rtl, int first)
45566 default_encode_section_info (decl, rtl, first);
45568 if (ix86_in_large_data_p (decl))
45569 SYMBOL_REF_FLAGS (XEXP (rtl, 0)) |= SYMBOL_FLAG_FAR_ADDR;
45572 /* Worker function for REVERSE_CONDITION. */
45575 ix86_reverse_condition (enum rtx_code code, machine_mode mode)
45577 return (mode != CCFPmode && mode != CCFPUmode
45578 ? reverse_condition (code)
45579 : reverse_condition_maybe_unordered (code));
45582 /* Output code to perform an x87 FP register move, from OPERANDS[1]
45586 output_387_reg_move (rtx insn, rtx *operands)
45588 if (REG_P (operands[0]))
45590 if (REG_P (operands[1])
45591 && find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
45593 if (REGNO (operands[0]) == FIRST_STACK_REG)
45594 return output_387_ffreep (operands, 0);
45595 return "fstp\t%y0";
45597 if (STACK_TOP_P (operands[0]))
45598 return "fld%Z1\t%y1";
45601 else if (MEM_P (operands[0]))
45603 gcc_assert (REG_P (operands[1]));
45604 if (find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
45605 return "fstp%Z0\t%y0";
45608 /* There is no non-popping store to memory for XFmode.
45609 So if we need one, follow the store with a load. */
45610 if (GET_MODE (operands[0]) == XFmode)
45611 return "fstp%Z0\t%y0\n\tfld%Z0\t%y0";
45613 return "fst%Z0\t%y0";
45620 /* Output code to perform a conditional jump to LABEL, if C2 flag in
45621 FP status register is set. */
45624 ix86_emit_fp_unordered_jump (rtx label)
45626 rtx reg = gen_reg_rtx (HImode);
45629 emit_insn (gen_x86_fnstsw_1 (reg));
45631 if (TARGET_SAHF && (TARGET_USE_SAHF || optimize_insn_for_size_p ()))
45633 emit_insn (gen_x86_sahf_1 (reg));
45635 temp = gen_rtx_REG (CCmode, FLAGS_REG);
45636 temp = gen_rtx_UNORDERED (VOIDmode, temp, const0_rtx);
45640 emit_insn (gen_testqi_ext_ccno_0 (reg, GEN_INT (0x04)));
45642 temp = gen_rtx_REG (CCNOmode, FLAGS_REG);
45643 temp = gen_rtx_NE (VOIDmode, temp, const0_rtx);
45646 temp = gen_rtx_IF_THEN_ELSE (VOIDmode, temp,
45647 gen_rtx_LABEL_REF (VOIDmode, label),
45649 temp = gen_rtx_SET (pc_rtx, temp);
45651 emit_jump_insn (temp);
45652 predict_jump (REG_BR_PROB_BASE * 10 / 100);
45655 /* Output code to perform a log1p XFmode calculation. */
45657 void ix86_emit_i387_log1p (rtx op0, rtx op1)
45659 rtx_code_label *label1 = gen_label_rtx ();
45660 rtx_code_label *label2 = gen_label_rtx ();
45662 rtx tmp = gen_reg_rtx (XFmode);
45663 rtx tmp2 = gen_reg_rtx (XFmode);
45666 emit_insn (gen_absxf2 (tmp, op1));
45667 test = gen_rtx_GE (VOIDmode, tmp,
45668 CONST_DOUBLE_FROM_REAL_VALUE (
45669 REAL_VALUE_ATOF ("0.29289321881345247561810596348408353", XFmode),
45671 emit_jump_insn (gen_cbranchxf4 (test, XEXP (test, 0), XEXP (test, 1), label1));
45673 emit_move_insn (tmp2, standard_80387_constant_rtx (4)); /* fldln2 */
45674 emit_insn (gen_fyl2xp1xf3_i387 (op0, op1, tmp2));
45675 emit_jump (label2);
45677 emit_label (label1);
45678 emit_move_insn (tmp, CONST1_RTX (XFmode));
45679 emit_insn (gen_addxf3 (tmp, op1, tmp));
45680 emit_move_insn (tmp2, standard_80387_constant_rtx (4)); /* fldln2 */
45681 emit_insn (gen_fyl2xxf3_i387 (op0, tmp, tmp2));
45683 emit_label (label2);
45686 /* Emit code for round calculation. */
45687 void ix86_emit_i387_round (rtx op0, rtx op1)
45689 machine_mode inmode = GET_MODE (op1);
45690 machine_mode outmode = GET_MODE (op0);
45691 rtx e1, e2, res, tmp, tmp1, half;
45692 rtx scratch = gen_reg_rtx (HImode);
45693 rtx flags = gen_rtx_REG (CCNOmode, FLAGS_REG);
45694 rtx_code_label *jump_label = gen_label_rtx ();
45696 rtx (*gen_abs) (rtx, rtx);
45697 rtx (*gen_neg) (rtx, rtx);
45702 gen_abs = gen_abssf2;
45705 gen_abs = gen_absdf2;
45708 gen_abs = gen_absxf2;
45711 gcc_unreachable ();
45717 gen_neg = gen_negsf2;
45720 gen_neg = gen_negdf2;
45723 gen_neg = gen_negxf2;
45726 gen_neg = gen_neghi2;
45729 gen_neg = gen_negsi2;
45732 gen_neg = gen_negdi2;
45735 gcc_unreachable ();
45738 e1 = gen_reg_rtx (inmode);
45739 e2 = gen_reg_rtx (inmode);
45740 res = gen_reg_rtx (outmode);
45742 half = CONST_DOUBLE_FROM_REAL_VALUE (dconsthalf, inmode);
45744 /* round(a) = sgn(a) * floor(fabs(a) + 0.5) */
45746 /* scratch = fxam(op1) */
45747 emit_insn (gen_rtx_SET (scratch,
45748 gen_rtx_UNSPEC (HImode, gen_rtvec (1, op1),
45750 /* e1 = fabs(op1) */
45751 emit_insn (gen_abs (e1, op1));
45753 /* e2 = e1 + 0.5 */
45754 half = force_reg (inmode, half);
45755 emit_insn (gen_rtx_SET (e2, gen_rtx_PLUS (inmode, e1, half)));
45757 /* res = floor(e2) */
45758 if (inmode != XFmode)
45760 tmp1 = gen_reg_rtx (XFmode);
45762 emit_insn (gen_rtx_SET (tmp1, gen_rtx_FLOAT_EXTEND (XFmode, e2)));
45772 rtx tmp0 = gen_reg_rtx (XFmode);
45774 emit_insn (gen_frndintxf2_floor (tmp0, tmp1));
45776 emit_insn (gen_rtx_SET (res,
45777 gen_rtx_UNSPEC (outmode, gen_rtvec (1, tmp0),
45778 UNSPEC_TRUNC_NOOP)));
45782 emit_insn (gen_frndintxf2_floor (res, tmp1));
45785 emit_insn (gen_lfloorxfhi2 (res, tmp1));
45788 emit_insn (gen_lfloorxfsi2 (res, tmp1));
45791 emit_insn (gen_lfloorxfdi2 (res, tmp1));
45794 gcc_unreachable ();
45797 /* flags = signbit(a) */
45798 emit_insn (gen_testqi_ext_ccno_0 (scratch, GEN_INT (0x02)));
45800 /* if (flags) then res = -res */
45801 tmp = gen_rtx_IF_THEN_ELSE (VOIDmode,
45802 gen_rtx_EQ (VOIDmode, flags, const0_rtx),
45803 gen_rtx_LABEL_REF (VOIDmode, jump_label),
45805 insn = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
45806 predict_jump (REG_BR_PROB_BASE * 50 / 100);
45807 JUMP_LABEL (insn) = jump_label;
45809 emit_insn (gen_neg (res, res));
45811 emit_label (jump_label);
45812 LABEL_NUSES (jump_label) = 1;
45814 emit_move_insn (op0, res);
45817 /* Output code to perform a Newton-Rhapson approximation of a single precision
45818 floating point divide [http://en.wikipedia.org/wiki/N-th_root_algorithm]. */
45820 void ix86_emit_swdivsf (rtx res, rtx a, rtx b, machine_mode mode)
45822 rtx x0, x1, e0, e1;
45824 x0 = gen_reg_rtx (mode);
45825 e0 = gen_reg_rtx (mode);
45826 e1 = gen_reg_rtx (mode);
45827 x1 = gen_reg_rtx (mode);
45829 /* a / b = a * ((rcp(b) + rcp(b)) - (b * rcp(b) * rcp (b))) */
45831 b = force_reg (mode, b);
45833 /* x0 = rcp(b) estimate */
45834 if (mode == V16SFmode || mode == V8DFmode)
45835 emit_insn (gen_rtx_SET (x0, gen_rtx_UNSPEC (mode, gen_rtvec (1, b),
45838 emit_insn (gen_rtx_SET (x0, gen_rtx_UNSPEC (mode, gen_rtvec (1, b),
45842 emit_insn (gen_rtx_SET (e0, gen_rtx_MULT (mode, x0, b)));
45845 emit_insn (gen_rtx_SET (e0, gen_rtx_MULT (mode, x0, e0)));
45848 emit_insn (gen_rtx_SET (e1, gen_rtx_PLUS (mode, x0, x0)));
45851 emit_insn (gen_rtx_SET (x1, gen_rtx_MINUS (mode, e1, e0)));
45854 emit_insn (gen_rtx_SET (res, gen_rtx_MULT (mode, a, x1)));
45857 /* Output code to perform a Newton-Rhapson approximation of a
45858 single precision floating point [reciprocal] square root. */
45860 void ix86_emit_swsqrtsf (rtx res, rtx a, machine_mode mode,
45863 rtx x0, e0, e1, e2, e3, mthree, mhalf;
45867 x0 = gen_reg_rtx (mode);
45868 e0 = gen_reg_rtx (mode);
45869 e1 = gen_reg_rtx (mode);
45870 e2 = gen_reg_rtx (mode);
45871 e3 = gen_reg_rtx (mode);
45873 real_from_integer (&r, VOIDmode, -3, SIGNED);
45874 mthree = CONST_DOUBLE_FROM_REAL_VALUE (r, SFmode);
45876 real_arithmetic (&r, NEGATE_EXPR, &dconsthalf, NULL);
45877 mhalf = CONST_DOUBLE_FROM_REAL_VALUE (r, SFmode);
45878 unspec = UNSPEC_RSQRT;
45880 if (VECTOR_MODE_P (mode))
45882 mthree = ix86_build_const_vector (mode, true, mthree);
45883 mhalf = ix86_build_const_vector (mode, true, mhalf);
45884 /* There is no 512-bit rsqrt. There is however rsqrt14. */
45885 if (GET_MODE_SIZE (mode) == 64)
45886 unspec = UNSPEC_RSQRT14;
45889 /* sqrt(a) = -0.5 * a * rsqrtss(a) * (a * rsqrtss(a) * rsqrtss(a) - 3.0)
45890 rsqrt(a) = -0.5 * rsqrtss(a) * (a * rsqrtss(a) * rsqrtss(a) - 3.0) */
45892 a = force_reg (mode, a);
45894 /* x0 = rsqrt(a) estimate */
45895 emit_insn (gen_rtx_SET (x0, gen_rtx_UNSPEC (mode, gen_rtvec (1, a),
45898 /* If (a == 0.0) Filter out infinity to prevent NaN for sqrt(0.0). */
45903 zero = gen_reg_rtx (mode);
45904 mask = gen_reg_rtx (mode);
45906 zero = force_reg (mode, CONST0_RTX(mode));
45908 /* Handle masked compare. */
45909 if (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 64)
45911 mask = gen_reg_rtx (HImode);
45912 /* Imm value 0x4 corresponds to not-equal comparison. */
45913 emit_insn (gen_avx512f_cmpv16sf3 (mask, zero, a, GEN_INT (0x4)));
45914 emit_insn (gen_avx512f_blendmv16sf (x0, zero, x0, mask));
45918 emit_insn (gen_rtx_SET (mask, gen_rtx_NE (mode, zero, a)));
45920 emit_insn (gen_rtx_SET (x0, gen_rtx_AND (mode, x0, mask)));
45925 emit_insn (gen_rtx_SET (e0, gen_rtx_MULT (mode, x0, a)));
45927 emit_insn (gen_rtx_SET (e1, gen_rtx_MULT (mode, e0, x0)));
45930 mthree = force_reg (mode, mthree);
45931 emit_insn (gen_rtx_SET (e2, gen_rtx_PLUS (mode, e1, mthree)));
45933 mhalf = force_reg (mode, mhalf);
45935 /* e3 = -.5 * x0 */
45936 emit_insn (gen_rtx_SET (e3, gen_rtx_MULT (mode, x0, mhalf)));
45938 /* e3 = -.5 * e0 */
45939 emit_insn (gen_rtx_SET (e3, gen_rtx_MULT (mode, e0, mhalf)));
45940 /* ret = e2 * e3 */
45941 emit_insn (gen_rtx_SET (res, gen_rtx_MULT (mode, e2, e3)));
45944 #ifdef TARGET_SOLARIS
45945 /* Solaris implementation of TARGET_ASM_NAMED_SECTION. */
45948 i386_solaris_elf_named_section (const char *name, unsigned int flags,
45951 /* With Binutils 2.15, the "@unwind" marker must be specified on
45952 every occurrence of the ".eh_frame" section, not just the first
45955 && strcmp (name, ".eh_frame") == 0)
45957 fprintf (asm_out_file, "\t.section\t%s,\"%s\",@unwind\n", name,
45958 flags & SECTION_WRITE ? "aw" : "a");
45963 if (HAVE_COMDAT_GROUP && flags & SECTION_LINKONCE)
45965 solaris_elf_asm_comdat_section (name, flags, decl);
45970 default_elf_asm_named_section (name, flags, decl);
45972 #endif /* TARGET_SOLARIS */
45974 /* Return the mangling of TYPE if it is an extended fundamental type. */
45976 static const char *
45977 ix86_mangle_type (const_tree type)
45979 type = TYPE_MAIN_VARIANT (type);
45981 if (TREE_CODE (type) != VOID_TYPE && TREE_CODE (type) != BOOLEAN_TYPE
45982 && TREE_CODE (type) != INTEGER_TYPE && TREE_CODE (type) != REAL_TYPE)
45985 switch (TYPE_MODE (type))
45988 /* __float128 is "g". */
45991 /* "long double" or __float80 is "e". */
45998 /* For 32-bit code we can save PIC register setup by using
45999 __stack_chk_fail_local hidden function instead of calling
46000 __stack_chk_fail directly. 64-bit code doesn't need to setup any PIC
46001 register, so it is better to call __stack_chk_fail directly. */
46003 static tree ATTRIBUTE_UNUSED
46004 ix86_stack_protect_fail (void)
46006 return TARGET_64BIT
46007 ? default_external_stack_protect_fail ()
46008 : default_hidden_stack_protect_fail ();
46011 /* Select a format to encode pointers in exception handling data. CODE
46012 is 0 for data, 1 for code labels, 2 for function pointers. GLOBAL is
46013 true if the symbol may be affected by dynamic relocations.
46015 ??? All x86 object file formats are capable of representing this.
46016 After all, the relocation needed is the same as for the call insn.
46017 Whether or not a particular assembler allows us to enter such, I
46018 guess we'll have to see. */
46020 asm_preferred_eh_data_format (int code, int global)
46024 int type = DW_EH_PE_sdata8;
46026 || ix86_cmodel == CM_SMALL_PIC
46027 || (ix86_cmodel == CM_MEDIUM_PIC && (global || code)))
46028 type = DW_EH_PE_sdata4;
46029 return (global ? DW_EH_PE_indirect : 0) | DW_EH_PE_pcrel | type;
46031 if (ix86_cmodel == CM_SMALL
46032 || (ix86_cmodel == CM_MEDIUM && code))
46033 return DW_EH_PE_udata4;
46034 return DW_EH_PE_absptr;
46037 /* Expand copysign from SIGN to the positive value ABS_VALUE
46038 storing in RESULT. If MASK is non-null, it shall be a mask to mask out
46041 ix86_sse_copysign_to_positive (rtx result, rtx abs_value, rtx sign, rtx mask)
46043 machine_mode mode = GET_MODE (sign);
46044 rtx sgn = gen_reg_rtx (mode);
46045 if (mask == NULL_RTX)
46047 machine_mode vmode;
46049 if (mode == SFmode)
46051 else if (mode == DFmode)
46056 mask = ix86_build_signbit_mask (vmode, VECTOR_MODE_P (mode), false);
46057 if (!VECTOR_MODE_P (mode))
46059 /* We need to generate a scalar mode mask in this case. */
46060 rtx tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, const0_rtx));
46061 tmp = gen_rtx_VEC_SELECT (mode, mask, tmp);
46062 mask = gen_reg_rtx (mode);
46063 emit_insn (gen_rtx_SET (mask, tmp));
46067 mask = gen_rtx_NOT (mode, mask);
46068 emit_insn (gen_rtx_SET (sgn, gen_rtx_AND (mode, mask, sign)));
46069 emit_insn (gen_rtx_SET (result, gen_rtx_IOR (mode, abs_value, sgn)));
46072 /* Expand fabs (OP0) and return a new rtx that holds the result. The
46073 mask for masking out the sign-bit is stored in *SMASK, if that is
46076 ix86_expand_sse_fabs (rtx op0, rtx *smask)
46078 machine_mode vmode, mode = GET_MODE (op0);
46081 xa = gen_reg_rtx (mode);
46082 if (mode == SFmode)
46084 else if (mode == DFmode)
46088 mask = ix86_build_signbit_mask (vmode, VECTOR_MODE_P (mode), true);
46089 if (!VECTOR_MODE_P (mode))
46091 /* We need to generate a scalar mode mask in this case. */
46092 rtx tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, const0_rtx));
46093 tmp = gen_rtx_VEC_SELECT (mode, mask, tmp);
46094 mask = gen_reg_rtx (mode);
46095 emit_insn (gen_rtx_SET (mask, tmp));
46097 emit_insn (gen_rtx_SET (xa, gen_rtx_AND (mode, op0, mask)));
46105 /* Expands a comparison of OP0 with OP1 using comparison code CODE,
46106 swapping the operands if SWAP_OPERANDS is true. The expanded
46107 code is a forward jump to a newly created label in case the
46108 comparison is true. The generated label rtx is returned. */
46109 static rtx_code_label *
46110 ix86_expand_sse_compare_and_jump (enum rtx_code code, rtx op0, rtx op1,
46111 bool swap_operands)
46113 machine_mode fpcmp_mode = ix86_fp_compare_mode (code);
46114 rtx_code_label *label;
46118 std::swap (op0, op1);
46120 label = gen_label_rtx ();
46121 tmp = gen_rtx_REG (fpcmp_mode, FLAGS_REG);
46122 emit_insn (gen_rtx_SET (tmp, gen_rtx_COMPARE (fpcmp_mode, op0, op1)));
46123 tmp = gen_rtx_fmt_ee (code, VOIDmode, tmp, const0_rtx);
46124 tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
46125 gen_rtx_LABEL_REF (VOIDmode, label), pc_rtx);
46126 tmp = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp));
46127 JUMP_LABEL (tmp) = label;
46132 /* Expand a mask generating SSE comparison instruction comparing OP0 with OP1
46133 using comparison code CODE. Operands are swapped for the comparison if
46134 SWAP_OPERANDS is true. Returns a rtx for the generated mask. */
46136 ix86_expand_sse_compare_mask (enum rtx_code code, rtx op0, rtx op1,
46137 bool swap_operands)
46139 rtx (*insn)(rtx, rtx, rtx, rtx);
46140 machine_mode mode = GET_MODE (op0);
46141 rtx mask = gen_reg_rtx (mode);
46144 std::swap (op0, op1);
46146 insn = mode == DFmode ? gen_setcc_df_sse : gen_setcc_sf_sse;
46148 emit_insn (insn (mask, op0, op1,
46149 gen_rtx_fmt_ee (code, mode, op0, op1)));
46153 /* Generate and return a rtx of mode MODE for 2**n where n is the number
46154 of bits of the mantissa of MODE, which must be one of DFmode or SFmode. */
46156 ix86_gen_TWO52 (machine_mode mode)
46158 REAL_VALUE_TYPE TWO52r;
46161 real_ldexp (&TWO52r, &dconst1, mode == DFmode ? 52 : 23);
46162 TWO52 = const_double_from_real_value (TWO52r, mode);
46163 TWO52 = force_reg (mode, TWO52);
46168 /* Expand SSE sequence for computing lround from OP1 storing
46171 ix86_expand_lround (rtx op0, rtx op1)
46173 /* C code for the stuff we're doing below:
46174 tmp = op1 + copysign (nextafter (0.5, 0.0), op1)
46177 machine_mode mode = GET_MODE (op1);
46178 const struct real_format *fmt;
46179 REAL_VALUE_TYPE pred_half, half_minus_pred_half;
46182 /* load nextafter (0.5, 0.0) */
46183 fmt = REAL_MODE_FORMAT (mode);
46184 real_2expN (&half_minus_pred_half, -(fmt->p) - 1, mode);
46185 REAL_ARITHMETIC (pred_half, MINUS_EXPR, dconsthalf, half_minus_pred_half);
46187 /* adj = copysign (0.5, op1) */
46188 adj = force_reg (mode, const_double_from_real_value (pred_half, mode));
46189 ix86_sse_copysign_to_positive (adj, adj, force_reg (mode, op1), NULL_RTX);
46191 /* adj = op1 + adj */
46192 adj = expand_simple_binop (mode, PLUS, adj, op1, NULL_RTX, 0, OPTAB_DIRECT);
46194 /* op0 = (imode)adj */
46195 expand_fix (op0, adj, 0);
46198 /* Expand SSE2 sequence for computing lround from OPERAND1 storing
46201 ix86_expand_lfloorceil (rtx op0, rtx op1, bool do_floor)
46203 /* C code for the stuff we're doing below (for do_floor):
46205 xi -= (double)xi > op1 ? 1 : 0;
46208 machine_mode fmode = GET_MODE (op1);
46209 machine_mode imode = GET_MODE (op0);
46210 rtx ireg, freg, tmp;
46211 rtx_code_label *label;
46213 /* reg = (long)op1 */
46214 ireg = gen_reg_rtx (imode);
46215 expand_fix (ireg, op1, 0);
46217 /* freg = (double)reg */
46218 freg = gen_reg_rtx (fmode);
46219 expand_float (freg, ireg, 0);
46221 /* ireg = (freg > op1) ? ireg - 1 : ireg */
46222 label = ix86_expand_sse_compare_and_jump (UNLE,
46223 freg, op1, !do_floor);
46224 tmp = expand_simple_binop (imode, do_floor ? MINUS : PLUS,
46225 ireg, const1_rtx, NULL_RTX, 0, OPTAB_DIRECT);
46226 emit_move_insn (ireg, tmp);
46228 emit_label (label);
46229 LABEL_NUSES (label) = 1;
46231 emit_move_insn (op0, ireg);
46234 /* Expand rint (IEEE round to nearest) rounding OPERAND1 and storing the
46235 result in OPERAND0. */
46237 ix86_expand_rint (rtx operand0, rtx operand1)
46239 /* C code for the stuff we're doing below:
46240 xa = fabs (operand1);
46241 if (!isless (xa, 2**52))
46243 xa = xa + 2**52 - 2**52;
46244 return copysign (xa, operand1);
46246 machine_mode mode = GET_MODE (operand0);
46247 rtx res, xa, TWO52, mask;
46248 rtx_code_label *label;
46250 res = gen_reg_rtx (mode);
46251 emit_move_insn (res, operand1);
46253 /* xa = abs (operand1) */
46254 xa = ix86_expand_sse_fabs (res, &mask);
46256 /* if (!isless (xa, TWO52)) goto label; */
46257 TWO52 = ix86_gen_TWO52 (mode);
46258 label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
46260 xa = expand_simple_binop (mode, PLUS, xa, TWO52, NULL_RTX, 0, OPTAB_DIRECT);
46261 xa = expand_simple_binop (mode, MINUS, xa, TWO52, xa, 0, OPTAB_DIRECT);
46263 ix86_sse_copysign_to_positive (res, xa, res, mask);
46265 emit_label (label);
46266 LABEL_NUSES (label) = 1;
46268 emit_move_insn (operand0, res);
46271 /* Expand SSE2 sequence for computing floor or ceil from OPERAND1 storing
46274 ix86_expand_floorceildf_32 (rtx operand0, rtx operand1, bool do_floor)
46276 /* C code for the stuff we expand below.
46277 double xa = fabs (x), x2;
46278 if (!isless (xa, TWO52))
46280 xa = xa + TWO52 - TWO52;
46281 x2 = copysign (xa, x);
46290 machine_mode mode = GET_MODE (operand0);
46291 rtx xa, TWO52, tmp, one, res, mask;
46292 rtx_code_label *label;
46294 TWO52 = ix86_gen_TWO52 (mode);
46296 /* Temporary for holding the result, initialized to the input
46297 operand to ease control flow. */
46298 res = gen_reg_rtx (mode);
46299 emit_move_insn (res, operand1);
46301 /* xa = abs (operand1) */
46302 xa = ix86_expand_sse_fabs (res, &mask);
46304 /* if (!isless (xa, TWO52)) goto label; */
46305 label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
46307 /* xa = xa + TWO52 - TWO52; */
46308 xa = expand_simple_binop (mode, PLUS, xa, TWO52, NULL_RTX, 0, OPTAB_DIRECT);
46309 xa = expand_simple_binop (mode, MINUS, xa, TWO52, xa, 0, OPTAB_DIRECT);
46311 /* xa = copysign (xa, operand1) */
46312 ix86_sse_copysign_to_positive (xa, xa, res, mask);
46314 /* generate 1.0 or -1.0 */
46315 one = force_reg (mode,
46316 const_double_from_real_value (do_floor
46317 ? dconst1 : dconstm1, mode));
46319 /* Compensate: xa = xa - (xa > operand1 ? 1 : 0) */
46320 tmp = ix86_expand_sse_compare_mask (UNGT, xa, res, !do_floor);
46321 emit_insn (gen_rtx_SET (tmp, gen_rtx_AND (mode, one, tmp)));
46322 /* We always need to subtract here to preserve signed zero. */
46323 tmp = expand_simple_binop (mode, MINUS,
46324 xa, tmp, NULL_RTX, 0, OPTAB_DIRECT);
46325 emit_move_insn (res, tmp);
46327 emit_label (label);
46328 LABEL_NUSES (label) = 1;
46330 emit_move_insn (operand0, res);
46333 /* Expand SSE2 sequence for computing floor or ceil from OPERAND1 storing
46336 ix86_expand_floorceil (rtx operand0, rtx operand1, bool do_floor)
46338 /* C code for the stuff we expand below.
46339 double xa = fabs (x), x2;
46340 if (!isless (xa, TWO52))
46342 x2 = (double)(long)x;
46349 if (HONOR_SIGNED_ZEROS (mode))
46350 return copysign (x2, x);
46353 machine_mode mode = GET_MODE (operand0);
46354 rtx xa, xi, TWO52, tmp, one, res, mask;
46355 rtx_code_label *label;
46357 TWO52 = ix86_gen_TWO52 (mode);
46359 /* Temporary for holding the result, initialized to the input
46360 operand to ease control flow. */
46361 res = gen_reg_rtx (mode);
46362 emit_move_insn (res, operand1);
46364 /* xa = abs (operand1) */
46365 xa = ix86_expand_sse_fabs (res, &mask);
46367 /* if (!isless (xa, TWO52)) goto label; */
46368 label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
46370 /* xa = (double)(long)x */
46371 xi = gen_reg_rtx (mode == DFmode ? DImode : SImode);
46372 expand_fix (xi, res, 0);
46373 expand_float (xa, xi, 0);
46376 one = force_reg (mode, const_double_from_real_value (dconst1, mode));
46378 /* Compensate: xa = xa - (xa > operand1 ? 1 : 0) */
46379 tmp = ix86_expand_sse_compare_mask (UNGT, xa, res, !do_floor);
46380 emit_insn (gen_rtx_SET (tmp, gen_rtx_AND (mode, one, tmp)));
46381 tmp = expand_simple_binop (mode, do_floor ? MINUS : PLUS,
46382 xa, tmp, NULL_RTX, 0, OPTAB_DIRECT);
46383 emit_move_insn (res, tmp);
46385 if (HONOR_SIGNED_ZEROS (mode))
46386 ix86_sse_copysign_to_positive (res, res, force_reg (mode, operand1), mask);
46388 emit_label (label);
46389 LABEL_NUSES (label) = 1;
46391 emit_move_insn (operand0, res);
46394 /* Expand SSE sequence for computing round from OPERAND1 storing
46395 into OPERAND0. Sequence that works without relying on DImode truncation
46396 via cvttsd2siq that is only available on 64bit targets. */
46398 ix86_expand_rounddf_32 (rtx operand0, rtx operand1)
46400 /* C code for the stuff we expand below.
46401 double xa = fabs (x), xa2, x2;
46402 if (!isless (xa, TWO52))
46404 Using the absolute value and copying back sign makes
46405 -0.0 -> -0.0 correct.
46406 xa2 = xa + TWO52 - TWO52;
46411 else if (dxa > 0.5)
46413 x2 = copysign (xa2, x);
46416 machine_mode mode = GET_MODE (operand0);
46417 rtx xa, xa2, dxa, TWO52, tmp, half, mhalf, one, res, mask;
46418 rtx_code_label *label;
46420 TWO52 = ix86_gen_TWO52 (mode);
46422 /* Temporary for holding the result, initialized to the input
46423 operand to ease control flow. */
46424 res = gen_reg_rtx (mode);
46425 emit_move_insn (res, operand1);
46427 /* xa = abs (operand1) */
46428 xa = ix86_expand_sse_fabs (res, &mask);
46430 /* if (!isless (xa, TWO52)) goto label; */
46431 label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
46433 /* xa2 = xa + TWO52 - TWO52; */
46434 xa2 = expand_simple_binop (mode, PLUS, xa, TWO52, NULL_RTX, 0, OPTAB_DIRECT);
46435 xa2 = expand_simple_binop (mode, MINUS, xa2, TWO52, xa2, 0, OPTAB_DIRECT);
46437 /* dxa = xa2 - xa; */
46438 dxa = expand_simple_binop (mode, MINUS, xa2, xa, NULL_RTX, 0, OPTAB_DIRECT);
46440 /* generate 0.5, 1.0 and -0.5 */
46441 half = force_reg (mode, const_double_from_real_value (dconsthalf, mode));
46442 one = expand_simple_binop (mode, PLUS, half, half, NULL_RTX, 0, OPTAB_DIRECT);
46443 mhalf = expand_simple_binop (mode, MINUS, half, one, NULL_RTX,
46447 tmp = gen_reg_rtx (mode);
46448 /* xa2 = xa2 - (dxa > 0.5 ? 1 : 0) */
46449 tmp = ix86_expand_sse_compare_mask (UNGT, dxa, half, false);
46450 emit_insn (gen_rtx_SET (tmp, gen_rtx_AND (mode, one, tmp)));
46451 xa2 = expand_simple_binop (mode, MINUS, xa2, tmp, NULL_RTX, 0, OPTAB_DIRECT);
46452 /* xa2 = xa2 + (dxa <= -0.5 ? 1 : 0) */
46453 tmp = ix86_expand_sse_compare_mask (UNGE, mhalf, dxa, false);
46454 emit_insn (gen_rtx_SET (tmp, gen_rtx_AND (mode, one, tmp)));
46455 xa2 = expand_simple_binop (mode, PLUS, xa2, tmp, NULL_RTX, 0, OPTAB_DIRECT);
46457 /* res = copysign (xa2, operand1) */
46458 ix86_sse_copysign_to_positive (res, xa2, force_reg (mode, operand1), mask);
46460 emit_label (label);
46461 LABEL_NUSES (label) = 1;
46463 emit_move_insn (operand0, res);
46466 /* Expand SSE sequence for computing trunc from OPERAND1 storing
46469 ix86_expand_trunc (rtx operand0, rtx operand1)
46471 /* C code for SSE variant we expand below.
46472 double xa = fabs (x), x2;
46473 if (!isless (xa, TWO52))
46475 x2 = (double)(long)x;
46476 if (HONOR_SIGNED_ZEROS (mode))
46477 return copysign (x2, x);
46480 machine_mode mode = GET_MODE (operand0);
46481 rtx xa, xi, TWO52, res, mask;
46482 rtx_code_label *label;
46484 TWO52 = ix86_gen_TWO52 (mode);
46486 /* Temporary for holding the result, initialized to the input
46487 operand to ease control flow. */
46488 res = gen_reg_rtx (mode);
46489 emit_move_insn (res, operand1);
46491 /* xa = abs (operand1) */
46492 xa = ix86_expand_sse_fabs (res, &mask);
46494 /* if (!isless (xa, TWO52)) goto label; */
46495 label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
46497 /* x = (double)(long)x */
46498 xi = gen_reg_rtx (mode == DFmode ? DImode : SImode);
46499 expand_fix (xi, res, 0);
46500 expand_float (res, xi, 0);
46502 if (HONOR_SIGNED_ZEROS (mode))
46503 ix86_sse_copysign_to_positive (res, res, force_reg (mode, operand1), mask);
46505 emit_label (label);
46506 LABEL_NUSES (label) = 1;
46508 emit_move_insn (operand0, res);
46511 /* Expand SSE sequence for computing trunc from OPERAND1 storing
46514 ix86_expand_truncdf_32 (rtx operand0, rtx operand1)
46516 machine_mode mode = GET_MODE (operand0);
46517 rtx xa, mask, TWO52, one, res, smask, tmp;
46518 rtx_code_label *label;
46520 /* C code for SSE variant we expand below.
46521 double xa = fabs (x), x2;
46522 if (!isless (xa, TWO52))
46524 xa2 = xa + TWO52 - TWO52;
46528 x2 = copysign (xa2, x);
46532 TWO52 = ix86_gen_TWO52 (mode);
46534 /* Temporary for holding the result, initialized to the input
46535 operand to ease control flow. */
46536 res = gen_reg_rtx (mode);
46537 emit_move_insn (res, operand1);
46539 /* xa = abs (operand1) */
46540 xa = ix86_expand_sse_fabs (res, &smask);
46542 /* if (!isless (xa, TWO52)) goto label; */
46543 label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
46545 /* res = xa + TWO52 - TWO52; */
46546 tmp = expand_simple_binop (mode, PLUS, xa, TWO52, NULL_RTX, 0, OPTAB_DIRECT);
46547 tmp = expand_simple_binop (mode, MINUS, tmp, TWO52, tmp, 0, OPTAB_DIRECT);
46548 emit_move_insn (res, tmp);
46551 one = force_reg (mode, const_double_from_real_value (dconst1, mode));
46553 /* Compensate: res = xa2 - (res > xa ? 1 : 0) */
46554 mask = ix86_expand_sse_compare_mask (UNGT, res, xa, false);
46555 emit_insn (gen_rtx_SET (mask, gen_rtx_AND (mode, mask, one)));
46556 tmp = expand_simple_binop (mode, MINUS,
46557 res, mask, NULL_RTX, 0, OPTAB_DIRECT);
46558 emit_move_insn (res, tmp);
46560 /* res = copysign (res, operand1) */
46561 ix86_sse_copysign_to_positive (res, res, force_reg (mode, operand1), smask);
46563 emit_label (label);
46564 LABEL_NUSES (label) = 1;
46566 emit_move_insn (operand0, res);
46569 /* Expand SSE sequence for computing round from OPERAND1 storing
46572 ix86_expand_round (rtx operand0, rtx operand1)
46574 /* C code for the stuff we're doing below:
46575 double xa = fabs (x);
46576 if (!isless (xa, TWO52))
46578 xa = (double)(long)(xa + nextafter (0.5, 0.0));
46579 return copysign (xa, x);
46581 machine_mode mode = GET_MODE (operand0);
46582 rtx res, TWO52, xa, xi, half, mask;
46583 rtx_code_label *label;
46584 const struct real_format *fmt;
46585 REAL_VALUE_TYPE pred_half, half_minus_pred_half;
46587 /* Temporary for holding the result, initialized to the input
46588 operand to ease control flow. */
46589 res = gen_reg_rtx (mode);
46590 emit_move_insn (res, operand1);
46592 TWO52 = ix86_gen_TWO52 (mode);
46593 xa = ix86_expand_sse_fabs (res, &mask);
46594 label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
46596 /* load nextafter (0.5, 0.0) */
46597 fmt = REAL_MODE_FORMAT (mode);
46598 real_2expN (&half_minus_pred_half, -(fmt->p) - 1, mode);
46599 REAL_ARITHMETIC (pred_half, MINUS_EXPR, dconsthalf, half_minus_pred_half);
46601 /* xa = xa + 0.5 */
46602 half = force_reg (mode, const_double_from_real_value (pred_half, mode));
46603 xa = expand_simple_binop (mode, PLUS, xa, half, NULL_RTX, 0, OPTAB_DIRECT);
46605 /* xa = (double)(int64_t)xa */
46606 xi = gen_reg_rtx (mode == DFmode ? DImode : SImode);
46607 expand_fix (xi, xa, 0);
46608 expand_float (xa, xi, 0);
46610 /* res = copysign (xa, operand1) */
46611 ix86_sse_copysign_to_positive (res, xa, force_reg (mode, operand1), mask);
46613 emit_label (label);
46614 LABEL_NUSES (label) = 1;
46616 emit_move_insn (operand0, res);
46619 /* Expand SSE sequence for computing round
46620 from OP1 storing into OP0 using sse4 round insn. */
46622 ix86_expand_round_sse4 (rtx op0, rtx op1)
46624 machine_mode mode = GET_MODE (op0);
46625 rtx e1, e2, res, half;
46626 const struct real_format *fmt;
46627 REAL_VALUE_TYPE pred_half, half_minus_pred_half;
46628 rtx (*gen_copysign) (rtx, rtx, rtx);
46629 rtx (*gen_round) (rtx, rtx, rtx);
46634 gen_copysign = gen_copysignsf3;
46635 gen_round = gen_sse4_1_roundsf2;
46638 gen_copysign = gen_copysigndf3;
46639 gen_round = gen_sse4_1_rounddf2;
46642 gcc_unreachable ();
46645 /* round (a) = trunc (a + copysign (0.5, a)) */
46647 /* load nextafter (0.5, 0.0) */
46648 fmt = REAL_MODE_FORMAT (mode);
46649 real_2expN (&half_minus_pred_half, -(fmt->p) - 1, mode);
46650 REAL_ARITHMETIC (pred_half, MINUS_EXPR, dconsthalf, half_minus_pred_half);
46651 half = const_double_from_real_value (pred_half, mode);
46653 /* e1 = copysign (0.5, op1) */
46654 e1 = gen_reg_rtx (mode);
46655 emit_insn (gen_copysign (e1, half, op1));
46657 /* e2 = op1 + e1 */
46658 e2 = expand_simple_binop (mode, PLUS, op1, e1, NULL_RTX, 0, OPTAB_DIRECT);
46660 /* res = trunc (e2) */
46661 res = gen_reg_rtx (mode);
46662 emit_insn (gen_round (res, e2, GEN_INT (ROUND_TRUNC)));
46664 emit_move_insn (op0, res);
46668 /* Table of valid machine attributes. */
46669 static const struct attribute_spec ix86_attribute_table[] =
46671 /* { name, min_len, max_len, decl_req, type_req, fn_type_req, handler,
46672 affects_type_identity } */
46673 /* Stdcall attribute says callee is responsible for popping arguments
46674 if they are not variable. */
46675 { "stdcall", 0, 0, false, true, true, ix86_handle_cconv_attribute,
46677 /* Fastcall attribute says callee is responsible for popping arguments
46678 if they are not variable. */
46679 { "fastcall", 0, 0, false, true, true, ix86_handle_cconv_attribute,
46681 /* Thiscall attribute says callee is responsible for popping arguments
46682 if they are not variable. */
46683 { "thiscall", 0, 0, false, true, true, ix86_handle_cconv_attribute,
46685 /* Cdecl attribute says the callee is a normal C declaration */
46686 { "cdecl", 0, 0, false, true, true, ix86_handle_cconv_attribute,
46688 /* Regparm attribute specifies how many integer arguments are to be
46689 passed in registers. */
46690 { "regparm", 1, 1, false, true, true, ix86_handle_cconv_attribute,
46692 /* Sseregparm attribute says we are using x86_64 calling conventions
46693 for FP arguments. */
46694 { "sseregparm", 0, 0, false, true, true, ix86_handle_cconv_attribute,
46696 /* The transactional memory builtins are implicitly regparm or fastcall
46697 depending on the ABI. Override the generic do-nothing attribute that
46698 these builtins were declared with. */
46699 { "*tm regparm", 0, 0, false, true, true, ix86_handle_tm_regparm_attribute,
46701 /* force_align_arg_pointer says this function realigns the stack at entry. */
46702 { (const char *)&ix86_force_align_arg_pointer_string, 0, 0,
46703 false, true, true, ix86_handle_cconv_attribute, false },
46704 #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
46705 { "dllimport", 0, 0, false, false, false, handle_dll_attribute, false },
46706 { "dllexport", 0, 0, false, false, false, handle_dll_attribute, false },
46707 { "shared", 0, 0, true, false, false, ix86_handle_shared_attribute,
46710 { "ms_struct", 0, 0, false, false, false, ix86_handle_struct_attribute,
46712 { "gcc_struct", 0, 0, false, false, false, ix86_handle_struct_attribute,
46714 #ifdef SUBTARGET_ATTRIBUTE_TABLE
46715 SUBTARGET_ATTRIBUTE_TABLE,
46717 /* ms_abi and sysv_abi calling convention function attributes. */
46718 { "ms_abi", 0, 0, false, true, true, ix86_handle_abi_attribute, true },
46719 { "sysv_abi", 0, 0, false, true, true, ix86_handle_abi_attribute, true },
46720 { "ms_hook_prologue", 0, 0, true, false, false, ix86_handle_fndecl_attribute,
46722 { "callee_pop_aggregate_return", 1, 1, false, true, true,
46723 ix86_handle_callee_pop_aggregate_return, true },
46725 { NULL, 0, 0, false, false, false, NULL, false }
46728 /* Implement targetm.vectorize.builtin_vectorization_cost. */
46730 ix86_builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost,
46735 switch (type_of_cost)
46738 return ix86_cost->scalar_stmt_cost;
46741 return ix86_cost->scalar_load_cost;
46744 return ix86_cost->scalar_store_cost;
46747 return ix86_cost->vec_stmt_cost;
46750 return ix86_cost->vec_align_load_cost;
46753 return ix86_cost->vec_store_cost;
46755 case vec_to_scalar:
46756 return ix86_cost->vec_to_scalar_cost;
46758 case scalar_to_vec:
46759 return ix86_cost->scalar_to_vec_cost;
46761 case unaligned_load:
46762 case unaligned_store:
46763 return ix86_cost->vec_unalign_load_cost;
46765 case cond_branch_taken:
46766 return ix86_cost->cond_taken_branch_cost;
46768 case cond_branch_not_taken:
46769 return ix86_cost->cond_not_taken_branch_cost;
46772 case vec_promote_demote:
46773 return ix86_cost->vec_stmt_cost;
46775 case vec_construct:
46776 elements = TYPE_VECTOR_SUBPARTS (vectype);
46777 return ix86_cost->vec_stmt_cost * (elements / 2 + 1);
46780 gcc_unreachable ();
46784 /* A cached (set (nil) (vselect (vconcat (nil) (nil)) (parallel [])))
46785 insn, so that expand_vselect{,_vconcat} doesn't have to create a fresh
46786 insn every time. */
46788 static GTY(()) rtx_insn *vselect_insn;
46790 /* Initialize vselect_insn. */
46793 init_vselect_insn (void)
46798 x = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (MAX_VECT_LEN));
46799 for (i = 0; i < MAX_VECT_LEN; ++i)
46800 XVECEXP (x, 0, i) = const0_rtx;
46801 x = gen_rtx_VEC_SELECT (V2DFmode, gen_rtx_VEC_CONCAT (V4DFmode, const0_rtx,
46803 x = gen_rtx_SET (const0_rtx, x);
46805 vselect_insn = emit_insn (x);
46809 /* Construct (set target (vec_select op0 (parallel perm))) and
46810 return true if that's a valid instruction in the active ISA. */
46813 expand_vselect (rtx target, rtx op0, const unsigned char *perm,
46814 unsigned nelt, bool testing_p)
46817 rtx x, save_vconcat;
46820 if (vselect_insn == NULL_RTX)
46821 init_vselect_insn ();
46823 x = XEXP (SET_SRC (PATTERN (vselect_insn)), 1);
46824 PUT_NUM_ELEM (XVEC (x, 0), nelt);
46825 for (i = 0; i < nelt; ++i)
46826 XVECEXP (x, 0, i) = GEN_INT (perm[i]);
46827 save_vconcat = XEXP (SET_SRC (PATTERN (vselect_insn)), 0);
46828 XEXP (SET_SRC (PATTERN (vselect_insn)), 0) = op0;
46829 PUT_MODE (SET_SRC (PATTERN (vselect_insn)), GET_MODE (target));
46830 SET_DEST (PATTERN (vselect_insn)) = target;
46831 icode = recog_memoized (vselect_insn);
46833 if (icode >= 0 && !testing_p)
46834 emit_insn (copy_rtx (PATTERN (vselect_insn)));
46836 SET_DEST (PATTERN (vselect_insn)) = const0_rtx;
46837 XEXP (SET_SRC (PATTERN (vselect_insn)), 0) = save_vconcat;
46838 INSN_CODE (vselect_insn) = -1;
46843 /* Similar, but generate a vec_concat from op0 and op1 as well. */
46846 expand_vselect_vconcat (rtx target, rtx op0, rtx op1,
46847 const unsigned char *perm, unsigned nelt,
46850 machine_mode v2mode;
46854 if (vselect_insn == NULL_RTX)
46855 init_vselect_insn ();
46857 v2mode = GET_MODE_2XWIDER_MODE (GET_MODE (op0));
46858 x = XEXP (SET_SRC (PATTERN (vselect_insn)), 0);
46859 PUT_MODE (x, v2mode);
46862 ok = expand_vselect (target, x, perm, nelt, testing_p);
46863 XEXP (x, 0) = const0_rtx;
46864 XEXP (x, 1) = const0_rtx;
46868 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
46869 in terms of blendp[sd] / pblendw / pblendvb / vpblendd. */
46872 expand_vec_perm_blend (struct expand_vec_perm_d *d)
46874 machine_mode mmode, vmode = d->vmode;
46875 unsigned i, mask, nelt = d->nelt;
46876 rtx target, op0, op1, maskop, x;
46877 rtx rperm[32], vperm;
46879 if (d->one_operand_p)
46881 if (TARGET_AVX512F && GET_MODE_SIZE (vmode) == 64
46882 && (TARGET_AVX512BW
46883 || GET_MODE_SIZE (GET_MODE_INNER (vmode)) >= 4))
46885 else if (TARGET_AVX2 && GET_MODE_SIZE (vmode) == 32)
46887 else if (TARGET_AVX && (vmode == V4DFmode || vmode == V8SFmode))
46889 else if (TARGET_SSE4_1 && GET_MODE_SIZE (vmode) == 16)
46894 /* This is a blend, not a permute. Elements must stay in their
46895 respective lanes. */
46896 for (i = 0; i < nelt; ++i)
46898 unsigned e = d->perm[i];
46899 if (!(e == i || e == i + nelt))
46906 /* ??? Without SSE4.1, we could implement this with and/andn/or. This
46907 decision should be extracted elsewhere, so that we only try that
46908 sequence once all budget==3 options have been tried. */
46909 target = d->target;
46928 for (i = 0; i < nelt; ++i)
46929 mask |= (d->perm[i] >= nelt) << i;
46933 for (i = 0; i < 2; ++i)
46934 mask |= (d->perm[i] >= 2 ? 15 : 0) << (i * 4);
46939 for (i = 0; i < 4; ++i)
46940 mask |= (d->perm[i] >= 4 ? 3 : 0) << (i * 2);
46945 /* See if bytes move in pairs so we can use pblendw with
46946 an immediate argument, rather than pblendvb with a vector
46948 for (i = 0; i < 16; i += 2)
46949 if (d->perm[i] + 1 != d->perm[i + 1])
46952 for (i = 0; i < nelt; ++i)
46953 rperm[i] = (d->perm[i] < nelt ? const0_rtx : constm1_rtx);
46956 vperm = gen_rtx_CONST_VECTOR (vmode, gen_rtvec_v (nelt, rperm));
46957 vperm = force_reg (vmode, vperm);
46959 if (GET_MODE_SIZE (vmode) == 16)
46960 emit_insn (gen_sse4_1_pblendvb (target, op0, op1, vperm));
46962 emit_insn (gen_avx2_pblendvb (target, op0, op1, vperm));
46963 if (target != d->target)
46964 emit_move_insn (d->target, gen_lowpart (d->vmode, target));
46968 for (i = 0; i < 8; ++i)
46969 mask |= (d->perm[i * 2] >= 16) << i;
46974 target = gen_reg_rtx (vmode);
46975 op0 = gen_lowpart (vmode, op0);
46976 op1 = gen_lowpart (vmode, op1);
46980 /* See if bytes move in pairs. If not, vpblendvb must be used. */
46981 for (i = 0; i < 32; i += 2)
46982 if (d->perm[i] + 1 != d->perm[i + 1])
46984 /* See if bytes move in quadruplets. If yes, vpblendd
46985 with immediate can be used. */
46986 for (i = 0; i < 32; i += 4)
46987 if (d->perm[i] + 2 != d->perm[i + 2])
46991 /* See if bytes move the same in both lanes. If yes,
46992 vpblendw with immediate can be used. */
46993 for (i = 0; i < 16; i += 2)
46994 if (d->perm[i] + 16 != d->perm[i + 16])
46997 /* Use vpblendw. */
46998 for (i = 0; i < 16; ++i)
46999 mask |= (d->perm[i * 2] >= 32) << i;
47004 /* Use vpblendd. */
47005 for (i = 0; i < 8; ++i)
47006 mask |= (d->perm[i * 4] >= 32) << i;
47011 /* See if words move in pairs. If yes, vpblendd can be used. */
47012 for (i = 0; i < 16; i += 2)
47013 if (d->perm[i] + 1 != d->perm[i + 1])
47017 /* See if words move the same in both lanes. If not,
47018 vpblendvb must be used. */
47019 for (i = 0; i < 8; i++)
47020 if (d->perm[i] + 8 != d->perm[i + 8])
47022 /* Use vpblendvb. */
47023 for (i = 0; i < 32; ++i)
47024 rperm[i] = (d->perm[i / 2] < 16 ? const0_rtx : constm1_rtx);
47028 target = gen_reg_rtx (vmode);
47029 op0 = gen_lowpart (vmode, op0);
47030 op1 = gen_lowpart (vmode, op1);
47031 goto finish_pblendvb;
47034 /* Use vpblendw. */
47035 for (i = 0; i < 16; ++i)
47036 mask |= (d->perm[i] >= 16) << i;
47040 /* Use vpblendd. */
47041 for (i = 0; i < 8; ++i)
47042 mask |= (d->perm[i * 2] >= 16) << i;
47047 /* Use vpblendd. */
47048 for (i = 0; i < 4; ++i)
47049 mask |= (d->perm[i] >= 4 ? 3 : 0) << (i * 2);
47054 gcc_unreachable ();
47077 if (mmode != VOIDmode)
47078 maskop = force_reg (mmode, gen_int_mode (mask, mmode));
47080 maskop = GEN_INT (mask);
47082 /* This matches five different patterns with the different modes. */
47083 x = gen_rtx_VEC_MERGE (vmode, op1, op0, maskop);
47084 x = gen_rtx_SET (target, x);
47086 if (target != d->target)
47087 emit_move_insn (d->target, gen_lowpart (d->vmode, target));
47092 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
47093 in terms of the variable form of vpermilps.
47095 Note that we will have already failed the immediate input vpermilps,
47096 which requires that the high and low part shuffle be identical; the
47097 variable form doesn't require that. */
47100 expand_vec_perm_vpermil (struct expand_vec_perm_d *d)
47102 rtx rperm[8], vperm;
47105 if (!TARGET_AVX || d->vmode != V8SFmode || !d->one_operand_p)
47108 /* We can only permute within the 128-bit lane. */
47109 for (i = 0; i < 8; ++i)
47111 unsigned e = d->perm[i];
47112 if (i < 4 ? e >= 4 : e < 4)
47119 for (i = 0; i < 8; ++i)
47121 unsigned e = d->perm[i];
47123 /* Within each 128-bit lane, the elements of op0 are numbered
47124 from 0 and the elements of op1 are numbered from 4. */
47130 rperm[i] = GEN_INT (e);
47133 vperm = gen_rtx_CONST_VECTOR (V8SImode, gen_rtvec_v (8, rperm));
47134 vperm = force_reg (V8SImode, vperm);
47135 emit_insn (gen_avx_vpermilvarv8sf3 (d->target, d->op0, vperm));
47140 /* Return true if permutation D can be performed as VMODE permutation
47144 valid_perm_using_mode_p (machine_mode vmode, struct expand_vec_perm_d *d)
47146 unsigned int i, j, chunk;
47148 if (GET_MODE_CLASS (vmode) != MODE_VECTOR_INT
47149 || GET_MODE_CLASS (d->vmode) != MODE_VECTOR_INT
47150 || GET_MODE_SIZE (vmode) != GET_MODE_SIZE (d->vmode))
47153 if (GET_MODE_NUNITS (vmode) >= d->nelt)
47156 chunk = d->nelt / GET_MODE_NUNITS (vmode);
47157 for (i = 0; i < d->nelt; i += chunk)
47158 if (d->perm[i] & (chunk - 1))
47161 for (j = 1; j < chunk; ++j)
47162 if (d->perm[i] + j != d->perm[i + j])
47168 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
47169 in terms of pshufb, vpperm, vpermq, vpermd, vpermps or vperm2i128. */
47172 expand_vec_perm_pshufb (struct expand_vec_perm_d *d)
47174 unsigned i, nelt, eltsz, mask;
47175 unsigned char perm[64];
47176 machine_mode vmode = V16QImode;
47177 rtx rperm[64], vperm, target, op0, op1;
47181 if (!d->one_operand_p)
47183 if (!TARGET_XOP || GET_MODE_SIZE (d->vmode) != 16)
47186 && valid_perm_using_mode_p (V2TImode, d))
47191 /* Use vperm2i128 insn. The pattern uses
47192 V4DImode instead of V2TImode. */
47193 target = d->target;
47194 if (d->vmode != V4DImode)
47195 target = gen_reg_rtx (V4DImode);
47196 op0 = gen_lowpart (V4DImode, d->op0);
47197 op1 = gen_lowpart (V4DImode, d->op1);
47199 = GEN_INT ((d->perm[0] / (nelt / 2))
47200 | ((d->perm[nelt / 2] / (nelt / 2)) * 16));
47201 emit_insn (gen_avx2_permv2ti (target, op0, op1, rperm[0]));
47202 if (target != d->target)
47203 emit_move_insn (d->target, gen_lowpart (d->vmode, target));
47211 if (GET_MODE_SIZE (d->vmode) == 16)
47216 else if (GET_MODE_SIZE (d->vmode) == 32)
47221 /* V4DImode should be already handled through
47222 expand_vselect by vpermq instruction. */
47223 gcc_assert (d->vmode != V4DImode);
47226 if (d->vmode == V8SImode
47227 || d->vmode == V16HImode
47228 || d->vmode == V32QImode)
47230 /* First see if vpermq can be used for
47231 V8SImode/V16HImode/V32QImode. */
47232 if (valid_perm_using_mode_p (V4DImode, d))
47234 for (i = 0; i < 4; i++)
47235 perm[i] = (d->perm[i * nelt / 4] * 4 / nelt) & 3;
47238 target = gen_reg_rtx (V4DImode);
47239 if (expand_vselect (target, gen_lowpart (V4DImode, d->op0),
47242 emit_move_insn (d->target,
47243 gen_lowpart (d->vmode, target));
47249 /* Next see if vpermd can be used. */
47250 if (valid_perm_using_mode_p (V8SImode, d))
47253 /* Or if vpermps can be used. */
47254 else if (d->vmode == V8SFmode)
47257 if (vmode == V32QImode)
47259 /* vpshufb only works intra lanes, it is not
47260 possible to shuffle bytes in between the lanes. */
47261 for (i = 0; i < nelt; ++i)
47262 if ((d->perm[i] ^ i) & (nelt / 2))
47266 else if (GET_MODE_SIZE (d->vmode) == 64)
47268 if (!TARGET_AVX512BW)
47271 /* If vpermq didn't work, vpshufb won't work either. */
47272 if (d->vmode == V8DFmode || d->vmode == V8DImode)
47276 if (d->vmode == V16SImode
47277 || d->vmode == V32HImode
47278 || d->vmode == V64QImode)
47280 /* First see if vpermq can be used for
47281 V16SImode/V32HImode/V64QImode. */
47282 if (valid_perm_using_mode_p (V8DImode, d))
47284 for (i = 0; i < 8; i++)
47285 perm[i] = (d->perm[i * nelt / 8] * 8 / nelt) & 7;
47288 target = gen_reg_rtx (V8DImode);
47289 if (expand_vselect (target, gen_lowpart (V8DImode, d->op0),
47292 emit_move_insn (d->target,
47293 gen_lowpart (d->vmode, target));
47299 /* Next see if vpermd can be used. */
47300 if (valid_perm_using_mode_p (V16SImode, d))
47303 /* Or if vpermps can be used. */
47304 else if (d->vmode == V16SFmode)
47306 if (vmode == V64QImode)
47308 /* vpshufb only works intra lanes, it is not
47309 possible to shuffle bytes in between the lanes. */
47310 for (i = 0; i < nelt; ++i)
47311 if ((d->perm[i] ^ i) & (nelt / 4))
47322 if (vmode == V8SImode)
47323 for (i = 0; i < 8; ++i)
47324 rperm[i] = GEN_INT ((d->perm[i * nelt / 8] * 8 / nelt) & 7);
47325 else if (vmode == V16SImode)
47326 for (i = 0; i < 16; ++i)
47327 rperm[i] = GEN_INT ((d->perm[i * nelt / 16] * 16 / nelt) & 15);
47330 eltsz = GET_MODE_SIZE (GET_MODE_INNER (d->vmode));
47331 if (!d->one_operand_p)
47332 mask = 2 * nelt - 1;
47333 else if (vmode == V16QImode)
47335 else if (vmode == V64QImode)
47336 mask = nelt / 4 - 1;
47338 mask = nelt / 2 - 1;
47340 for (i = 0; i < nelt; ++i)
47342 unsigned j, e = d->perm[i] & mask;
47343 for (j = 0; j < eltsz; ++j)
47344 rperm[i * eltsz + j] = GEN_INT (e * eltsz + j);
47348 vperm = gen_rtx_CONST_VECTOR (vmode,
47349 gen_rtvec_v (GET_MODE_NUNITS (vmode), rperm));
47350 vperm = force_reg (vmode, vperm);
47352 target = d->target;
47353 if (d->vmode != vmode)
47354 target = gen_reg_rtx (vmode);
47355 op0 = gen_lowpart (vmode, d->op0);
47356 if (d->one_operand_p)
47358 if (vmode == V16QImode)
47359 emit_insn (gen_ssse3_pshufbv16qi3 (target, op0, vperm));
47360 else if (vmode == V32QImode)
47361 emit_insn (gen_avx2_pshufbv32qi3 (target, op0, vperm));
47362 else if (vmode == V64QImode)
47363 emit_insn (gen_avx512bw_pshufbv64qi3 (target, op0, vperm));
47364 else if (vmode == V8SFmode)
47365 emit_insn (gen_avx2_permvarv8sf (target, op0, vperm));
47366 else if (vmode == V8SImode)
47367 emit_insn (gen_avx2_permvarv8si (target, op0, vperm));
47368 else if (vmode == V16SFmode)
47369 emit_insn (gen_avx512f_permvarv16sf (target, op0, vperm));
47370 else if (vmode == V16SImode)
47371 emit_insn (gen_avx512f_permvarv16si (target, op0, vperm));
47373 gcc_unreachable ();
47377 op1 = gen_lowpart (vmode, d->op1);
47378 emit_insn (gen_xop_pperm (target, op0, op1, vperm));
47380 if (target != d->target)
47381 emit_move_insn (d->target, gen_lowpart (d->vmode, target));
47386 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to instantiate D
47387 in a single instruction. */
47390 expand_vec_perm_1 (struct expand_vec_perm_d *d)
47392 unsigned i, nelt = d->nelt;
47393 unsigned char perm2[MAX_VECT_LEN];
47395 /* Check plain VEC_SELECT first, because AVX has instructions that could
47396 match both SEL and SEL+CONCAT, but the plain SEL will allow a memory
47397 input where SEL+CONCAT may not. */
47398 if (d->one_operand_p)
47400 int mask = nelt - 1;
47401 bool identity_perm = true;
47402 bool broadcast_perm = true;
47404 for (i = 0; i < nelt; i++)
47406 perm2[i] = d->perm[i] & mask;
47408 identity_perm = false;
47410 broadcast_perm = false;
47416 emit_move_insn (d->target, d->op0);
47419 else if (broadcast_perm && TARGET_AVX2)
47421 /* Use vpbroadcast{b,w,d}. */
47422 rtx (*gen) (rtx, rtx) = NULL;
47426 if (TARGET_AVX512BW)
47427 gen = gen_avx512bw_vec_dupv64qi_1;
47430 gen = gen_avx2_pbroadcastv32qi_1;
47433 if (TARGET_AVX512BW)
47434 gen = gen_avx512bw_vec_dupv32hi_1;
47437 gen = gen_avx2_pbroadcastv16hi_1;
47440 if (TARGET_AVX512F)
47441 gen = gen_avx512f_vec_dupv16si_1;
47444 gen = gen_avx2_pbroadcastv8si_1;
47447 gen = gen_avx2_pbroadcastv16qi;
47450 gen = gen_avx2_pbroadcastv8hi;
47453 if (TARGET_AVX512F)
47454 gen = gen_avx512f_vec_dupv16sf_1;
47457 gen = gen_avx2_vec_dupv8sf_1;
47460 if (TARGET_AVX512F)
47461 gen = gen_avx512f_vec_dupv8df_1;
47464 if (TARGET_AVX512F)
47465 gen = gen_avx512f_vec_dupv8di_1;
47467 /* For other modes prefer other shuffles this function creates. */
47473 emit_insn (gen (d->target, d->op0));
47478 if (expand_vselect (d->target, d->op0, perm2, nelt, d->testing_p))
47481 /* There are plenty of patterns in sse.md that are written for
47482 SEL+CONCAT and are not replicated for a single op. Perhaps
47483 that should be changed, to avoid the nastiness here. */
47485 /* Recognize interleave style patterns, which means incrementing
47486 every other permutation operand. */
47487 for (i = 0; i < nelt; i += 2)
47489 perm2[i] = d->perm[i] & mask;
47490 perm2[i + 1] = (d->perm[i + 1] & mask) + nelt;
47492 if (expand_vselect_vconcat (d->target, d->op0, d->op0, perm2, nelt,
47496 /* Recognize shufps, which means adding {0, 0, nelt, nelt}. */
47499 for (i = 0; i < nelt; i += 4)
47501 perm2[i + 0] = d->perm[i + 0] & mask;
47502 perm2[i + 1] = d->perm[i + 1] & mask;
47503 perm2[i + 2] = (d->perm[i + 2] & mask) + nelt;
47504 perm2[i + 3] = (d->perm[i + 3] & mask) + nelt;
47507 if (expand_vselect_vconcat (d->target, d->op0, d->op0, perm2, nelt,
47513 /* Finally, try the fully general two operand permute. */
47514 if (expand_vselect_vconcat (d->target, d->op0, d->op1, d->perm, nelt,
47518 /* Recognize interleave style patterns with reversed operands. */
47519 if (!d->one_operand_p)
47521 for (i = 0; i < nelt; ++i)
47523 unsigned e = d->perm[i];
47531 if (expand_vselect_vconcat (d->target, d->op1, d->op0, perm2, nelt,
47536 /* Try the SSE4.1 blend variable merge instructions. */
47537 if (expand_vec_perm_blend (d))
47540 /* Try one of the AVX vpermil variable permutations. */
47541 if (expand_vec_perm_vpermil (d))
47544 /* Try the SSSE3 pshufb or XOP vpperm or AVX2 vperm2i128,
47545 vpshufb, vpermd, vpermps or vpermq variable permutation. */
47546 if (expand_vec_perm_pshufb (d))
47549 /* Try the AVX2 vpalignr instruction. */
47550 if (expand_vec_perm_palignr (d, true))
47553 /* Try the AVX512F vpermi2 instructions. */
47554 if (ix86_expand_vec_perm_vpermi2 (NULL_RTX, NULL_RTX, NULL_RTX, NULL_RTX, d))
47560 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
47561 in terms of a pair of pshuflw + pshufhw instructions. */
47564 expand_vec_perm_pshuflw_pshufhw (struct expand_vec_perm_d *d)
47566 unsigned char perm2[MAX_VECT_LEN];
47570 if (d->vmode != V8HImode || !d->one_operand_p)
47573 /* The two permutations only operate in 64-bit lanes. */
47574 for (i = 0; i < 4; ++i)
47575 if (d->perm[i] >= 4)
47577 for (i = 4; i < 8; ++i)
47578 if (d->perm[i] < 4)
47584 /* Emit the pshuflw. */
47585 memcpy (perm2, d->perm, 4);
47586 for (i = 4; i < 8; ++i)
47588 ok = expand_vselect (d->target, d->op0, perm2, 8, d->testing_p);
47591 /* Emit the pshufhw. */
47592 memcpy (perm2 + 4, d->perm + 4, 4);
47593 for (i = 0; i < 4; ++i)
47595 ok = expand_vselect (d->target, d->target, perm2, 8, d->testing_p);
47601 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
47602 the permutation using the SSSE3 palignr instruction. This succeeds
47603 when all of the elements in PERM fit within one vector and we merely
47604 need to shift them down so that a single vector permutation has a
47605 chance to succeed. If SINGLE_INSN_ONLY_P, succeed if only
47606 the vpalignr instruction itself can perform the requested permutation. */
47609 expand_vec_perm_palignr (struct expand_vec_perm_d *d, bool single_insn_only_p)
47611 unsigned i, nelt = d->nelt;
47612 unsigned min, max, minswap, maxswap;
47613 bool in_order, ok, swap = false;
47615 struct expand_vec_perm_d dcopy;
47617 /* Even with AVX, palignr only operates on 128-bit vectors,
47618 in AVX2 palignr operates on both 128-bit lanes. */
47619 if ((!TARGET_SSSE3 || GET_MODE_SIZE (d->vmode) != 16)
47620 && (!TARGET_AVX2 || GET_MODE_SIZE (d->vmode) != 32))
47625 minswap = 2 * nelt;
47627 for (i = 0; i < nelt; ++i)
47629 unsigned e = d->perm[i];
47630 unsigned eswap = d->perm[i] ^ nelt;
47631 if (GET_MODE_SIZE (d->vmode) == 32)
47633 e = (e & ((nelt / 2) - 1)) | ((e & nelt) >> 1);
47634 eswap = e ^ (nelt / 2);
47640 if (eswap < minswap)
47642 if (eswap > maxswap)
47646 || max - min >= (GET_MODE_SIZE (d->vmode) == 32 ? nelt / 2 : nelt))
47648 if (d->one_operand_p
47650 || maxswap - minswap >= (GET_MODE_SIZE (d->vmode) == 32
47651 ? nelt / 2 : nelt))
47658 /* Given that we have SSSE3, we know we'll be able to implement the
47659 single operand permutation after the palignr with pshufb for
47660 128-bit vectors. If SINGLE_INSN_ONLY_P, in_order has to be computed
47662 if (d->testing_p && GET_MODE_SIZE (d->vmode) == 16 && !single_insn_only_p)
47668 dcopy.op0 = d->op1;
47669 dcopy.op1 = d->op0;
47670 for (i = 0; i < nelt; ++i)
47671 dcopy.perm[i] ^= nelt;
47675 for (i = 0; i < nelt; ++i)
47677 unsigned e = dcopy.perm[i];
47678 if (GET_MODE_SIZE (d->vmode) == 32
47680 && (e & (nelt / 2 - 1)) < min)
47681 e = e - min - (nelt / 2);
47688 dcopy.one_operand_p = true;
47690 if (single_insn_only_p && !in_order)
47693 /* For AVX2, test whether we can permute the result in one instruction. */
47698 dcopy.op1 = dcopy.op0;
47699 return expand_vec_perm_1 (&dcopy);
47702 shift = GEN_INT (min * GET_MODE_BITSIZE (GET_MODE_INNER (d->vmode)));
47703 if (GET_MODE_SIZE (d->vmode) == 16)
47705 target = gen_reg_rtx (TImode);
47706 emit_insn (gen_ssse3_palignrti (target, gen_lowpart (TImode, dcopy.op1),
47707 gen_lowpart (TImode, dcopy.op0), shift));
47711 target = gen_reg_rtx (V2TImode);
47712 emit_insn (gen_avx2_palignrv2ti (target,
47713 gen_lowpart (V2TImode, dcopy.op1),
47714 gen_lowpart (V2TImode, dcopy.op0),
47718 dcopy.op0 = dcopy.op1 = gen_lowpart (d->vmode, target);
47720 /* Test for the degenerate case where the alignment by itself
47721 produces the desired permutation. */
47724 emit_move_insn (d->target, dcopy.op0);
47728 ok = expand_vec_perm_1 (&dcopy);
47729 gcc_assert (ok || GET_MODE_SIZE (d->vmode) == 32);
47734 /* A subroutine of ix86_expand_vec_perm_const_1. Try to simplify
47735 the permutation using the SSE4_1 pblendv instruction. Potentially
47736 reduces permutation from 2 pshufb and or to 1 pshufb and pblendv. */
47739 expand_vec_perm_pblendv (struct expand_vec_perm_d *d)
47741 unsigned i, which, nelt = d->nelt;
47742 struct expand_vec_perm_d dcopy, dcopy1;
47743 machine_mode vmode = d->vmode;
47746 /* Use the same checks as in expand_vec_perm_blend. */
47747 if (d->one_operand_p)
47749 if (TARGET_AVX2 && GET_MODE_SIZE (vmode) == 32)
47751 else if (TARGET_AVX && (vmode == V4DFmode || vmode == V8SFmode))
47753 else if (TARGET_SSE4_1 && GET_MODE_SIZE (vmode) == 16)
47758 /* Figure out where permutation elements stay not in their
47759 respective lanes. */
47760 for (i = 0, which = 0; i < nelt; ++i)
47762 unsigned e = d->perm[i];
47764 which |= (e < nelt ? 1 : 2);
47766 /* We can pblend the part where elements stay not in their
47767 respective lanes only when these elements are all in one
47768 half of a permutation.
47769 {0 1 8 3 4 5 9 7} is ok as 8, 9 are at not at their respective
47770 lanes, but both 8 and 9 >= 8
47771 {0 1 8 3 4 5 2 7} is not ok as 2 and 8 are not at their
47772 respective lanes and 8 >= 8, but 2 not. */
47773 if (which != 1 && which != 2)
47775 if (d->testing_p && GET_MODE_SIZE (vmode) == 16)
47778 /* First we apply one operand permutation to the part where
47779 elements stay not in their respective lanes. */
47782 dcopy.op0 = dcopy.op1 = d->op1;
47784 dcopy.op0 = dcopy.op1 = d->op0;
47786 dcopy.target = gen_reg_rtx (vmode);
47787 dcopy.one_operand_p = true;
47789 for (i = 0; i < nelt; ++i)
47790 dcopy.perm[i] = d->perm[i] & (nelt - 1);
47792 ok = expand_vec_perm_1 (&dcopy);
47793 if (GET_MODE_SIZE (vmode) != 16 && !ok)
47800 /* Next we put permuted elements into their positions. */
47803 dcopy1.op1 = dcopy.target;
47805 dcopy1.op0 = dcopy.target;
47807 for (i = 0; i < nelt; ++i)
47808 dcopy1.perm[i] = ((d->perm[i] >= nelt) ? (nelt + i) : i);
47810 ok = expand_vec_perm_blend (&dcopy1);
47816 static bool expand_vec_perm_interleave3 (struct expand_vec_perm_d *d);
47818 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
47819 a two vector permutation into a single vector permutation by using
47820 an interleave operation to merge the vectors. */
47823 expand_vec_perm_interleave2 (struct expand_vec_perm_d *d)
47825 struct expand_vec_perm_d dremap, dfinal;
47826 unsigned i, nelt = d->nelt, nelt2 = nelt / 2;
47827 unsigned HOST_WIDE_INT contents;
47828 unsigned char remap[2 * MAX_VECT_LEN];
47830 bool ok, same_halves = false;
47832 if (GET_MODE_SIZE (d->vmode) == 16)
47834 if (d->one_operand_p)
47837 else if (GET_MODE_SIZE (d->vmode) == 32)
47841 /* For 32-byte modes allow even d->one_operand_p.
47842 The lack of cross-lane shuffling in some instructions
47843 might prevent a single insn shuffle. */
47845 dfinal.testing_p = true;
47846 /* If expand_vec_perm_interleave3 can expand this into
47847 a 3 insn sequence, give up and let it be expanded as
47848 3 insn sequence. While that is one insn longer,
47849 it doesn't need a memory operand and in the common
47850 case that both interleave low and high permutations
47851 with the same operands are adjacent needs 4 insns
47852 for both after CSE. */
47853 if (expand_vec_perm_interleave3 (&dfinal))
47859 /* Examine from whence the elements come. */
47861 for (i = 0; i < nelt; ++i)
47862 contents |= HOST_WIDE_INT_1U << d->perm[i];
47864 memset (remap, 0xff, sizeof (remap));
47867 if (GET_MODE_SIZE (d->vmode) == 16)
47869 unsigned HOST_WIDE_INT h1, h2, h3, h4;
47871 /* Split the two input vectors into 4 halves. */
47872 h1 = (HOST_WIDE_INT_1U << nelt2) - 1;
47877 /* If the elements from the low halves use interleave low, and similarly
47878 for interleave high. If the elements are from mis-matched halves, we
47879 can use shufps for V4SF/V4SI or do a DImode shuffle. */
47880 if ((contents & (h1 | h3)) == contents)
47883 for (i = 0; i < nelt2; ++i)
47886 remap[i + nelt] = i * 2 + 1;
47887 dremap.perm[i * 2] = i;
47888 dremap.perm[i * 2 + 1] = i + nelt;
47890 if (!TARGET_SSE2 && d->vmode == V4SImode)
47891 dremap.vmode = V4SFmode;
47893 else if ((contents & (h2 | h4)) == contents)
47896 for (i = 0; i < nelt2; ++i)
47898 remap[i + nelt2] = i * 2;
47899 remap[i + nelt + nelt2] = i * 2 + 1;
47900 dremap.perm[i * 2] = i + nelt2;
47901 dremap.perm[i * 2 + 1] = i + nelt + nelt2;
47903 if (!TARGET_SSE2 && d->vmode == V4SImode)
47904 dremap.vmode = V4SFmode;
47906 else if ((contents & (h1 | h4)) == contents)
47909 for (i = 0; i < nelt2; ++i)
47912 remap[i + nelt + nelt2] = i + nelt2;
47913 dremap.perm[i] = i;
47914 dremap.perm[i + nelt2] = i + nelt + nelt2;
47919 dremap.vmode = V2DImode;
47921 dremap.perm[0] = 0;
47922 dremap.perm[1] = 3;
47925 else if ((contents & (h2 | h3)) == contents)
47928 for (i = 0; i < nelt2; ++i)
47930 remap[i + nelt2] = i;
47931 remap[i + nelt] = i + nelt2;
47932 dremap.perm[i] = i + nelt2;
47933 dremap.perm[i + nelt2] = i + nelt;
47938 dremap.vmode = V2DImode;
47940 dremap.perm[0] = 1;
47941 dremap.perm[1] = 2;
47949 unsigned int nelt4 = nelt / 4, nzcnt = 0;
47950 unsigned HOST_WIDE_INT q[8];
47951 unsigned int nonzero_halves[4];
47953 /* Split the two input vectors into 8 quarters. */
47954 q[0] = (HOST_WIDE_INT_1U << nelt4) - 1;
47955 for (i = 1; i < 8; ++i)
47956 q[i] = q[0] << (nelt4 * i);
47957 for (i = 0; i < 4; ++i)
47958 if (((q[2 * i] | q[2 * i + 1]) & contents) != 0)
47960 nonzero_halves[nzcnt] = i;
47966 gcc_assert (d->one_operand_p);
47967 nonzero_halves[1] = nonzero_halves[0];
47968 same_halves = true;
47970 else if (d->one_operand_p)
47972 gcc_assert (nonzero_halves[0] == 0);
47973 gcc_assert (nonzero_halves[1] == 1);
47978 if (d->perm[0] / nelt2 == nonzero_halves[1])
47980 /* Attempt to increase the likelihood that dfinal
47981 shuffle will be intra-lane. */
47982 char tmph = nonzero_halves[0];
47983 nonzero_halves[0] = nonzero_halves[1];
47984 nonzero_halves[1] = tmph;
47987 /* vperm2f128 or vperm2i128. */
47988 for (i = 0; i < nelt2; ++i)
47990 remap[i + nonzero_halves[1] * nelt2] = i + nelt2;
47991 remap[i + nonzero_halves[0] * nelt2] = i;
47992 dremap.perm[i + nelt2] = i + nonzero_halves[1] * nelt2;
47993 dremap.perm[i] = i + nonzero_halves[0] * nelt2;
47996 if (d->vmode != V8SFmode
47997 && d->vmode != V4DFmode
47998 && d->vmode != V8SImode)
48000 dremap.vmode = V8SImode;
48002 for (i = 0; i < 4; ++i)
48004 dremap.perm[i] = i + nonzero_halves[0] * 4;
48005 dremap.perm[i + 4] = i + nonzero_halves[1] * 4;
48009 else if (d->one_operand_p)
48011 else if (TARGET_AVX2
48012 && (contents & (q[0] | q[2] | q[4] | q[6])) == contents)
48015 for (i = 0; i < nelt4; ++i)
48018 remap[i + nelt] = i * 2 + 1;
48019 remap[i + nelt2] = i * 2 + nelt2;
48020 remap[i + nelt + nelt2] = i * 2 + nelt2 + 1;
48021 dremap.perm[i * 2] = i;
48022 dremap.perm[i * 2 + 1] = i + nelt;
48023 dremap.perm[i * 2 + nelt2] = i + nelt2;
48024 dremap.perm[i * 2 + nelt2 + 1] = i + nelt + nelt2;
48027 else if (TARGET_AVX2
48028 && (contents & (q[1] | q[3] | q[5] | q[7])) == contents)
48031 for (i = 0; i < nelt4; ++i)
48033 remap[i + nelt4] = i * 2;
48034 remap[i + nelt + nelt4] = i * 2 + 1;
48035 remap[i + nelt2 + nelt4] = i * 2 + nelt2;
48036 remap[i + nelt + nelt2 + nelt4] = i * 2 + nelt2 + 1;
48037 dremap.perm[i * 2] = i + nelt4;
48038 dremap.perm[i * 2 + 1] = i + nelt + nelt4;
48039 dremap.perm[i * 2 + nelt2] = i + nelt2 + nelt4;
48040 dremap.perm[i * 2 + nelt2 + 1] = i + nelt + nelt2 + nelt4;
48047 /* Use the remapping array set up above to move the elements from their
48048 swizzled locations into their final destinations. */
48050 for (i = 0; i < nelt; ++i)
48052 unsigned e = remap[d->perm[i]];
48053 gcc_assert (e < nelt);
48054 /* If same_halves is true, both halves of the remapped vector are the
48055 same. Avoid cross-lane accesses if possible. */
48056 if (same_halves && i >= nelt2)
48058 gcc_assert (e < nelt2);
48059 dfinal.perm[i] = e + nelt2;
48062 dfinal.perm[i] = e;
48066 dremap.target = gen_reg_rtx (dremap.vmode);
48067 dfinal.op0 = gen_lowpart (dfinal.vmode, dremap.target);
48069 dfinal.op1 = dfinal.op0;
48070 dfinal.one_operand_p = true;
48072 /* Test if the final remap can be done with a single insn. For V4SFmode or
48073 V4SImode this *will* succeed. For V8HImode or V16QImode it may not. */
48075 ok = expand_vec_perm_1 (&dfinal);
48076 seq = get_insns ();
48085 if (dremap.vmode != dfinal.vmode)
48087 dremap.op0 = gen_lowpart (dremap.vmode, dremap.op0);
48088 dremap.op1 = gen_lowpart (dremap.vmode, dremap.op1);
48091 ok = expand_vec_perm_1 (&dremap);
48098 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
48099 a single vector cross-lane permutation into vpermq followed
48100 by any of the single insn permutations. */
48103 expand_vec_perm_vpermq_perm_1 (struct expand_vec_perm_d *d)
48105 struct expand_vec_perm_d dremap, dfinal;
48106 unsigned i, j, nelt = d->nelt, nelt2 = nelt / 2, nelt4 = nelt / 4;
48107 unsigned contents[2];
48111 && (d->vmode == V32QImode || d->vmode == V16HImode)
48112 && d->one_operand_p))
48117 for (i = 0; i < nelt2; ++i)
48119 contents[0] |= 1u << (d->perm[i] / nelt4);
48120 contents[1] |= 1u << (d->perm[i + nelt2] / nelt4);
48123 for (i = 0; i < 2; ++i)
48125 unsigned int cnt = 0;
48126 for (j = 0; j < 4; ++j)
48127 if ((contents[i] & (1u << j)) != 0 && ++cnt > 2)
48135 dremap.vmode = V4DImode;
48137 dremap.target = gen_reg_rtx (V4DImode);
48138 dremap.op0 = gen_lowpart (V4DImode, d->op0);
48139 dremap.op1 = dremap.op0;
48140 dremap.one_operand_p = true;
48141 for (i = 0; i < 2; ++i)
48143 unsigned int cnt = 0;
48144 for (j = 0; j < 4; ++j)
48145 if ((contents[i] & (1u << j)) != 0)
48146 dremap.perm[2 * i + cnt++] = j;
48147 for (; cnt < 2; ++cnt)
48148 dremap.perm[2 * i + cnt] = 0;
48152 dfinal.op0 = gen_lowpart (dfinal.vmode, dremap.target);
48153 dfinal.op1 = dfinal.op0;
48154 dfinal.one_operand_p = true;
48155 for (i = 0, j = 0; i < nelt; ++i)
48159 dfinal.perm[i] = (d->perm[i] & (nelt4 - 1)) | (j ? nelt2 : 0);
48160 if ((d->perm[i] / nelt4) == dremap.perm[j])
48162 else if ((d->perm[i] / nelt4) == dremap.perm[j + 1])
48163 dfinal.perm[i] |= nelt4;
48165 gcc_unreachable ();
48168 ok = expand_vec_perm_1 (&dremap);
48171 ok = expand_vec_perm_1 (&dfinal);
48177 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to expand
48178 a vector permutation using two instructions, vperm2f128 resp.
48179 vperm2i128 followed by any single in-lane permutation. */
48182 expand_vec_perm_vperm2f128 (struct expand_vec_perm_d *d)
48184 struct expand_vec_perm_d dfirst, dsecond;
48185 unsigned i, j, nelt = d->nelt, nelt2 = nelt / 2, perm;
48189 || GET_MODE_SIZE (d->vmode) != 32
48190 || (d->vmode != V8SFmode && d->vmode != V4DFmode && !TARGET_AVX2))
48194 dsecond.one_operand_p = false;
48195 dsecond.testing_p = true;
48197 /* ((perm << 2)|perm) & 0x33 is the vperm2[fi]128
48198 immediate. For perm < 16 the second permutation uses
48199 d->op0 as first operand, for perm >= 16 it uses d->op1
48200 as first operand. The second operand is the result of
48202 for (perm = 0; perm < 32; perm++)
48204 /* Ignore permutations which do not move anything cross-lane. */
48207 /* The second shuffle for e.g. V4DFmode has
48208 0123 and ABCD operands.
48209 Ignore AB23, as 23 is already in the second lane
48210 of the first operand. */
48211 if ((perm & 0xc) == (1 << 2)) continue;
48212 /* And 01CD, as 01 is in the first lane of the first
48214 if ((perm & 3) == 0) continue;
48215 /* And 4567, as then the vperm2[fi]128 doesn't change
48216 anything on the original 4567 second operand. */
48217 if ((perm & 0xf) == ((3 << 2) | 2)) continue;
48221 /* The second shuffle for e.g. V4DFmode has
48222 4567 and ABCD operands.
48223 Ignore AB67, as 67 is already in the second lane
48224 of the first operand. */
48225 if ((perm & 0xc) == (3 << 2)) continue;
48226 /* And 45CD, as 45 is in the first lane of the first
48228 if ((perm & 3) == 2) continue;
48229 /* And 0123, as then the vperm2[fi]128 doesn't change
48230 anything on the original 0123 first operand. */
48231 if ((perm & 0xf) == (1 << 2)) continue;
48234 for (i = 0; i < nelt; i++)
48236 j = d->perm[i] / nelt2;
48237 if (j == ((perm >> (2 * (i >= nelt2))) & 3))
48238 dsecond.perm[i] = nelt + (i & nelt2) + (d->perm[i] & (nelt2 - 1));
48239 else if (j == (unsigned) (i >= nelt2) + 2 * (perm >= 16))
48240 dsecond.perm[i] = d->perm[i] & (nelt - 1);
48248 ok = expand_vec_perm_1 (&dsecond);
48259 /* Found a usable second shuffle. dfirst will be
48260 vperm2f128 on d->op0 and d->op1. */
48261 dsecond.testing_p = false;
48263 dfirst.target = gen_reg_rtx (d->vmode);
48264 for (i = 0; i < nelt; i++)
48265 dfirst.perm[i] = (i & (nelt2 - 1))
48266 + ((perm >> (2 * (i >= nelt2))) & 3) * nelt2;
48268 canonicalize_perm (&dfirst);
48269 ok = expand_vec_perm_1 (&dfirst);
48272 /* And dsecond is some single insn shuffle, taking
48273 d->op0 and result of vperm2f128 (if perm < 16) or
48274 d->op1 and result of vperm2f128 (otherwise). */
48276 dsecond.op0 = dsecond.op1;
48277 dsecond.op1 = dfirst.target;
48279 ok = expand_vec_perm_1 (&dsecond);
48285 /* For one operand, the only useful vperm2f128 permutation is 0x01
48287 if (d->one_operand_p)
48294 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
48295 a two vector permutation using 2 intra-lane interleave insns
48296 and cross-lane shuffle for 32-byte vectors. */
48299 expand_vec_perm_interleave3 (struct expand_vec_perm_d *d)
48302 rtx (*gen) (rtx, rtx, rtx);
48304 if (d->one_operand_p)
48306 if (TARGET_AVX2 && GET_MODE_SIZE (d->vmode) == 32)
48308 else if (TARGET_AVX && (d->vmode == V8SFmode || d->vmode == V4DFmode))
48314 if (d->perm[0] != 0 && d->perm[0] != nelt / 2)
48316 for (i = 0; i < nelt; i += 2)
48317 if (d->perm[i] != d->perm[0] + i / 2
48318 || d->perm[i + 1] != d->perm[0] + i / 2 + nelt)
48328 gen = gen_vec_interleave_highv32qi;
48330 gen = gen_vec_interleave_lowv32qi;
48334 gen = gen_vec_interleave_highv16hi;
48336 gen = gen_vec_interleave_lowv16hi;
48340 gen = gen_vec_interleave_highv8si;
48342 gen = gen_vec_interleave_lowv8si;
48346 gen = gen_vec_interleave_highv4di;
48348 gen = gen_vec_interleave_lowv4di;
48352 gen = gen_vec_interleave_highv8sf;
48354 gen = gen_vec_interleave_lowv8sf;
48358 gen = gen_vec_interleave_highv4df;
48360 gen = gen_vec_interleave_lowv4df;
48363 gcc_unreachable ();
48366 emit_insn (gen (d->target, d->op0, d->op1));
48370 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement
48371 a single vector permutation using a single intra-lane vector
48372 permutation, vperm2f128 swapping the lanes and vblend* insn blending
48373 the non-swapped and swapped vectors together. */
48376 expand_vec_perm_vperm2f128_vblend (struct expand_vec_perm_d *d)
48378 struct expand_vec_perm_d dfirst, dsecond;
48379 unsigned i, j, msk, nelt = d->nelt, nelt2 = nelt / 2;
48382 rtx (*blend) (rtx, rtx, rtx, rtx) = NULL;
48386 || (d->vmode != V8SFmode && d->vmode != V4DFmode)
48387 || !d->one_operand_p)
48391 for (i = 0; i < nelt; i++)
48392 dfirst.perm[i] = 0xff;
48393 for (i = 0, msk = 0; i < nelt; i++)
48395 j = (d->perm[i] & nelt2) ? i | nelt2 : i & ~nelt2;
48396 if (dfirst.perm[j] != 0xff && dfirst.perm[j] != d->perm[i])
48398 dfirst.perm[j] = d->perm[i];
48402 for (i = 0; i < nelt; i++)
48403 if (dfirst.perm[i] == 0xff)
48404 dfirst.perm[i] = i;
48407 dfirst.target = gen_reg_rtx (dfirst.vmode);
48410 ok = expand_vec_perm_1 (&dfirst);
48411 seq = get_insns ();
48423 dsecond.op0 = dfirst.target;
48424 dsecond.op1 = dfirst.target;
48425 dsecond.one_operand_p = true;
48426 dsecond.target = gen_reg_rtx (dsecond.vmode);
48427 for (i = 0; i < nelt; i++)
48428 dsecond.perm[i] = i ^ nelt2;
48430 ok = expand_vec_perm_1 (&dsecond);
48433 blend = d->vmode == V8SFmode ? gen_avx_blendps256 : gen_avx_blendpd256;
48434 emit_insn (blend (d->target, dfirst.target, dsecond.target, GEN_INT (msk)));
48438 /* A subroutine of ix86_expand_vec_perm_builtin_1. Implement a V4DF
48439 permutation using two vperm2f128, followed by a vshufpd insn blending
48440 the two vectors together. */
48443 expand_vec_perm_2vperm2f128_vshuf (struct expand_vec_perm_d *d)
48445 struct expand_vec_perm_d dfirst, dsecond, dthird;
48448 if (!TARGET_AVX || (d->vmode != V4DFmode))
48458 dfirst.perm[0] = (d->perm[0] & ~1);
48459 dfirst.perm[1] = (d->perm[0] & ~1) + 1;
48460 dfirst.perm[2] = (d->perm[2] & ~1);
48461 dfirst.perm[3] = (d->perm[2] & ~1) + 1;
48462 dsecond.perm[0] = (d->perm[1] & ~1);
48463 dsecond.perm[1] = (d->perm[1] & ~1) + 1;
48464 dsecond.perm[2] = (d->perm[3] & ~1);
48465 dsecond.perm[3] = (d->perm[3] & ~1) + 1;
48466 dthird.perm[0] = (d->perm[0] % 2);
48467 dthird.perm[1] = (d->perm[1] % 2) + 4;
48468 dthird.perm[2] = (d->perm[2] % 2) + 2;
48469 dthird.perm[3] = (d->perm[3] % 2) + 6;
48471 dfirst.target = gen_reg_rtx (dfirst.vmode);
48472 dsecond.target = gen_reg_rtx (dsecond.vmode);
48473 dthird.op0 = dfirst.target;
48474 dthird.op1 = dsecond.target;
48475 dthird.one_operand_p = false;
48477 canonicalize_perm (&dfirst);
48478 canonicalize_perm (&dsecond);
48480 ok = expand_vec_perm_1 (&dfirst)
48481 && expand_vec_perm_1 (&dsecond)
48482 && expand_vec_perm_1 (&dthird);
48489 /* A subroutine of expand_vec_perm_even_odd_1. Implement the double-word
48490 permutation with two pshufb insns and an ior. We should have already
48491 failed all two instruction sequences. */
48494 expand_vec_perm_pshufb2 (struct expand_vec_perm_d *d)
48496 rtx rperm[2][16], vperm, l, h, op, m128;
48497 unsigned int i, nelt, eltsz;
48499 if (!TARGET_SSSE3 || GET_MODE_SIZE (d->vmode) != 16)
48501 gcc_assert (!d->one_operand_p);
48507 eltsz = GET_MODE_SIZE (GET_MODE_INNER (d->vmode));
48509 /* Generate two permutation masks. If the required element is within
48510 the given vector it is shuffled into the proper lane. If the required
48511 element is in the other vector, force a zero into the lane by setting
48512 bit 7 in the permutation mask. */
48513 m128 = GEN_INT (-128);
48514 for (i = 0; i < nelt; ++i)
48516 unsigned j, e = d->perm[i];
48517 unsigned which = (e >= nelt);
48521 for (j = 0; j < eltsz; ++j)
48523 rperm[which][i*eltsz + j] = GEN_INT (e*eltsz + j);
48524 rperm[1-which][i*eltsz + j] = m128;
48528 vperm = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, rperm[0]));
48529 vperm = force_reg (V16QImode, vperm);
48531 l = gen_reg_rtx (V16QImode);
48532 op = gen_lowpart (V16QImode, d->op0);
48533 emit_insn (gen_ssse3_pshufbv16qi3 (l, op, vperm));
48535 vperm = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, rperm[1]));
48536 vperm = force_reg (V16QImode, vperm);
48538 h = gen_reg_rtx (V16QImode);
48539 op = gen_lowpart (V16QImode, d->op1);
48540 emit_insn (gen_ssse3_pshufbv16qi3 (h, op, vperm));
48543 if (d->vmode != V16QImode)
48544 op = gen_reg_rtx (V16QImode);
48545 emit_insn (gen_iorv16qi3 (op, l, h));
48546 if (op != d->target)
48547 emit_move_insn (d->target, gen_lowpart (d->vmode, op));
48552 /* Implement arbitrary permutation of one V32QImode and V16QImode operand
48553 with two vpshufb insns, vpermq and vpor. We should have already failed
48554 all two or three instruction sequences. */
48557 expand_vec_perm_vpshufb2_vpermq (struct expand_vec_perm_d *d)
48559 rtx rperm[2][32], vperm, l, h, hp, op, m128;
48560 unsigned int i, nelt, eltsz;
48563 || !d->one_operand_p
48564 || (d->vmode != V32QImode && d->vmode != V16HImode))
48571 eltsz = GET_MODE_SIZE (GET_MODE_INNER (d->vmode));
48573 /* Generate two permutation masks. If the required element is within
48574 the same lane, it is shuffled in. If the required element from the
48575 other lane, force a zero by setting bit 7 in the permutation mask.
48576 In the other mask the mask has non-negative elements if element
48577 is requested from the other lane, but also moved to the other lane,
48578 so that the result of vpshufb can have the two V2TImode halves
48580 m128 = GEN_INT (-128);
48581 for (i = 0; i < nelt; ++i)
48583 unsigned j, e = d->perm[i] & (nelt / 2 - 1);
48584 unsigned which = ((d->perm[i] ^ i) & (nelt / 2)) * eltsz;
48586 for (j = 0; j < eltsz; ++j)
48588 rperm[!!which][(i * eltsz + j) ^ which] = GEN_INT (e * eltsz + j);
48589 rperm[!which][(i * eltsz + j) ^ (which ^ 16)] = m128;
48593 vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[1]));
48594 vperm = force_reg (V32QImode, vperm);
48596 h = gen_reg_rtx (V32QImode);
48597 op = gen_lowpart (V32QImode, d->op0);
48598 emit_insn (gen_avx2_pshufbv32qi3 (h, op, vperm));
48600 /* Swap the 128-byte lanes of h into hp. */
48601 hp = gen_reg_rtx (V4DImode);
48602 op = gen_lowpart (V4DImode, h);
48603 emit_insn (gen_avx2_permv4di_1 (hp, op, const2_rtx, GEN_INT (3), const0_rtx,
48606 vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[0]));
48607 vperm = force_reg (V32QImode, vperm);
48609 l = gen_reg_rtx (V32QImode);
48610 op = gen_lowpart (V32QImode, d->op0);
48611 emit_insn (gen_avx2_pshufbv32qi3 (l, op, vperm));
48614 if (d->vmode != V32QImode)
48615 op = gen_reg_rtx (V32QImode);
48616 emit_insn (gen_iorv32qi3 (op, l, gen_lowpart (V32QImode, hp)));
48617 if (op != d->target)
48618 emit_move_insn (d->target, gen_lowpart (d->vmode, op));
48623 /* A subroutine of expand_vec_perm_even_odd_1. Implement extract-even
48624 and extract-odd permutations of two V32QImode and V16QImode operand
48625 with two vpshufb insns, vpor and vpermq. We should have already
48626 failed all two or three instruction sequences. */
48629 expand_vec_perm_vpshufb2_vpermq_even_odd (struct expand_vec_perm_d *d)
48631 rtx rperm[2][32], vperm, l, h, ior, op, m128;
48632 unsigned int i, nelt, eltsz;
48635 || d->one_operand_p
48636 || (d->vmode != V32QImode && d->vmode != V16HImode))
48639 for (i = 0; i < d->nelt; ++i)
48640 if ((d->perm[i] ^ (i * 2)) & (3 * d->nelt / 2))
48647 eltsz = GET_MODE_SIZE (GET_MODE_INNER (d->vmode));
48649 /* Generate two permutation masks. In the first permutation mask
48650 the first quarter will contain indexes for the first half
48651 of the op0, the second quarter will contain bit 7 set, third quarter
48652 will contain indexes for the second half of the op0 and the
48653 last quarter bit 7 set. In the second permutation mask
48654 the first quarter will contain bit 7 set, the second quarter
48655 indexes for the first half of the op1, the third quarter bit 7 set
48656 and last quarter indexes for the second half of the op1.
48657 I.e. the first mask e.g. for V32QImode extract even will be:
48658 0, 2, ..., 0xe, -128, ..., -128, 0, 2, ..., 0xe, -128, ..., -128
48659 (all values masked with 0xf except for -128) and second mask
48660 for extract even will be
48661 -128, ..., -128, 0, 2, ..., 0xe, -128, ..., -128, 0, 2, ..., 0xe. */
48662 m128 = GEN_INT (-128);
48663 for (i = 0; i < nelt; ++i)
48665 unsigned j, e = d->perm[i] & (nelt / 2 - 1);
48666 unsigned which = d->perm[i] >= nelt;
48667 unsigned xorv = (i >= nelt / 4 && i < 3 * nelt / 4) ? 24 : 0;
48669 for (j = 0; j < eltsz; ++j)
48671 rperm[which][(i * eltsz + j) ^ xorv] = GEN_INT (e * eltsz + j);
48672 rperm[1 - which][(i * eltsz + j) ^ xorv] = m128;
48676 vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[0]));
48677 vperm = force_reg (V32QImode, vperm);
48679 l = gen_reg_rtx (V32QImode);
48680 op = gen_lowpart (V32QImode, d->op0);
48681 emit_insn (gen_avx2_pshufbv32qi3 (l, op, vperm));
48683 vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[1]));
48684 vperm = force_reg (V32QImode, vperm);
48686 h = gen_reg_rtx (V32QImode);
48687 op = gen_lowpart (V32QImode, d->op1);
48688 emit_insn (gen_avx2_pshufbv32qi3 (h, op, vperm));
48690 ior = gen_reg_rtx (V32QImode);
48691 emit_insn (gen_iorv32qi3 (ior, l, h));
48693 /* Permute the V4DImode quarters using { 0, 2, 1, 3 } permutation. */
48694 op = gen_reg_rtx (V4DImode);
48695 ior = gen_lowpart (V4DImode, ior);
48696 emit_insn (gen_avx2_permv4di_1 (op, ior, const0_rtx, const2_rtx,
48697 const1_rtx, GEN_INT (3)));
48698 emit_move_insn (d->target, gen_lowpart (d->vmode, op));
48703 /* A subroutine of expand_vec_perm_even_odd_1. Implement extract-even
48704 and extract-odd permutations of two V16QI, V8HI, V16HI or V32QI operands
48705 with two "and" and "pack" or two "shift" and "pack" insns. We should
48706 have already failed all two instruction sequences. */
48709 expand_vec_perm_even_odd_pack (struct expand_vec_perm_d *d)
48711 rtx op, dop0, dop1, t, rperm[16];
48712 unsigned i, odd, c, s, nelt = d->nelt;
48713 bool end_perm = false;
48714 machine_mode half_mode;
48715 rtx (*gen_and) (rtx, rtx, rtx);
48716 rtx (*gen_pack) (rtx, rtx, rtx);
48717 rtx (*gen_shift) (rtx, rtx, rtx);
48719 if (d->one_operand_p)
48725 /* Required for "pack". */
48726 if (!TARGET_SSE4_1)
48730 half_mode = V4SImode;
48731 gen_and = gen_andv4si3;
48732 gen_pack = gen_sse4_1_packusdw;
48733 gen_shift = gen_lshrv4si3;
48736 /* No check as all instructions are SSE2. */
48739 half_mode = V8HImode;
48740 gen_and = gen_andv8hi3;
48741 gen_pack = gen_sse2_packuswb;
48742 gen_shift = gen_lshrv8hi3;
48749 half_mode = V8SImode;
48750 gen_and = gen_andv8si3;
48751 gen_pack = gen_avx2_packusdw;
48752 gen_shift = gen_lshrv8si3;
48760 half_mode = V16HImode;
48761 gen_and = gen_andv16hi3;
48762 gen_pack = gen_avx2_packuswb;
48763 gen_shift = gen_lshrv16hi3;
48767 /* Only V8HI, V16QI, V16HI and V32QI modes are more profitable than
48768 general shuffles. */
48772 /* Check that permutation is even or odd. */
48777 for (i = 1; i < nelt; ++i)
48778 if (d->perm[i] != 2 * i + odd)
48784 dop0 = gen_reg_rtx (half_mode);
48785 dop1 = gen_reg_rtx (half_mode);
48788 for (i = 0; i < nelt / 2; i++)
48789 rperm[i] = GEN_INT (c);
48790 t = gen_rtx_CONST_VECTOR (half_mode, gen_rtvec_v (nelt / 2, rperm));
48791 t = force_reg (half_mode, t);
48792 emit_insn (gen_and (dop0, t, gen_lowpart (half_mode, d->op0)));
48793 emit_insn (gen_and (dop1, t, gen_lowpart (half_mode, d->op1)));
48797 emit_insn (gen_shift (dop0,
48798 gen_lowpart (half_mode, d->op0),
48800 emit_insn (gen_shift (dop1,
48801 gen_lowpart (half_mode, d->op1),
48804 /* In AVX2 for 256 bit case we need to permute pack result. */
48805 if (TARGET_AVX2 && end_perm)
48807 op = gen_reg_rtx (d->vmode);
48808 t = gen_reg_rtx (V4DImode);
48809 emit_insn (gen_pack (op, dop0, dop1));
48810 emit_insn (gen_avx2_permv4di_1 (t,
48811 gen_lowpart (V4DImode, op),
48816 emit_move_insn (d->target, gen_lowpart (d->vmode, t));
48819 emit_insn (gen_pack (d->target, dop0, dop1));
48824 /* A subroutine of ix86_expand_vec_perm_builtin_1. Implement extract-even
48825 and extract-odd permutations. */
48828 expand_vec_perm_even_odd_1 (struct expand_vec_perm_d *d, unsigned odd)
48830 rtx t1, t2, t3, t4, t5;
48837 t1 = gen_reg_rtx (V4DFmode);
48838 t2 = gen_reg_rtx (V4DFmode);
48840 /* Shuffle the lanes around into { 0 1 4 5 } and { 2 3 6 7 }. */
48841 emit_insn (gen_avx_vperm2f128v4df3 (t1, d->op0, d->op1, GEN_INT (0x20)));
48842 emit_insn (gen_avx_vperm2f128v4df3 (t2, d->op0, d->op1, GEN_INT (0x31)));
48844 /* Now an unpck[lh]pd will produce the result required. */
48846 t3 = gen_avx_unpckhpd256 (d->target, t1, t2);
48848 t3 = gen_avx_unpcklpd256 (d->target, t1, t2);
48854 int mask = odd ? 0xdd : 0x88;
48858 t1 = gen_reg_rtx (V8SFmode);
48859 t2 = gen_reg_rtx (V8SFmode);
48860 t3 = gen_reg_rtx (V8SFmode);
48862 /* Shuffle within the 128-bit lanes to produce:
48863 { 0 2 8 a 4 6 c e } | { 1 3 9 b 5 7 d f }. */
48864 emit_insn (gen_avx_shufps256 (t1, d->op0, d->op1,
48867 /* Shuffle the lanes around to produce:
48868 { 4 6 c e 0 2 8 a } and { 5 7 d f 1 3 9 b }. */
48869 emit_insn (gen_avx_vperm2f128v8sf3 (t2, t1, t1,
48872 /* Shuffle within the 128-bit lanes to produce:
48873 { 0 2 4 6 4 6 0 2 } | { 1 3 5 7 5 7 1 3 }. */
48874 emit_insn (gen_avx_shufps256 (t3, t1, t2, GEN_INT (0x44)));
48876 /* Shuffle within the 128-bit lanes to produce:
48877 { 8 a c e c e 8 a } | { 9 b d f d f 9 b }. */
48878 emit_insn (gen_avx_shufps256 (t2, t1, t2, GEN_INT (0xee)));
48880 /* Shuffle the lanes around to produce:
48881 { 0 2 4 6 8 a c e } | { 1 3 5 7 9 b d f }. */
48882 emit_insn (gen_avx_vperm2f128v8sf3 (d->target, t3, t2,
48891 /* These are always directly implementable by expand_vec_perm_1. */
48892 gcc_unreachable ();
48896 return expand_vec_perm_even_odd_pack (d);
48897 else if (TARGET_SSSE3 && !TARGET_SLOW_PSHUFB)
48898 return expand_vec_perm_pshufb2 (d);
48903 /* We need 2*log2(N)-1 operations to achieve odd/even
48904 with interleave. */
48905 t1 = gen_reg_rtx (V8HImode);
48906 t2 = gen_reg_rtx (V8HImode);
48907 emit_insn (gen_vec_interleave_highv8hi (t1, d->op0, d->op1));
48908 emit_insn (gen_vec_interleave_lowv8hi (d->target, d->op0, d->op1));
48909 emit_insn (gen_vec_interleave_highv8hi (t2, d->target, t1));
48910 emit_insn (gen_vec_interleave_lowv8hi (d->target, d->target, t1));
48912 t3 = gen_vec_interleave_highv8hi (d->target, d->target, t2);
48914 t3 = gen_vec_interleave_lowv8hi (d->target, d->target, t2);
48920 return expand_vec_perm_even_odd_pack (d);
48924 return expand_vec_perm_even_odd_pack (d);
48929 struct expand_vec_perm_d d_copy = *d;
48930 d_copy.vmode = V4DFmode;
48932 d_copy.target = gen_lowpart (V4DFmode, d->target);
48934 d_copy.target = gen_reg_rtx (V4DFmode);
48935 d_copy.op0 = gen_lowpart (V4DFmode, d->op0);
48936 d_copy.op1 = gen_lowpart (V4DFmode, d->op1);
48937 if (expand_vec_perm_even_odd_1 (&d_copy, odd))
48940 emit_move_insn (d->target,
48941 gen_lowpart (V4DImode, d_copy.target));
48950 t1 = gen_reg_rtx (V4DImode);
48951 t2 = gen_reg_rtx (V4DImode);
48953 /* Shuffle the lanes around into { 0 1 4 5 } and { 2 3 6 7 }. */
48954 emit_insn (gen_avx2_permv2ti (t1, d->op0, d->op1, GEN_INT (0x20)));
48955 emit_insn (gen_avx2_permv2ti (t2, d->op0, d->op1, GEN_INT (0x31)));
48957 /* Now an vpunpck[lh]qdq will produce the result required. */
48959 t3 = gen_avx2_interleave_highv4di (d->target, t1, t2);
48961 t3 = gen_avx2_interleave_lowv4di (d->target, t1, t2);
48968 struct expand_vec_perm_d d_copy = *d;
48969 d_copy.vmode = V8SFmode;
48971 d_copy.target = gen_lowpart (V8SFmode, d->target);
48973 d_copy.target = gen_reg_rtx (V8SFmode);
48974 d_copy.op0 = gen_lowpart (V8SFmode, d->op0);
48975 d_copy.op1 = gen_lowpart (V8SFmode, d->op1);
48976 if (expand_vec_perm_even_odd_1 (&d_copy, odd))
48979 emit_move_insn (d->target,
48980 gen_lowpart (V8SImode, d_copy.target));
48989 t1 = gen_reg_rtx (V8SImode);
48990 t2 = gen_reg_rtx (V8SImode);
48991 t3 = gen_reg_rtx (V4DImode);
48992 t4 = gen_reg_rtx (V4DImode);
48993 t5 = gen_reg_rtx (V4DImode);
48995 /* Shuffle the lanes around into
48996 { 0 1 2 3 8 9 a b } and { 4 5 6 7 c d e f }. */
48997 emit_insn (gen_avx2_permv2ti (t3, gen_lowpart (V4DImode, d->op0),
48998 gen_lowpart (V4DImode, d->op1),
49000 emit_insn (gen_avx2_permv2ti (t4, gen_lowpart (V4DImode, d->op0),
49001 gen_lowpart (V4DImode, d->op1),
49004 /* Swap the 2nd and 3rd position in each lane into
49005 { 0 2 1 3 8 a 9 b } and { 4 6 5 7 c e d f }. */
49006 emit_insn (gen_avx2_pshufdv3 (t1, gen_lowpart (V8SImode, t3),
49007 GEN_INT (2 * 4 + 1 * 16 + 3 * 64)));
49008 emit_insn (gen_avx2_pshufdv3 (t2, gen_lowpart (V8SImode, t4),
49009 GEN_INT (2 * 4 + 1 * 16 + 3 * 64)));
49011 /* Now an vpunpck[lh]qdq will produce
49012 { 0 2 4 6 8 a c e } resp. { 1 3 5 7 9 b d f }. */
49014 t3 = gen_avx2_interleave_highv4di (t5, gen_lowpart (V4DImode, t1),
49015 gen_lowpart (V4DImode, t2));
49017 t3 = gen_avx2_interleave_lowv4di (t5, gen_lowpart (V4DImode, t1),
49018 gen_lowpart (V4DImode, t2));
49020 emit_move_insn (d->target, gen_lowpart (V8SImode, t5));
49024 gcc_unreachable ();
49030 /* A subroutine of ix86_expand_vec_perm_builtin_1. Pattern match
49031 extract-even and extract-odd permutations. */
49034 expand_vec_perm_even_odd (struct expand_vec_perm_d *d)
49036 unsigned i, odd, nelt = d->nelt;
49039 if (odd != 0 && odd != 1)
49042 for (i = 1; i < nelt; ++i)
49043 if (d->perm[i] != 2 * i + odd)
49046 return expand_vec_perm_even_odd_1 (d, odd);
49049 /* A subroutine of ix86_expand_vec_perm_builtin_1. Implement broadcast
49050 permutations. We assume that expand_vec_perm_1 has already failed. */
49053 expand_vec_perm_broadcast_1 (struct expand_vec_perm_d *d)
49055 unsigned elt = d->perm[0], nelt2 = d->nelt / 2;
49056 machine_mode vmode = d->vmode;
49057 unsigned char perm2[4];
49058 rtx op0 = d->op0, dest;
49065 /* These are special-cased in sse.md so that we can optionally
49066 use the vbroadcast instruction. They expand to two insns
49067 if the input happens to be in a register. */
49068 gcc_unreachable ();
49074 /* These are always implementable using standard shuffle patterns. */
49075 gcc_unreachable ();
49079 /* These can be implemented via interleave. We save one insn by
49080 stopping once we have promoted to V4SImode and then use pshufd. */
49086 rtx (*gen) (rtx, rtx, rtx)
49087 = vmode == V16QImode ? gen_vec_interleave_lowv16qi
49088 : gen_vec_interleave_lowv8hi;
49092 gen = vmode == V16QImode ? gen_vec_interleave_highv16qi
49093 : gen_vec_interleave_highv8hi;
49098 dest = gen_reg_rtx (vmode);
49099 emit_insn (gen (dest, op0, op0));
49100 vmode = get_mode_wider_vector (vmode);
49101 op0 = gen_lowpart (vmode, dest);
49103 while (vmode != V4SImode);
49105 memset (perm2, elt, 4);
49106 dest = gen_reg_rtx (V4SImode);
49107 ok = expand_vselect (dest, op0, perm2, 4, d->testing_p);
49110 emit_move_insn (d->target, gen_lowpart (d->vmode, dest));
49118 /* For AVX2 broadcasts of the first element vpbroadcast* or
49119 vpermq should be used by expand_vec_perm_1. */
49120 gcc_assert (!TARGET_AVX2 || d->perm[0]);
49124 gcc_unreachable ();
49128 /* A subroutine of ix86_expand_vec_perm_builtin_1. Pattern match
49129 broadcast permutations. */
49132 expand_vec_perm_broadcast (struct expand_vec_perm_d *d)
49134 unsigned i, elt, nelt = d->nelt;
49136 if (!d->one_operand_p)
49140 for (i = 1; i < nelt; ++i)
49141 if (d->perm[i] != elt)
49144 return expand_vec_perm_broadcast_1 (d);
49147 /* Implement arbitrary permutations of two V64QImode operands
49148 will 2 vpermi2w, 2 vpshufb and one vpor instruction. */
49150 expand_vec_perm_vpermi2_vpshub2 (struct expand_vec_perm_d *d)
49152 if (!TARGET_AVX512BW || !(d->vmode == V64QImode))
49158 struct expand_vec_perm_d ds[2];
49159 rtx rperm[128], vperm, target0, target1;
49160 unsigned int i, nelt;
49161 machine_mode vmode;
49166 for (i = 0; i < 2; i++)
49169 ds[i].vmode = V32HImode;
49171 ds[i].target = gen_reg_rtx (V32HImode);
49172 ds[i].op0 = gen_lowpart (V32HImode, d->op0);
49173 ds[i].op1 = gen_lowpart (V32HImode, d->op1);
49176 /* Prepare permutations such that the first one takes care of
49177 putting the even bytes into the right positions or one higher
49178 positions (ds[0]) and the second one takes care of
49179 putting the odd bytes into the right positions or one below
49182 for (i = 0; i < nelt; i++)
49184 ds[i & 1].perm[i / 2] = d->perm[i] / 2;
49187 rperm[i] = constm1_rtx;
49188 rperm[i + 64] = GEN_INT ((i & 14) + (d->perm[i] & 1));
49192 rperm[i] = GEN_INT ((i & 14) + (d->perm[i] & 1));
49193 rperm[i + 64] = constm1_rtx;
49197 bool ok = expand_vec_perm_1 (&ds[0]);
49199 ds[0].target = gen_lowpart (V64QImode, ds[0].target);
49201 ok = expand_vec_perm_1 (&ds[1]);
49203 ds[1].target = gen_lowpart (V64QImode, ds[1].target);
49205 vperm = gen_rtx_CONST_VECTOR (V64QImode, gen_rtvec_v (64, rperm));
49206 vperm = force_reg (vmode, vperm);
49207 target0 = gen_reg_rtx (V64QImode);
49208 emit_insn (gen_avx512bw_pshufbv64qi3 (target0, ds[0].target, vperm));
49210 vperm = gen_rtx_CONST_VECTOR (V64QImode, gen_rtvec_v (64, rperm + 64));
49211 vperm = force_reg (vmode, vperm);
49212 target1 = gen_reg_rtx (V64QImode);
49213 emit_insn (gen_avx512bw_pshufbv64qi3 (target1, ds[1].target, vperm));
49215 emit_insn (gen_iorv64qi3 (d->target, target0, target1));
49219 /* Implement arbitrary permutation of two V32QImode and V16QImode operands
49220 with 4 vpshufb insns, 2 vpermq and 3 vpor. We should have already failed
49221 all the shorter instruction sequences. */
49224 expand_vec_perm_vpshufb4_vpermq2 (struct expand_vec_perm_d *d)
49226 rtx rperm[4][32], vperm, l[2], h[2], op, m128;
49227 unsigned int i, nelt, eltsz;
49231 || d->one_operand_p
49232 || (d->vmode != V32QImode && d->vmode != V16HImode))
49239 eltsz = GET_MODE_SIZE (GET_MODE_INNER (d->vmode));
49241 /* Generate 4 permutation masks. If the required element is within
49242 the same lane, it is shuffled in. If the required element from the
49243 other lane, force a zero by setting bit 7 in the permutation mask.
49244 In the other mask the mask has non-negative elements if element
49245 is requested from the other lane, but also moved to the other lane,
49246 so that the result of vpshufb can have the two V2TImode halves
49248 m128 = GEN_INT (-128);
49249 for (i = 0; i < 32; ++i)
49251 rperm[0][i] = m128;
49252 rperm[1][i] = m128;
49253 rperm[2][i] = m128;
49254 rperm[3][i] = m128;
49260 for (i = 0; i < nelt; ++i)
49262 unsigned j, e = d->perm[i] & (nelt / 2 - 1);
49263 unsigned xlane = ((d->perm[i] ^ i) & (nelt / 2)) * eltsz;
49264 unsigned int which = ((d->perm[i] & nelt) ? 2 : 0) + (xlane ? 1 : 0);
49266 for (j = 0; j < eltsz; ++j)
49267 rperm[which][(i * eltsz + j) ^ xlane] = GEN_INT (e * eltsz + j);
49268 used[which] = true;
49271 for (i = 0; i < 2; ++i)
49273 if (!used[2 * i + 1])
49278 vperm = gen_rtx_CONST_VECTOR (V32QImode,
49279 gen_rtvec_v (32, rperm[2 * i + 1]));
49280 vperm = force_reg (V32QImode, vperm);
49281 h[i] = gen_reg_rtx (V32QImode);
49282 op = gen_lowpart (V32QImode, i ? d->op1 : d->op0);
49283 emit_insn (gen_avx2_pshufbv32qi3 (h[i], op, vperm));
49286 /* Swap the 128-byte lanes of h[X]. */
49287 for (i = 0; i < 2; ++i)
49289 if (h[i] == NULL_RTX)
49291 op = gen_reg_rtx (V4DImode);
49292 emit_insn (gen_avx2_permv4di_1 (op, gen_lowpart (V4DImode, h[i]),
49293 const2_rtx, GEN_INT (3), const0_rtx,
49295 h[i] = gen_lowpart (V32QImode, op);
49298 for (i = 0; i < 2; ++i)
49305 vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[2 * i]));
49306 vperm = force_reg (V32QImode, vperm);
49307 l[i] = gen_reg_rtx (V32QImode);
49308 op = gen_lowpart (V32QImode, i ? d->op1 : d->op0);
49309 emit_insn (gen_avx2_pshufbv32qi3 (l[i], op, vperm));
49312 for (i = 0; i < 2; ++i)
49316 op = gen_reg_rtx (V32QImode);
49317 emit_insn (gen_iorv32qi3 (op, l[i], h[i]));
49324 gcc_assert (l[0] && l[1]);
49326 if (d->vmode != V32QImode)
49327 op = gen_reg_rtx (V32QImode);
49328 emit_insn (gen_iorv32qi3 (op, l[0], l[1]));
49329 if (op != d->target)
49330 emit_move_insn (d->target, gen_lowpart (d->vmode, op));
49334 /* The guts of ix86_expand_vec_perm_const, also used by the ok hook.
49335 With all of the interface bits taken care of, perform the expansion
49336 in D and return true on success. */
49339 ix86_expand_vec_perm_const_1 (struct expand_vec_perm_d *d)
49341 /* Try a single instruction expansion. */
49342 if (expand_vec_perm_1 (d))
49345 /* Try sequences of two instructions. */
49347 if (expand_vec_perm_pshuflw_pshufhw (d))
49350 if (expand_vec_perm_palignr (d, false))
49353 if (expand_vec_perm_interleave2 (d))
49356 if (expand_vec_perm_broadcast (d))
49359 if (expand_vec_perm_vpermq_perm_1 (d))
49362 if (expand_vec_perm_vperm2f128 (d))
49365 if (expand_vec_perm_pblendv (d))
49368 /* Try sequences of three instructions. */
49370 if (expand_vec_perm_even_odd_pack (d))
49373 if (expand_vec_perm_2vperm2f128_vshuf (d))
49376 if (expand_vec_perm_pshufb2 (d))
49379 if (expand_vec_perm_interleave3 (d))
49382 if (expand_vec_perm_vperm2f128_vblend (d))
49385 /* Try sequences of four instructions. */
49387 if (expand_vec_perm_vpshufb2_vpermq (d))
49390 if (expand_vec_perm_vpshufb2_vpermq_even_odd (d))
49393 if (expand_vec_perm_vpermi2_vpshub2 (d))
49396 /* ??? Look for narrow permutations whose element orderings would
49397 allow the promotion to a wider mode. */
49399 /* ??? Look for sequences of interleave or a wider permute that place
49400 the data into the correct lanes for a half-vector shuffle like
49401 pshuf[lh]w or vpermilps. */
49403 /* ??? Look for sequences of interleave that produce the desired results.
49404 The combinatorics of punpck[lh] get pretty ugly... */
49406 if (expand_vec_perm_even_odd (d))
49409 /* Even longer sequences. */
49410 if (expand_vec_perm_vpshufb4_vpermq2 (d))
49416 /* If a permutation only uses one operand, make it clear. Returns true
49417 if the permutation references both operands. */
49420 canonicalize_perm (struct expand_vec_perm_d *d)
49422 int i, which, nelt = d->nelt;
49424 for (i = which = 0; i < nelt; ++i)
49425 which |= (d->perm[i] < nelt ? 1 : 2);
49427 d->one_operand_p = true;
49434 if (!rtx_equal_p (d->op0, d->op1))
49436 d->one_operand_p = false;
49439 /* The elements of PERM do not suggest that only the first operand
49440 is used, but both operands are identical. Allow easier matching
49441 of the permutation by folding the permutation into the single
49446 for (i = 0; i < nelt; ++i)
49447 d->perm[i] &= nelt - 1;
49456 return (which == 3);
49460 ix86_expand_vec_perm_const (rtx operands[4])
49462 struct expand_vec_perm_d d;
49463 unsigned char perm[MAX_VECT_LEN];
49468 d.target = operands[0];
49469 d.op0 = operands[1];
49470 d.op1 = operands[2];
49473 d.vmode = GET_MODE (d.target);
49474 gcc_assert (VECTOR_MODE_P (d.vmode));
49475 d.nelt = nelt = GET_MODE_NUNITS (d.vmode);
49476 d.testing_p = false;
49478 gcc_assert (GET_CODE (sel) == CONST_VECTOR);
49479 gcc_assert (XVECLEN (sel, 0) == nelt);
49480 gcc_checking_assert (sizeof (d.perm) == sizeof (perm));
49482 for (i = 0; i < nelt; ++i)
49484 rtx e = XVECEXP (sel, 0, i);
49485 int ei = INTVAL (e) & (2 * nelt - 1);
49490 two_args = canonicalize_perm (&d);
49492 if (ix86_expand_vec_perm_const_1 (&d))
49495 /* If the selector says both arguments are needed, but the operands are the
49496 same, the above tried to expand with one_operand_p and flattened selector.
49497 If that didn't work, retry without one_operand_p; we succeeded with that
49499 if (two_args && d.one_operand_p)
49501 d.one_operand_p = false;
49502 memcpy (d.perm, perm, sizeof (perm));
49503 return ix86_expand_vec_perm_const_1 (&d);
49509 /* Implement targetm.vectorize.vec_perm_const_ok. */
49512 ix86_vectorize_vec_perm_const_ok (machine_mode vmode,
49513 const unsigned char *sel)
49515 struct expand_vec_perm_d d;
49516 unsigned int i, nelt, which;
49520 d.nelt = nelt = GET_MODE_NUNITS (d.vmode);
49521 d.testing_p = true;
49523 /* Given sufficient ISA support we can just return true here
49524 for selected vector modes. */
49531 if (TARGET_AVX512F)
49532 /* All implementable with a single vpermi2 insn. */
49536 if (TARGET_AVX512BW)
49537 /* All implementable with a single vpermi2 insn. */
49541 if (TARGET_AVX512BW)
49542 /* Implementable with 2 vpermi2, 2 vpshufb and 1 or insn. */
49549 if (TARGET_AVX512VL)
49550 /* All implementable with a single vpermi2 insn. */
49555 /* Implementable with 4 vpshufb insns, 2 vpermq and 3 vpor insns. */
49560 /* Implementable with 4 vpshufb insns, 2 vpermq and 3 vpor insns. */
49567 /* All implementable with a single vpperm insn. */
49570 /* All implementable with 2 pshufb + 1 ior. */
49576 /* All implementable with shufpd or unpck[lh]pd. */
49582 /* Extract the values from the vector CST into the permutation
49584 memcpy (d.perm, sel, nelt);
49585 for (i = which = 0; i < nelt; ++i)
49587 unsigned char e = d.perm[i];
49588 gcc_assert (e < 2 * nelt);
49589 which |= (e < nelt ? 1 : 2);
49592 /* For all elements from second vector, fold the elements to first. */
49594 for (i = 0; i < nelt; ++i)
49597 /* Check whether the mask can be applied to the vector type. */
49598 d.one_operand_p = (which != 3);
49600 /* Implementable with shufps or pshufd. */
49601 if (d.one_operand_p && (d.vmode == V4SFmode || d.vmode == V4SImode))
49604 /* Otherwise we have to go through the motions and see if we can
49605 figure out how to generate the requested permutation. */
49606 d.target = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 1);
49607 d.op1 = d.op0 = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 2);
49608 if (!d.one_operand_p)
49609 d.op1 = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 3);
49612 ret = ix86_expand_vec_perm_const_1 (&d);
49619 ix86_expand_vec_extract_even_odd (rtx targ, rtx op0, rtx op1, unsigned odd)
49621 struct expand_vec_perm_d d;
49627 d.vmode = GET_MODE (targ);
49628 d.nelt = nelt = GET_MODE_NUNITS (d.vmode);
49629 d.one_operand_p = false;
49630 d.testing_p = false;
49632 for (i = 0; i < nelt; ++i)
49633 d.perm[i] = i * 2 + odd;
49635 /* We'll either be able to implement the permutation directly... */
49636 if (expand_vec_perm_1 (&d))
49639 /* ... or we use the special-case patterns. */
49640 expand_vec_perm_even_odd_1 (&d, odd);
49644 ix86_expand_vec_interleave (rtx targ, rtx op0, rtx op1, bool high_p)
49646 struct expand_vec_perm_d d;
49647 unsigned i, nelt, base;
49653 d.vmode = GET_MODE (targ);
49654 d.nelt = nelt = GET_MODE_NUNITS (d.vmode);
49655 d.one_operand_p = false;
49656 d.testing_p = false;
49658 base = high_p ? nelt / 2 : 0;
49659 for (i = 0; i < nelt / 2; ++i)
49661 d.perm[i * 2] = i + base;
49662 d.perm[i * 2 + 1] = i + base + nelt;
49665 /* Note that for AVX this isn't one instruction. */
49666 ok = ix86_expand_vec_perm_const_1 (&d);
49671 /* Expand a vector operation CODE for a V*QImode in terms of the
49672 same operation on V*HImode. */
49675 ix86_expand_vecop_qihi (enum rtx_code code, rtx dest, rtx op1, rtx op2)
49677 machine_mode qimode = GET_MODE (dest);
49678 machine_mode himode;
49679 rtx (*gen_il) (rtx, rtx, rtx);
49680 rtx (*gen_ih) (rtx, rtx, rtx);
49681 rtx op1_l, op1_h, op2_l, op2_h, res_l, res_h;
49682 struct expand_vec_perm_d d;
49683 bool ok, full_interleave;
49684 bool uns_p = false;
49691 gen_il = gen_vec_interleave_lowv16qi;
49692 gen_ih = gen_vec_interleave_highv16qi;
49695 himode = V16HImode;
49696 gen_il = gen_avx2_interleave_lowv32qi;
49697 gen_ih = gen_avx2_interleave_highv32qi;
49700 himode = V32HImode;
49701 gen_il = gen_avx512bw_interleave_lowv64qi;
49702 gen_ih = gen_avx512bw_interleave_highv64qi;
49705 gcc_unreachable ();
49708 op2_l = op2_h = op2;
49712 /* Unpack data such that we've got a source byte in each low byte of
49713 each word. We don't care what goes into the high byte of each word.
49714 Rather than trying to get zero in there, most convenient is to let
49715 it be a copy of the low byte. */
49716 op2_l = gen_reg_rtx (qimode);
49717 op2_h = gen_reg_rtx (qimode);
49718 emit_insn (gen_il (op2_l, op2, op2));
49719 emit_insn (gen_ih (op2_h, op2, op2));
49722 op1_l = gen_reg_rtx (qimode);
49723 op1_h = gen_reg_rtx (qimode);
49724 emit_insn (gen_il (op1_l, op1, op1));
49725 emit_insn (gen_ih (op1_h, op1, op1));
49726 full_interleave = qimode == V16QImode;
49734 op1_l = gen_reg_rtx (himode);
49735 op1_h = gen_reg_rtx (himode);
49736 ix86_expand_sse_unpack (op1_l, op1, uns_p, false);
49737 ix86_expand_sse_unpack (op1_h, op1, uns_p, true);
49738 full_interleave = true;
49741 gcc_unreachable ();
49744 /* Perform the operation. */
49745 res_l = expand_simple_binop (himode, code, op1_l, op2_l, NULL_RTX,
49747 res_h = expand_simple_binop (himode, code, op1_h, op2_h, NULL_RTX,
49749 gcc_assert (res_l && res_h);
49751 /* Merge the data back into the right place. */
49753 d.op0 = gen_lowpart (qimode, res_l);
49754 d.op1 = gen_lowpart (qimode, res_h);
49756 d.nelt = GET_MODE_NUNITS (qimode);
49757 d.one_operand_p = false;
49758 d.testing_p = false;
49760 if (full_interleave)
49762 /* For SSE2, we used an full interleave, so the desired
49763 results are in the even elements. */
49764 for (i = 0; i < 64; ++i)
49769 /* For AVX, the interleave used above was not cross-lane. So the
49770 extraction is evens but with the second and third quarter swapped.
49771 Happily, that is even one insn shorter than even extraction. */
49772 for (i = 0; i < 64; ++i)
49773 d.perm[i] = i * 2 + ((i & 24) == 8 ? 16 : (i & 24) == 16 ? -16 : 0);
49776 ok = ix86_expand_vec_perm_const_1 (&d);
49779 set_unique_reg_note (get_last_insn (), REG_EQUAL,
49780 gen_rtx_fmt_ee (code, qimode, op1, op2));
49783 /* Helper function of ix86_expand_mul_widen_evenodd. Return true
49784 if op is CONST_VECTOR with all odd elements equal to their
49785 preceding element. */
49788 const_vector_equal_evenodd_p (rtx op)
49790 machine_mode mode = GET_MODE (op);
49791 int i, nunits = GET_MODE_NUNITS (mode);
49792 if (GET_CODE (op) != CONST_VECTOR
49793 || nunits != CONST_VECTOR_NUNITS (op))
49795 for (i = 0; i < nunits; i += 2)
49796 if (CONST_VECTOR_ELT (op, i) != CONST_VECTOR_ELT (op, i + 1))
49802 ix86_expand_mul_widen_evenodd (rtx dest, rtx op1, rtx op2,
49803 bool uns_p, bool odd_p)
49805 machine_mode mode = GET_MODE (op1);
49806 machine_mode wmode = GET_MODE (dest);
49808 rtx orig_op1 = op1, orig_op2 = op2;
49810 if (!nonimmediate_operand (op1, mode))
49811 op1 = force_reg (mode, op1);
49812 if (!nonimmediate_operand (op2, mode))
49813 op2 = force_reg (mode, op2);
49815 /* We only play even/odd games with vectors of SImode. */
49816 gcc_assert (mode == V4SImode || mode == V8SImode || mode == V16SImode);
49818 /* If we're looking for the odd results, shift those members down to
49819 the even slots. For some cpus this is faster than a PSHUFD. */
49822 /* For XOP use vpmacsdqh, but only for smult, as it is only
49824 if (TARGET_XOP && mode == V4SImode && !uns_p)
49826 x = force_reg (wmode, CONST0_RTX (wmode));
49827 emit_insn (gen_xop_pmacsdqh (dest, op1, op2, x));
49831 x = GEN_INT (GET_MODE_UNIT_BITSIZE (mode));
49832 if (!const_vector_equal_evenodd_p (orig_op1))
49833 op1 = expand_binop (wmode, lshr_optab, gen_lowpart (wmode, op1),
49834 x, NULL, 1, OPTAB_DIRECT);
49835 if (!const_vector_equal_evenodd_p (orig_op2))
49836 op2 = expand_binop (wmode, lshr_optab, gen_lowpart (wmode, op2),
49837 x, NULL, 1, OPTAB_DIRECT);
49838 op1 = gen_lowpart (mode, op1);
49839 op2 = gen_lowpart (mode, op2);
49842 if (mode == V16SImode)
49845 x = gen_vec_widen_umult_even_v16si (dest, op1, op2);
49847 x = gen_vec_widen_smult_even_v16si (dest, op1, op2);
49849 else if (mode == V8SImode)
49852 x = gen_vec_widen_umult_even_v8si (dest, op1, op2);
49854 x = gen_vec_widen_smult_even_v8si (dest, op1, op2);
49857 x = gen_vec_widen_umult_even_v4si (dest, op1, op2);
49858 else if (TARGET_SSE4_1)
49859 x = gen_sse4_1_mulv2siv2di3 (dest, op1, op2);
49862 rtx s1, s2, t0, t1, t2;
49864 /* The easiest way to implement this without PMULDQ is to go through
49865 the motions as if we are performing a full 64-bit multiply. With
49866 the exception that we need to do less shuffling of the elements. */
49868 /* Compute the sign-extension, aka highparts, of the two operands. */
49869 s1 = ix86_expand_sse_cmp (gen_reg_rtx (mode), GT, CONST0_RTX (mode),
49870 op1, pc_rtx, pc_rtx);
49871 s2 = ix86_expand_sse_cmp (gen_reg_rtx (mode), GT, CONST0_RTX (mode),
49872 op2, pc_rtx, pc_rtx);
49874 /* Multiply LO(A) * HI(B), and vice-versa. */
49875 t1 = gen_reg_rtx (wmode);
49876 t2 = gen_reg_rtx (wmode);
49877 emit_insn (gen_vec_widen_umult_even_v4si (t1, s1, op2));
49878 emit_insn (gen_vec_widen_umult_even_v4si (t2, s2, op1));
49880 /* Multiply LO(A) * LO(B). */
49881 t0 = gen_reg_rtx (wmode);
49882 emit_insn (gen_vec_widen_umult_even_v4si (t0, op1, op2));
49884 /* Combine and shift the highparts into place. */
49885 t1 = expand_binop (wmode, add_optab, t1, t2, t1, 1, OPTAB_DIRECT);
49886 t1 = expand_binop (wmode, ashl_optab, t1, GEN_INT (32), t1,
49889 /* Combine high and low parts. */
49890 force_expand_binop (wmode, add_optab, t0, t1, dest, 1, OPTAB_DIRECT);
49897 ix86_expand_mul_widen_hilo (rtx dest, rtx op1, rtx op2,
49898 bool uns_p, bool high_p)
49900 machine_mode wmode = GET_MODE (dest);
49901 machine_mode mode = GET_MODE (op1);
49902 rtx t1, t2, t3, t4, mask;
49907 t1 = gen_reg_rtx (mode);
49908 t2 = gen_reg_rtx (mode);
49909 if (TARGET_XOP && !uns_p)
49911 /* With XOP, we have pmacsdqh, aka mul_widen_odd. In this case,
49912 shuffle the elements once so that all elements are in the right
49913 place for immediate use: { A C B D }. */
49914 emit_insn (gen_sse2_pshufd_1 (t1, op1, const0_rtx, const2_rtx,
49915 const1_rtx, GEN_INT (3)));
49916 emit_insn (gen_sse2_pshufd_1 (t2, op2, const0_rtx, const2_rtx,
49917 const1_rtx, GEN_INT (3)));
49921 /* Put the elements into place for the multiply. */
49922 ix86_expand_vec_interleave (t1, op1, op1, high_p);
49923 ix86_expand_vec_interleave (t2, op2, op2, high_p);
49926 ix86_expand_mul_widen_evenodd (dest, t1, t2, uns_p, high_p);
49930 /* Shuffle the elements between the lanes. After this we
49931 have { A B E F | C D G H } for each operand. */
49932 t1 = gen_reg_rtx (V4DImode);
49933 t2 = gen_reg_rtx (V4DImode);
49934 emit_insn (gen_avx2_permv4di_1 (t1, gen_lowpart (V4DImode, op1),
49935 const0_rtx, const2_rtx,
49936 const1_rtx, GEN_INT (3)));
49937 emit_insn (gen_avx2_permv4di_1 (t2, gen_lowpart (V4DImode, op2),
49938 const0_rtx, const2_rtx,
49939 const1_rtx, GEN_INT (3)));
49941 /* Shuffle the elements within the lanes. After this we
49942 have { A A B B | C C D D } or { E E F F | G G H H }. */
49943 t3 = gen_reg_rtx (V8SImode);
49944 t4 = gen_reg_rtx (V8SImode);
49945 mask = GEN_INT (high_p
49946 ? 2 + (2 << 2) + (3 << 4) + (3 << 6)
49947 : 0 + (0 << 2) + (1 << 4) + (1 << 6));
49948 emit_insn (gen_avx2_pshufdv3 (t3, gen_lowpart (V8SImode, t1), mask));
49949 emit_insn (gen_avx2_pshufdv3 (t4, gen_lowpart (V8SImode, t2), mask));
49951 ix86_expand_mul_widen_evenodd (dest, t3, t4, uns_p, false);
49956 t1 = expand_binop (mode, smul_optab, op1, op2, NULL_RTX,
49957 uns_p, OPTAB_DIRECT);
49958 t2 = expand_binop (mode,
49959 uns_p ? umul_highpart_optab : smul_highpart_optab,
49960 op1, op2, NULL_RTX, uns_p, OPTAB_DIRECT);
49961 gcc_assert (t1 && t2);
49963 t3 = gen_reg_rtx (mode);
49964 ix86_expand_vec_interleave (t3, t1, t2, high_p);
49965 emit_move_insn (dest, gen_lowpart (wmode, t3));
49973 t1 = gen_reg_rtx (wmode);
49974 t2 = gen_reg_rtx (wmode);
49975 ix86_expand_sse_unpack (t1, op1, uns_p, high_p);
49976 ix86_expand_sse_unpack (t2, op2, uns_p, high_p);
49978 emit_insn (gen_rtx_SET (dest, gen_rtx_MULT (wmode, t1, t2)));
49982 gcc_unreachable ();
49987 ix86_expand_sse2_mulv4si3 (rtx op0, rtx op1, rtx op2)
49989 rtx res_1, res_2, res_3, res_4;
49991 res_1 = gen_reg_rtx (V4SImode);
49992 res_2 = gen_reg_rtx (V4SImode);
49993 res_3 = gen_reg_rtx (V2DImode);
49994 res_4 = gen_reg_rtx (V2DImode);
49995 ix86_expand_mul_widen_evenodd (res_3, op1, op2, true, false);
49996 ix86_expand_mul_widen_evenodd (res_4, op1, op2, true, true);
49998 /* Move the results in element 2 down to element 1; we don't care
49999 what goes in elements 2 and 3. Then we can merge the parts
50000 back together with an interleave.
50002 Note that two other sequences were tried:
50003 (1) Use interleaves at the start instead of psrldq, which allows
50004 us to use a single shufps to merge things back at the end.
50005 (2) Use shufps here to combine the two vectors, then pshufd to
50006 put the elements in the correct order.
50007 In both cases the cost of the reformatting stall was too high
50008 and the overall sequence slower. */
50010 emit_insn (gen_sse2_pshufd_1 (res_1, gen_lowpart (V4SImode, res_3),
50011 const0_rtx, const2_rtx,
50012 const0_rtx, const0_rtx));
50013 emit_insn (gen_sse2_pshufd_1 (res_2, gen_lowpart (V4SImode, res_4),
50014 const0_rtx, const2_rtx,
50015 const0_rtx, const0_rtx));
50016 res_1 = emit_insn (gen_vec_interleave_lowv4si (op0, res_1, res_2));
50018 set_unique_reg_note (res_1, REG_EQUAL, gen_rtx_MULT (V4SImode, op1, op2));
50022 ix86_expand_sse2_mulvxdi3 (rtx op0, rtx op1, rtx op2)
50024 machine_mode mode = GET_MODE (op0);
50025 rtx t1, t2, t3, t4, t5, t6;
50027 if (TARGET_AVX512DQ && mode == V8DImode)
50028 emit_insn (gen_avx512dq_mulv8di3 (op0, op1, op2));
50029 else if (TARGET_AVX512DQ && TARGET_AVX512VL && mode == V4DImode)
50030 emit_insn (gen_avx512dq_mulv4di3 (op0, op1, op2));
50031 else if (TARGET_AVX512DQ && TARGET_AVX512VL && mode == V2DImode)
50032 emit_insn (gen_avx512dq_mulv2di3 (op0, op1, op2));
50033 else if (TARGET_XOP && mode == V2DImode)
50035 /* op1: A,B,C,D, op2: E,F,G,H */
50036 op1 = gen_lowpart (V4SImode, op1);
50037 op2 = gen_lowpart (V4SImode, op2);
50039 t1 = gen_reg_rtx (V4SImode);
50040 t2 = gen_reg_rtx (V4SImode);
50041 t3 = gen_reg_rtx (V2DImode);
50042 t4 = gen_reg_rtx (V2DImode);
50045 emit_insn (gen_sse2_pshufd_1 (t1, op1,
50051 /* t2: (B*E),(A*F),(D*G),(C*H) */
50052 emit_insn (gen_mulv4si3 (t2, t1, op2));
50054 /* t3: (B*E)+(A*F), (D*G)+(C*H) */
50055 emit_insn (gen_xop_phadddq (t3, t2));
50057 /* t4: ((B*E)+(A*F))<<32, ((D*G)+(C*H))<<32 */
50058 emit_insn (gen_ashlv2di3 (t4, t3, GEN_INT (32)));
50060 /* Multiply lower parts and add all */
50061 t5 = gen_reg_rtx (V2DImode);
50062 emit_insn (gen_vec_widen_umult_even_v4si (t5,
50063 gen_lowpart (V4SImode, op1),
50064 gen_lowpart (V4SImode, op2)));
50065 op0 = expand_binop (mode, add_optab, t5, t4, op0, 1, OPTAB_DIRECT);
50070 machine_mode nmode;
50071 rtx (*umul) (rtx, rtx, rtx);
50073 if (mode == V2DImode)
50075 umul = gen_vec_widen_umult_even_v4si;
50078 else if (mode == V4DImode)
50080 umul = gen_vec_widen_umult_even_v8si;
50083 else if (mode == V8DImode)
50085 umul = gen_vec_widen_umult_even_v16si;
50089 gcc_unreachable ();
50092 /* Multiply low parts. */
50093 t1 = gen_reg_rtx (mode);
50094 emit_insn (umul (t1, gen_lowpart (nmode, op1), gen_lowpart (nmode, op2)));
50096 /* Shift input vectors right 32 bits so we can multiply high parts. */
50098 t2 = expand_binop (mode, lshr_optab, op1, t6, NULL, 1, OPTAB_DIRECT);
50099 t3 = expand_binop (mode, lshr_optab, op2, t6, NULL, 1, OPTAB_DIRECT);
50101 /* Multiply high parts by low parts. */
50102 t4 = gen_reg_rtx (mode);
50103 t5 = gen_reg_rtx (mode);
50104 emit_insn (umul (t4, gen_lowpart (nmode, t2), gen_lowpart (nmode, op2)));
50105 emit_insn (umul (t5, gen_lowpart (nmode, t3), gen_lowpart (nmode, op1)));
50107 /* Combine and shift the highparts back. */
50108 t4 = expand_binop (mode, add_optab, t4, t5, t4, 1, OPTAB_DIRECT);
50109 t4 = expand_binop (mode, ashl_optab, t4, t6, t4, 1, OPTAB_DIRECT);
50111 /* Combine high and low parts. */
50112 force_expand_binop (mode, add_optab, t1, t4, op0, 1, OPTAB_DIRECT);
50115 set_unique_reg_note (get_last_insn (), REG_EQUAL,
50116 gen_rtx_MULT (mode, op1, op2));
50119 /* Return 1 if control tansfer instruction INSN
50120 should be encoded with bnd prefix.
50121 If insn is NULL then return 1 when control
50122 transfer instructions should be prefixed with
50123 bnd by default for current function. */
50126 ix86_bnd_prefixed_insn_p (rtx insn)
50128 /* For call insns check special flag. */
50129 if (insn && CALL_P (insn))
50131 rtx call = get_call_rtx_from (insn);
50133 return CALL_EXPR_WITH_BOUNDS_P (call);
50136 /* All other insns are prefixed only if function is instrumented. */
50137 return chkp_function_instrumented_p (current_function_decl);
50140 /* Calculate integer abs() using only SSE2 instructions. */
50143 ix86_expand_sse2_abs (rtx target, rtx input)
50145 machine_mode mode = GET_MODE (target);
50150 /* For 32-bit signed integer X, the best way to calculate the absolute
50151 value of X is (((signed) X >> (W-1)) ^ X) - ((signed) X >> (W-1)). */
50153 tmp0 = expand_simple_binop (mode, ASHIFTRT, input,
50154 GEN_INT (GET_MODE_BITSIZE
50155 (GET_MODE_INNER (mode)) - 1),
50156 NULL, 0, OPTAB_DIRECT);
50157 tmp1 = expand_simple_binop (mode, XOR, tmp0, input,
50158 NULL, 0, OPTAB_DIRECT);
50159 x = expand_simple_binop (mode, MINUS, tmp1, tmp0,
50160 target, 0, OPTAB_DIRECT);
50163 /* For 16-bit signed integer X, the best way to calculate the absolute
50164 value of X is max (X, -X), as SSE2 provides the PMAXSW insn. */
50166 tmp0 = expand_unop (mode, neg_optab, input, NULL_RTX, 0);
50168 x = expand_simple_binop (mode, SMAX, tmp0, input,
50169 target, 0, OPTAB_DIRECT);
50172 /* For 8-bit signed integer X, the best way to calculate the absolute
50173 value of X is min ((unsigned char) X, (unsigned char) (-X)),
50174 as SSE2 provides the PMINUB insn. */
50176 tmp0 = expand_unop (mode, neg_optab, input, NULL_RTX, 0);
50178 x = expand_simple_binop (V16QImode, UMIN, tmp0, input,
50179 target, 0, OPTAB_DIRECT);
50183 gcc_unreachable ();
50187 emit_move_insn (target, x);
50190 /* Expand an insert into a vector register through pinsr insn.
50191 Return true if successful. */
50194 ix86_expand_pinsr (rtx *operands)
50196 rtx dst = operands[0];
50197 rtx src = operands[3];
50199 unsigned int size = INTVAL (operands[1]);
50200 unsigned int pos = INTVAL (operands[2]);
50202 if (GET_CODE (dst) == SUBREG)
50204 pos += SUBREG_BYTE (dst) * BITS_PER_UNIT;
50205 dst = SUBREG_REG (dst);
50208 if (GET_CODE (src) == SUBREG)
50209 src = SUBREG_REG (src);
50211 switch (GET_MODE (dst))
50218 machine_mode srcmode, dstmode;
50219 rtx (*pinsr)(rtx, rtx, rtx, rtx);
50221 srcmode = mode_for_size (size, MODE_INT, 0);
50226 if (!TARGET_SSE4_1)
50228 dstmode = V16QImode;
50229 pinsr = gen_sse4_1_pinsrb;
50235 dstmode = V8HImode;
50236 pinsr = gen_sse2_pinsrw;
50240 if (!TARGET_SSE4_1)
50242 dstmode = V4SImode;
50243 pinsr = gen_sse4_1_pinsrd;
50247 gcc_assert (TARGET_64BIT);
50248 if (!TARGET_SSE4_1)
50250 dstmode = V2DImode;
50251 pinsr = gen_sse4_1_pinsrq;
50259 if (GET_MODE (dst) != dstmode)
50260 d = gen_reg_rtx (dstmode);
50261 src = gen_lowpart (srcmode, src);
50265 emit_insn (pinsr (d, gen_lowpart (dstmode, dst), src,
50266 GEN_INT (1 << pos)));
50268 emit_move_insn (dst, gen_lowpart (GET_MODE (dst), d));
50277 /* This function returns the calling abi specific va_list type node.
50278 It returns the FNDECL specific va_list type. */
50281 ix86_fn_abi_va_list (tree fndecl)
50284 return va_list_type_node;
50285 gcc_assert (fndecl != NULL_TREE);
50287 if (ix86_function_abi ((const_tree) fndecl) == MS_ABI)
50288 return ms_va_list_type_node;
50290 return sysv_va_list_type_node;
50293 /* Returns the canonical va_list type specified by TYPE. If there
50294 is no valid TYPE provided, it return NULL_TREE. */
50297 ix86_canonical_va_list_type (tree type)
50301 /* Resolve references and pointers to va_list type. */
50302 if (TREE_CODE (type) == MEM_REF)
50303 type = TREE_TYPE (type);
50304 else if (POINTER_TYPE_P (type) && POINTER_TYPE_P (TREE_TYPE(type)))
50305 type = TREE_TYPE (type);
50306 else if (POINTER_TYPE_P (type) && TREE_CODE (TREE_TYPE (type)) == ARRAY_TYPE)
50307 type = TREE_TYPE (type);
50309 if (TARGET_64BIT && va_list_type_node != NULL_TREE)
50311 wtype = va_list_type_node;
50312 gcc_assert (wtype != NULL_TREE);
50314 if (TREE_CODE (wtype) == ARRAY_TYPE)
50316 /* If va_list is an array type, the argument may have decayed
50317 to a pointer type, e.g. by being passed to another function.
50318 In that case, unwrap both types so that we can compare the
50319 underlying records. */
50320 if (TREE_CODE (htype) == ARRAY_TYPE
50321 || POINTER_TYPE_P (htype))
50323 wtype = TREE_TYPE (wtype);
50324 htype = TREE_TYPE (htype);
50327 if (TYPE_MAIN_VARIANT (wtype) == TYPE_MAIN_VARIANT (htype))
50328 return va_list_type_node;
50329 wtype = sysv_va_list_type_node;
50330 gcc_assert (wtype != NULL_TREE);
50332 if (TREE_CODE (wtype) == ARRAY_TYPE)
50334 /* If va_list is an array type, the argument may have decayed
50335 to a pointer type, e.g. by being passed to another function.
50336 In that case, unwrap both types so that we can compare the
50337 underlying records. */
50338 if (TREE_CODE (htype) == ARRAY_TYPE
50339 || POINTER_TYPE_P (htype))
50341 wtype = TREE_TYPE (wtype);
50342 htype = TREE_TYPE (htype);
50345 if (TYPE_MAIN_VARIANT (wtype) == TYPE_MAIN_VARIANT (htype))
50346 return sysv_va_list_type_node;
50347 wtype = ms_va_list_type_node;
50348 gcc_assert (wtype != NULL_TREE);
50350 if (TREE_CODE (wtype) == ARRAY_TYPE)
50352 /* If va_list is an array type, the argument may have decayed
50353 to a pointer type, e.g. by being passed to another function.
50354 In that case, unwrap both types so that we can compare the
50355 underlying records. */
50356 if (TREE_CODE (htype) == ARRAY_TYPE
50357 || POINTER_TYPE_P (htype))
50359 wtype = TREE_TYPE (wtype);
50360 htype = TREE_TYPE (htype);
50363 if (TYPE_MAIN_VARIANT (wtype) == TYPE_MAIN_VARIANT (htype))
50364 return ms_va_list_type_node;
50367 return std_canonical_va_list_type (type);
50370 /* Iterate through the target-specific builtin types for va_list.
50371 IDX denotes the iterator, *PTREE is set to the result type of
50372 the va_list builtin, and *PNAME to its internal type.
50373 Returns zero if there is no element for this index, otherwise
50374 IDX should be increased upon the next call.
50375 Note, do not iterate a base builtin's name like __builtin_va_list.
50376 Used from c_common_nodes_and_builtins. */
50379 ix86_enum_va_list (int idx, const char **pname, tree *ptree)
50389 *ptree = ms_va_list_type_node;
50390 *pname = "__builtin_ms_va_list";
50394 *ptree = sysv_va_list_type_node;
50395 *pname = "__builtin_sysv_va_list";
50403 #undef TARGET_SCHED_DISPATCH
50404 #define TARGET_SCHED_DISPATCH has_dispatch
50405 #undef TARGET_SCHED_DISPATCH_DO
50406 #define TARGET_SCHED_DISPATCH_DO do_dispatch
50407 #undef TARGET_SCHED_REASSOCIATION_WIDTH
50408 #define TARGET_SCHED_REASSOCIATION_WIDTH ix86_reassociation_width
50409 #undef TARGET_SCHED_REORDER
50410 #define TARGET_SCHED_REORDER ix86_sched_reorder
50411 #undef TARGET_SCHED_ADJUST_PRIORITY
50412 #define TARGET_SCHED_ADJUST_PRIORITY ix86_adjust_priority
50413 #undef TARGET_SCHED_DEPENDENCIES_EVALUATION_HOOK
50414 #define TARGET_SCHED_DEPENDENCIES_EVALUATION_HOOK \
50415 ix86_dependencies_evaluation_hook
50417 /* The size of the dispatch window is the total number of bytes of
50418 object code allowed in a window. */
50419 #define DISPATCH_WINDOW_SIZE 16
50421 /* Number of dispatch windows considered for scheduling. */
50422 #define MAX_DISPATCH_WINDOWS 3
50424 /* Maximum number of instructions in a window. */
50427 /* Maximum number of immediate operands in a window. */
50430 /* Maximum number of immediate bits allowed in a window. */
50431 #define MAX_IMM_SIZE 128
50433 /* Maximum number of 32 bit immediates allowed in a window. */
50434 #define MAX_IMM_32 4
50436 /* Maximum number of 64 bit immediates allowed in a window. */
50437 #define MAX_IMM_64 2
50439 /* Maximum total of loads or prefetches allowed in a window. */
50442 /* Maximum total of stores allowed in a window. */
50443 #define MAX_STORE 1
50449 /* Dispatch groups. Istructions that affect the mix in a dispatch window. */
50450 enum dispatch_group {
50465 /* Number of allowable groups in a dispatch window. It is an array
50466 indexed by dispatch_group enum. 100 is used as a big number,
50467 because the number of these kind of operations does not have any
50468 effect in dispatch window, but we need them for other reasons in
50470 static unsigned int num_allowable_groups[disp_last] = {
50471 0, 2, 1, 1, 2, 4, 4, 2, 1, BIG, BIG
50474 char group_name[disp_last + 1][16] = {
50475 "disp_no_group", "disp_load", "disp_store", "disp_load_store",
50476 "disp_prefetch", "disp_imm", "disp_imm_32", "disp_imm_64",
50477 "disp_branch", "disp_cmp", "disp_jcc", "disp_last"
50480 /* Instruction path. */
50483 path_single, /* Single micro op. */
50484 path_double, /* Double micro op. */
50485 path_multi, /* Instructions with more than 2 micro op.. */
50489 /* sched_insn_info defines a window to the instructions scheduled in
50490 the basic block. It contains a pointer to the insn_info table and
50491 the instruction scheduled.
50493 Windows are allocated for each basic block and are linked
50495 typedef struct sched_insn_info_s {
50497 enum dispatch_group group;
50498 enum insn_path path;
50503 /* Linked list of dispatch windows. This is a two way list of
50504 dispatch windows of a basic block. It contains information about
50505 the number of uops in the window and the total number of
50506 instructions and of bytes in the object code for this dispatch
50508 typedef struct dispatch_windows_s {
50509 int num_insn; /* Number of insn in the window. */
50510 int num_uops; /* Number of uops in the window. */
50511 int window_size; /* Number of bytes in the window. */
50512 int window_num; /* Window number between 0 or 1. */
50513 int num_imm; /* Number of immediates in an insn. */
50514 int num_imm_32; /* Number of 32 bit immediates in an insn. */
50515 int num_imm_64; /* Number of 64 bit immediates in an insn. */
50516 int imm_size; /* Total immediates in the window. */
50517 int num_loads; /* Total memory loads in the window. */
50518 int num_stores; /* Total memory stores in the window. */
50519 int violation; /* Violation exists in window. */
50520 sched_insn_info *window; /* Pointer to the window. */
50521 struct dispatch_windows_s *next;
50522 struct dispatch_windows_s *prev;
50523 } dispatch_windows;
50525 /* Immediate valuse used in an insn. */
50526 typedef struct imm_info_s
50533 static dispatch_windows *dispatch_window_list;
50534 static dispatch_windows *dispatch_window_list1;
50536 /* Get dispatch group of insn. */
50538 static enum dispatch_group
50539 get_mem_group (rtx_insn *insn)
50541 enum attr_memory memory;
50543 if (INSN_CODE (insn) < 0)
50544 return disp_no_group;
50545 memory = get_attr_memory (insn);
50546 if (memory == MEMORY_STORE)
50549 if (memory == MEMORY_LOAD)
50552 if (memory == MEMORY_BOTH)
50553 return disp_load_store;
50555 return disp_no_group;
50558 /* Return true if insn is a compare instruction. */
50561 is_cmp (rtx_insn *insn)
50563 enum attr_type type;
50565 type = get_attr_type (insn);
50566 return (type == TYPE_TEST
50567 || type == TYPE_ICMP
50568 || type == TYPE_FCMP
50569 || GET_CODE (PATTERN (insn)) == COMPARE);
50572 /* Return true if a dispatch violation encountered. */
50575 dispatch_violation (void)
50577 if (dispatch_window_list->next)
50578 return dispatch_window_list->next->violation;
50579 return dispatch_window_list->violation;
50582 /* Return true if insn is a branch instruction. */
50585 is_branch (rtx_insn *insn)
50587 return (CALL_P (insn) || JUMP_P (insn));
50590 /* Return true if insn is a prefetch instruction. */
50593 is_prefetch (rtx_insn *insn)
50595 return NONJUMP_INSN_P (insn) && GET_CODE (PATTERN (insn)) == PREFETCH;
50598 /* This function initializes a dispatch window and the list container holding a
50599 pointer to the window. */
50602 init_window (int window_num)
50605 dispatch_windows *new_list;
50607 if (window_num == 0)
50608 new_list = dispatch_window_list;
50610 new_list = dispatch_window_list1;
50612 new_list->num_insn = 0;
50613 new_list->num_uops = 0;
50614 new_list->window_size = 0;
50615 new_list->next = NULL;
50616 new_list->prev = NULL;
50617 new_list->window_num = window_num;
50618 new_list->num_imm = 0;
50619 new_list->num_imm_32 = 0;
50620 new_list->num_imm_64 = 0;
50621 new_list->imm_size = 0;
50622 new_list->num_loads = 0;
50623 new_list->num_stores = 0;
50624 new_list->violation = false;
50626 for (i = 0; i < MAX_INSN; i++)
50628 new_list->window[i].insn = NULL;
50629 new_list->window[i].group = disp_no_group;
50630 new_list->window[i].path = no_path;
50631 new_list->window[i].byte_len = 0;
50632 new_list->window[i].imm_bytes = 0;
50637 /* This function allocates and initializes a dispatch window and the
50638 list container holding a pointer to the window. */
50640 static dispatch_windows *
50641 allocate_window (void)
50643 dispatch_windows *new_list = XNEW (struct dispatch_windows_s);
50644 new_list->window = XNEWVEC (struct sched_insn_info_s, MAX_INSN + 1);
50649 /* This routine initializes the dispatch scheduling information. It
50650 initiates building dispatch scheduler tables and constructs the
50651 first dispatch window. */
50654 init_dispatch_sched (void)
50656 /* Allocate a dispatch list and a window. */
50657 dispatch_window_list = allocate_window ();
50658 dispatch_window_list1 = allocate_window ();
50663 /* This function returns true if a branch is detected. End of a basic block
50664 does not have to be a branch, but here we assume only branches end a
50668 is_end_basic_block (enum dispatch_group group)
50670 return group == disp_branch;
50673 /* This function is called when the end of a window processing is reached. */
50676 process_end_window (void)
50678 gcc_assert (dispatch_window_list->num_insn <= MAX_INSN);
50679 if (dispatch_window_list->next)
50681 gcc_assert (dispatch_window_list1->num_insn <= MAX_INSN);
50682 gcc_assert (dispatch_window_list->window_size
50683 + dispatch_window_list1->window_size <= 48);
50689 /* Allocates a new dispatch window and adds it to WINDOW_LIST.
50690 WINDOW_NUM is either 0 or 1. A maximum of two windows are generated
50691 for 48 bytes of instructions. Note that these windows are not dispatch
50692 windows that their sizes are DISPATCH_WINDOW_SIZE. */
50694 static dispatch_windows *
50695 allocate_next_window (int window_num)
50697 if (window_num == 0)
50699 if (dispatch_window_list->next)
50702 return dispatch_window_list;
50705 dispatch_window_list->next = dispatch_window_list1;
50706 dispatch_window_list1->prev = dispatch_window_list;
50708 return dispatch_window_list1;
50711 /* Compute number of immediate operands of an instruction. */
50714 find_constant (rtx in_rtx, imm_info *imm_values)
50716 if (INSN_P (in_rtx))
50717 in_rtx = PATTERN (in_rtx);
50718 subrtx_iterator::array_type array;
50719 FOR_EACH_SUBRTX (iter, array, in_rtx, ALL)
50720 if (const_rtx x = *iter)
50721 switch (GET_CODE (x))
50726 (imm_values->imm)++;
50727 if (x86_64_immediate_operand (CONST_CAST_RTX (x), SImode))
50728 (imm_values->imm32)++;
50730 (imm_values->imm64)++;
50734 case CONST_WIDE_INT:
50735 (imm_values->imm)++;
50736 (imm_values->imm64)++;
50740 if (LABEL_KIND (x) == LABEL_NORMAL)
50742 (imm_values->imm)++;
50743 (imm_values->imm32)++;
50752 /* Return total size of immediate operands of an instruction along with number
50753 of corresponding immediate-operands. It initializes its parameters to zero
50754 befor calling FIND_CONSTANT.
50755 INSN is the input instruction. IMM is the total of immediates.
50756 IMM32 is the number of 32 bit immediates. IMM64 is the number of 64
50760 get_num_immediates (rtx_insn *insn, int *imm, int *imm32, int *imm64)
50762 imm_info imm_values = {0, 0, 0};
50764 find_constant (insn, &imm_values);
50765 *imm = imm_values.imm;
50766 *imm32 = imm_values.imm32;
50767 *imm64 = imm_values.imm64;
50768 return imm_values.imm32 * 4 + imm_values.imm64 * 8;
50771 /* This function indicates if an operand of an instruction is an
50775 has_immediate (rtx_insn *insn)
50777 int num_imm_operand;
50778 int num_imm32_operand;
50779 int num_imm64_operand;
50782 return get_num_immediates (insn, &num_imm_operand, &num_imm32_operand,
50783 &num_imm64_operand);
50787 /* Return single or double path for instructions. */
50789 static enum insn_path
50790 get_insn_path (rtx_insn *insn)
50792 enum attr_amdfam10_decode path = get_attr_amdfam10_decode (insn);
50794 if ((int)path == 0)
50795 return path_single;
50797 if ((int)path == 1)
50798 return path_double;
50803 /* Return insn dispatch group. */
50805 static enum dispatch_group
50806 get_insn_group (rtx_insn *insn)
50808 enum dispatch_group group = get_mem_group (insn);
50812 if (is_branch (insn))
50813 return disp_branch;
50818 if (has_immediate (insn))
50821 if (is_prefetch (insn))
50822 return disp_prefetch;
50824 return disp_no_group;
50827 /* Count number of GROUP restricted instructions in a dispatch
50828 window WINDOW_LIST. */
50831 count_num_restricted (rtx_insn *insn, dispatch_windows *window_list)
50833 enum dispatch_group group = get_insn_group (insn);
50835 int num_imm_operand;
50836 int num_imm32_operand;
50837 int num_imm64_operand;
50839 if (group == disp_no_group)
50842 if (group == disp_imm)
50844 imm_size = get_num_immediates (insn, &num_imm_operand, &num_imm32_operand,
50845 &num_imm64_operand);
50846 if (window_list->imm_size + imm_size > MAX_IMM_SIZE
50847 || num_imm_operand + window_list->num_imm > MAX_IMM
50848 || (num_imm32_operand > 0
50849 && (window_list->num_imm_32 + num_imm32_operand > MAX_IMM_32
50850 || window_list->num_imm_64 * 2 + num_imm32_operand > MAX_IMM_32))
50851 || (num_imm64_operand > 0
50852 && (window_list->num_imm_64 + num_imm64_operand > MAX_IMM_64
50853 || window_list->num_imm_32 + num_imm64_operand * 2 > MAX_IMM_32))
50854 || (window_list->imm_size + imm_size == MAX_IMM_SIZE
50855 && num_imm64_operand > 0
50856 && ((window_list->num_imm_64 > 0
50857 && window_list->num_insn >= 2)
50858 || window_list->num_insn >= 3)))
50864 if ((group == disp_load_store
50865 && (window_list->num_loads >= MAX_LOAD
50866 || window_list->num_stores >= MAX_STORE))
50867 || ((group == disp_load
50868 || group == disp_prefetch)
50869 && window_list->num_loads >= MAX_LOAD)
50870 || (group == disp_store
50871 && window_list->num_stores >= MAX_STORE))
50877 /* This function returns true if insn satisfies dispatch rules on the
50878 last window scheduled. */
50881 fits_dispatch_window (rtx_insn *insn)
50883 dispatch_windows *window_list = dispatch_window_list;
50884 dispatch_windows *window_list_next = dispatch_window_list->next;
50885 unsigned int num_restrict;
50886 enum dispatch_group group = get_insn_group (insn);
50887 enum insn_path path = get_insn_path (insn);
50890 /* Make disp_cmp and disp_jcc get scheduled at the latest. These
50891 instructions should be given the lowest priority in the
50892 scheduling process in Haifa scheduler to make sure they will be
50893 scheduled in the same dispatch window as the reference to them. */
50894 if (group == disp_jcc || group == disp_cmp)
50897 /* Check nonrestricted. */
50898 if (group == disp_no_group || group == disp_branch)
50901 /* Get last dispatch window. */
50902 if (window_list_next)
50903 window_list = window_list_next;
50905 if (window_list->window_num == 1)
50907 sum = window_list->prev->window_size + window_list->window_size;
50910 || (min_insn_size (insn) + sum) >= 48)
50911 /* Window 1 is full. Go for next window. */
50915 num_restrict = count_num_restricted (insn, window_list);
50917 if (num_restrict > num_allowable_groups[group])
50920 /* See if it fits in the first window. */
50921 if (window_list->window_num == 0)
50923 /* The first widow should have only single and double path
50925 if (path == path_double
50926 && (window_list->num_uops + 2) > MAX_INSN)
50928 else if (path != path_single)
50934 /* Add an instruction INSN with NUM_UOPS micro-operations to the
50935 dispatch window WINDOW_LIST. */
50938 add_insn_window (rtx_insn *insn, dispatch_windows *window_list, int num_uops)
50940 int byte_len = min_insn_size (insn);
50941 int num_insn = window_list->num_insn;
50943 sched_insn_info *window = window_list->window;
50944 enum dispatch_group group = get_insn_group (insn);
50945 enum insn_path path = get_insn_path (insn);
50946 int num_imm_operand;
50947 int num_imm32_operand;
50948 int num_imm64_operand;
50950 if (!window_list->violation && group != disp_cmp
50951 && !fits_dispatch_window (insn))
50952 window_list->violation = true;
50954 imm_size = get_num_immediates (insn, &num_imm_operand, &num_imm32_operand,
50955 &num_imm64_operand);
50957 /* Initialize window with new instruction. */
50958 window[num_insn].insn = insn;
50959 window[num_insn].byte_len = byte_len;
50960 window[num_insn].group = group;
50961 window[num_insn].path = path;
50962 window[num_insn].imm_bytes = imm_size;
50964 window_list->window_size += byte_len;
50965 window_list->num_insn = num_insn + 1;
50966 window_list->num_uops = window_list->num_uops + num_uops;
50967 window_list->imm_size += imm_size;
50968 window_list->num_imm += num_imm_operand;
50969 window_list->num_imm_32 += num_imm32_operand;
50970 window_list->num_imm_64 += num_imm64_operand;
50972 if (group == disp_store)
50973 window_list->num_stores += 1;
50974 else if (group == disp_load
50975 || group == disp_prefetch)
50976 window_list->num_loads += 1;
50977 else if (group == disp_load_store)
50979 window_list->num_stores += 1;
50980 window_list->num_loads += 1;
50984 /* Adds a scheduled instruction, INSN, to the current dispatch window.
50985 If the total bytes of instructions or the number of instructions in
50986 the window exceed allowable, it allocates a new window. */
50989 add_to_dispatch_window (rtx_insn *insn)
50992 dispatch_windows *window_list;
50993 dispatch_windows *next_list;
50994 dispatch_windows *window0_list;
50995 enum insn_path path;
50996 enum dispatch_group insn_group;
51004 if (INSN_CODE (insn) < 0)
51007 byte_len = min_insn_size (insn);
51008 window_list = dispatch_window_list;
51009 next_list = window_list->next;
51010 path = get_insn_path (insn);
51011 insn_group = get_insn_group (insn);
51013 /* Get the last dispatch window. */
51015 window_list = dispatch_window_list->next;
51017 if (path == path_single)
51019 else if (path == path_double)
51022 insn_num_uops = (int) path;
51024 /* If current window is full, get a new window.
51025 Window number zero is full, if MAX_INSN uops are scheduled in it.
51026 Window number one is full, if window zero's bytes plus window
51027 one's bytes is 32, or if the bytes of the new instruction added
51028 to the total makes it greater than 48, or it has already MAX_INSN
51029 instructions in it. */
51030 num_insn = window_list->num_insn;
51031 num_uops = window_list->num_uops;
51032 window_num = window_list->window_num;
51033 insn_fits = fits_dispatch_window (insn);
51035 if (num_insn >= MAX_INSN
51036 || num_uops + insn_num_uops > MAX_INSN
51039 window_num = ~window_num & 1;
51040 window_list = allocate_next_window (window_num);
51043 if (window_num == 0)
51045 add_insn_window (insn, window_list, insn_num_uops);
51046 if (window_list->num_insn >= MAX_INSN
51047 && insn_group == disp_branch)
51049 process_end_window ();
51053 else if (window_num == 1)
51055 window0_list = window_list->prev;
51056 sum = window0_list->window_size + window_list->window_size;
51058 || (byte_len + sum) >= 48)
51060 process_end_window ();
51061 window_list = dispatch_window_list;
51064 add_insn_window (insn, window_list, insn_num_uops);
51067 gcc_unreachable ();
51069 if (is_end_basic_block (insn_group))
51071 /* End of basic block is reached do end-basic-block process. */
51072 process_end_window ();
51077 /* Print the dispatch window, WINDOW_NUM, to FILE. */
51079 DEBUG_FUNCTION static void
51080 debug_dispatch_window_file (FILE *file, int window_num)
51082 dispatch_windows *list;
51085 if (window_num == 0)
51086 list = dispatch_window_list;
51088 list = dispatch_window_list1;
51090 fprintf (file, "Window #%d:\n", list->window_num);
51091 fprintf (file, " num_insn = %d, num_uops = %d, window_size = %d\n",
51092 list->num_insn, list->num_uops, list->window_size);
51093 fprintf (file, " num_imm = %d, num_imm_32 = %d, num_imm_64 = %d, imm_size = %d\n",
51094 list->num_imm, list->num_imm_32, list->num_imm_64, list->imm_size);
51096 fprintf (file, " num_loads = %d, num_stores = %d\n", list->num_loads,
51098 fprintf (file, " insn info:\n");
51100 for (i = 0; i < MAX_INSN; i++)
51102 if (!list->window[i].insn)
51104 fprintf (file, " group[%d] = %s, insn[%d] = %p, path[%d] = %d byte_len[%d] = %d, imm_bytes[%d] = %d\n",
51105 i, group_name[list->window[i].group],
51106 i, (void *)list->window[i].insn,
51107 i, list->window[i].path,
51108 i, list->window[i].byte_len,
51109 i, list->window[i].imm_bytes);
51113 /* Print to stdout a dispatch window. */
51115 DEBUG_FUNCTION void
51116 debug_dispatch_window (int window_num)
51118 debug_dispatch_window_file (stdout, window_num);
51121 /* Print INSN dispatch information to FILE. */
51123 DEBUG_FUNCTION static void
51124 debug_insn_dispatch_info_file (FILE *file, rtx_insn *insn)
51127 enum insn_path path;
51128 enum dispatch_group group;
51130 int num_imm_operand;
51131 int num_imm32_operand;
51132 int num_imm64_operand;
51134 if (INSN_CODE (insn) < 0)
51137 byte_len = min_insn_size (insn);
51138 path = get_insn_path (insn);
51139 group = get_insn_group (insn);
51140 imm_size = get_num_immediates (insn, &num_imm_operand, &num_imm32_operand,
51141 &num_imm64_operand);
51143 fprintf (file, " insn info:\n");
51144 fprintf (file, " group = %s, path = %d, byte_len = %d\n",
51145 group_name[group], path, byte_len);
51146 fprintf (file, " num_imm = %d, num_imm_32 = %d, num_imm_64 = %d, imm_size = %d\n",
51147 num_imm_operand, num_imm32_operand, num_imm64_operand, imm_size);
51150 /* Print to STDERR the status of the ready list with respect to
51151 dispatch windows. */
51153 DEBUG_FUNCTION void
51154 debug_ready_dispatch (void)
51157 int no_ready = number_in_ready ();
51159 fprintf (stdout, "Number of ready: %d\n", no_ready);
51161 for (i = 0; i < no_ready; i++)
51162 debug_insn_dispatch_info_file (stdout, get_ready_element (i));
51165 /* This routine is the driver of the dispatch scheduler. */
51168 do_dispatch (rtx_insn *insn, int mode)
51170 if (mode == DISPATCH_INIT)
51171 init_dispatch_sched ();
51172 else if (mode == ADD_TO_DISPATCH_WINDOW)
51173 add_to_dispatch_window (insn);
51176 /* Return TRUE if Dispatch Scheduling is supported. */
51179 has_dispatch (rtx_insn *insn, int action)
51181 if ((TARGET_BDVER1 || TARGET_BDVER2 || TARGET_BDVER3 || TARGET_BDVER4)
51182 && flag_dispatch_scheduler)
51188 case IS_DISPATCH_ON:
51193 return is_cmp (insn);
51195 case DISPATCH_VIOLATION:
51196 return dispatch_violation ();
51198 case FITS_DISPATCH_WINDOW:
51199 return fits_dispatch_window (insn);
51205 /* Implementation of reassociation_width target hook used by
51206 reassoc phase to identify parallelism level in reassociated
51207 tree. Statements tree_code is passed in OPC. Arguments type
51210 Currently parallel reassociation is enabled for Atom
51211 processors only and we set reassociation width to be 2
51212 because Atom may issue up to 2 instructions per cycle.
51214 Return value should be fixed if parallel reassociation is
51215 enabled for other processors. */
51218 ix86_reassociation_width (unsigned int, machine_mode mode)
51221 if (VECTOR_MODE_P (mode))
51223 if (TARGET_VECTOR_PARALLEL_EXECUTION)
51230 if (INTEGRAL_MODE_P (mode) && TARGET_REASSOC_INT_TO_PARALLEL)
51232 else if (FLOAT_MODE_P (mode) && TARGET_REASSOC_FP_TO_PARALLEL)
51238 /* ??? No autovectorization into MMX or 3DNOW until we can reliably
51239 place emms and femms instructions. */
51241 static machine_mode
51242 ix86_preferred_simd_mode (machine_mode mode)
51250 return TARGET_AVX512BW ? V64QImode :
51251 (TARGET_AVX && !TARGET_PREFER_AVX128) ? V32QImode : V16QImode;
51253 return TARGET_AVX512BW ? V32HImode :
51254 (TARGET_AVX && !TARGET_PREFER_AVX128) ? V16HImode : V8HImode;
51256 return TARGET_AVX512F ? V16SImode :
51257 (TARGET_AVX && !TARGET_PREFER_AVX128) ? V8SImode : V4SImode;
51259 return TARGET_AVX512F ? V8DImode :
51260 (TARGET_AVX && !TARGET_PREFER_AVX128) ? V4DImode : V2DImode;
51263 if (TARGET_AVX512F)
51265 else if (TARGET_AVX && !TARGET_PREFER_AVX128)
51271 if (!TARGET_VECTORIZE_DOUBLE)
51273 else if (TARGET_AVX512F)
51275 else if (TARGET_AVX && !TARGET_PREFER_AVX128)
51277 else if (TARGET_SSE2)
51286 /* If AVX is enabled then try vectorizing with both 256bit and 128bit
51287 vectors. If AVX512F is enabled then try vectorizing with 512bit,
51288 256bit and 128bit vectors. */
51290 static unsigned int
51291 ix86_autovectorize_vector_sizes (void)
51293 return TARGET_AVX512F ? 64 | 32 | 16 :
51294 (TARGET_AVX && !TARGET_PREFER_AVX128) ? 32 | 16 : 0;
51299 /* Return class of registers which could be used for pseudo of MODE
51300 and of class RCLASS for spilling instead of memory. Return NO_REGS
51301 if it is not possible or non-profitable. */
51303 ix86_spill_class (reg_class_t rclass, machine_mode mode)
51305 if (TARGET_SSE && TARGET_GENERAL_REGS_SSE_SPILL && ! TARGET_MMX
51306 && (mode == SImode || (TARGET_64BIT && mode == DImode))
51307 && rclass != NO_REGS && INTEGER_CLASS_P (rclass))
51308 return ALL_SSE_REGS;
51312 /* Implement targetm.vectorize.init_cost. */
51315 ix86_init_cost (struct loop *)
51317 unsigned *cost = XNEWVEC (unsigned, 3);
51318 cost[vect_prologue] = cost[vect_body] = cost[vect_epilogue] = 0;
51322 /* Implement targetm.vectorize.add_stmt_cost. */
51325 ix86_add_stmt_cost (void *data, int count, enum vect_cost_for_stmt kind,
51326 struct _stmt_vec_info *stmt_info, int misalign,
51327 enum vect_cost_model_location where)
51329 unsigned *cost = (unsigned *) data;
51330 unsigned retval = 0;
51332 tree vectype = stmt_info ? stmt_vectype (stmt_info) : NULL_TREE;
51333 int stmt_cost = ix86_builtin_vectorization_cost (kind, vectype, misalign);
51335 /* Statements in an inner loop relative to the loop being
51336 vectorized are weighted more heavily. The value here is
51337 arbitrary and could potentially be improved with analysis. */
51338 if (where == vect_body && stmt_info && stmt_in_inner_loop_p (stmt_info))
51339 count *= 50; /* FIXME. */
51341 retval = (unsigned) (count * stmt_cost);
51343 /* We need to multiply all vector stmt cost by 1.7 (estimated cost)
51344 for Silvermont as it has out of order integer pipeline and can execute
51345 2 scalar instruction per tick, but has in order SIMD pipeline. */
51346 if (TARGET_SILVERMONT || TARGET_INTEL)
51347 if (stmt_info && stmt_info->stmt)
51349 tree lhs_op = gimple_get_lhs (stmt_info->stmt);
51350 if (lhs_op && TREE_CODE (TREE_TYPE (lhs_op)) == INTEGER_TYPE)
51351 retval = (retval * 17) / 10;
51354 cost[where] += retval;
51359 /* Implement targetm.vectorize.finish_cost. */
51362 ix86_finish_cost (void *data, unsigned *prologue_cost,
51363 unsigned *body_cost, unsigned *epilogue_cost)
51365 unsigned *cost = (unsigned *) data;
51366 *prologue_cost = cost[vect_prologue];
51367 *body_cost = cost[vect_body];
51368 *epilogue_cost = cost[vect_epilogue];
51371 /* Implement targetm.vectorize.destroy_cost_data. */
51374 ix86_destroy_cost_data (void *data)
51379 /* Validate target specific memory model bits in VAL. */
51381 static unsigned HOST_WIDE_INT
51382 ix86_memmodel_check (unsigned HOST_WIDE_INT val)
51384 enum memmodel model = memmodel_from_int (val);
51387 if (val & ~(unsigned HOST_WIDE_INT)(IX86_HLE_ACQUIRE|IX86_HLE_RELEASE
51389 || ((val & IX86_HLE_ACQUIRE) && (val & IX86_HLE_RELEASE)))
51391 warning (OPT_Winvalid_memory_model,
51392 "Unknown architecture specific memory model");
51393 return MEMMODEL_SEQ_CST;
51395 strong = (is_mm_acq_rel (model) || is_mm_seq_cst (model));
51396 if (val & IX86_HLE_ACQUIRE && !(is_mm_acquire (model) || strong))
51398 warning (OPT_Winvalid_memory_model,
51399 "HLE_ACQUIRE not used with ACQUIRE or stronger memory model");
51400 return MEMMODEL_SEQ_CST | IX86_HLE_ACQUIRE;
51402 if (val & IX86_HLE_RELEASE && !(is_mm_release (model) || strong))
51404 warning (OPT_Winvalid_memory_model,
51405 "HLE_RELEASE not used with RELEASE or stronger memory model");
51406 return MEMMODEL_SEQ_CST | IX86_HLE_RELEASE;
51411 /* Set CLONEI->vecsize_mangle, CLONEI->vecsize_int,
51412 CLONEI->vecsize_float and if CLONEI->simdlen is 0, also
51413 CLONEI->simdlen. Return 0 if SIMD clones shouldn't be emitted,
51414 or number of vecsize_mangle variants that should be emitted. */
51417 ix86_simd_clone_compute_vecsize_and_simdlen (struct cgraph_node *node,
51418 struct cgraph_simd_clone *clonei,
51419 tree base_type, int num)
51423 if (clonei->simdlen
51424 && (clonei->simdlen < 2
51425 || clonei->simdlen > 16
51426 || (clonei->simdlen & (clonei->simdlen - 1)) != 0))
51428 warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
51429 "unsupported simdlen %d", clonei->simdlen);
51433 tree ret_type = TREE_TYPE (TREE_TYPE (node->decl));
51434 if (TREE_CODE (ret_type) != VOID_TYPE)
51435 switch (TYPE_MODE (ret_type))
51447 warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
51448 "unsupported return type %qT for simd\n", ret_type);
51455 for (t = DECL_ARGUMENTS (node->decl), i = 0; t; t = DECL_CHAIN (t), i++)
51456 /* FIXME: Shouldn't we allow such arguments if they are uniform? */
51457 switch (TYPE_MODE (TREE_TYPE (t)))
51469 warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
51470 "unsupported argument type %qT for simd\n", TREE_TYPE (t));
51474 if (clonei->cilk_elemental)
51476 /* Parse here processor clause. If not present, default to 'b'. */
51477 clonei->vecsize_mangle = 'b';
51479 else if (!TREE_PUBLIC (node->decl))
51481 /* If the function isn't exported, we can pick up just one ISA
51484 clonei->vecsize_mangle = 'd';
51485 else if (TARGET_AVX)
51486 clonei->vecsize_mangle = 'c';
51488 clonei->vecsize_mangle = 'b';
51493 clonei->vecsize_mangle = "bcd"[num];
51496 switch (clonei->vecsize_mangle)
51499 clonei->vecsize_int = 128;
51500 clonei->vecsize_float = 128;
51503 clonei->vecsize_int = 128;
51504 clonei->vecsize_float = 256;
51507 clonei->vecsize_int = 256;
51508 clonei->vecsize_float = 256;
51511 if (clonei->simdlen == 0)
51513 if (SCALAR_INT_MODE_P (TYPE_MODE (base_type)))
51514 clonei->simdlen = clonei->vecsize_int;
51516 clonei->simdlen = clonei->vecsize_float;
51517 clonei->simdlen /= GET_MODE_BITSIZE (TYPE_MODE (base_type));
51518 if (clonei->simdlen > 16)
51519 clonei->simdlen = 16;
51524 /* Add target attribute to SIMD clone NODE if needed. */
51527 ix86_simd_clone_adjust (struct cgraph_node *node)
51529 const char *str = NULL;
51530 gcc_assert (node->decl == cfun->decl);
51531 switch (node->simdclone->vecsize_mangle)
51546 gcc_unreachable ();
51551 tree args = build_tree_list (NULL_TREE, build_string (strlen (str), str));
51552 bool ok = ix86_valid_target_attribute_p (node->decl, NULL, args, 0);
51555 ix86_reset_previous_fndecl ();
51556 ix86_set_current_function (node->decl);
51559 /* If SIMD clone NODE can't be used in a vectorized loop
51560 in current function, return -1, otherwise return a badness of using it
51561 (0 if it is most desirable from vecsize_mangle point of view, 1
51562 slightly less desirable, etc.). */
51565 ix86_simd_clone_usable (struct cgraph_node *node)
51567 switch (node->simdclone->vecsize_mangle)
51574 return TARGET_AVX2 ? 2 : 1;
51578 return TARGET_AVX2 ? 1 : 0;
51585 gcc_unreachable ();
51589 /* This function adjusts the unroll factor based on
51590 the hardware capabilities. For ex, bdver3 has
51591 a loop buffer which makes unrolling of smaller
51592 loops less important. This function decides the
51593 unroll factor using number of memory references
51594 (value 32 is used) as a heuristic. */
51597 ix86_loop_unroll_adjust (unsigned nunroll, struct loop *loop)
51602 unsigned mem_count = 0;
51604 if (!TARGET_ADJUST_UNROLL)
51607 /* Count the number of memory references within the loop body.
51608 This value determines the unrolling factor for bdver3 and bdver4
51610 subrtx_iterator::array_type array;
51611 bbs = get_loop_body (loop);
51612 for (i = 0; i < loop->num_nodes; i++)
51613 FOR_BB_INSNS (bbs[i], insn)
51614 if (NONDEBUG_INSN_P (insn))
51615 FOR_EACH_SUBRTX (iter, array, PATTERN (insn), NONCONST)
51616 if (const_rtx x = *iter)
51619 machine_mode mode = GET_MODE (x);
51620 unsigned int n_words = GET_MODE_SIZE (mode) / UNITS_PER_WORD;
51628 if (mem_count && mem_count <=32)
51629 return 32/mem_count;
51635 /* Implement TARGET_FLOAT_EXCEPTIONS_ROUNDING_SUPPORTED_P. */
51638 ix86_float_exceptions_rounding_supported_p (void)
51640 /* For x87 floating point with standard excess precision handling,
51641 there is no adddf3 pattern (since x87 floating point only has
51642 XFmode operations) so the default hook implementation gets this
51644 return TARGET_80387 || TARGET_SSE_MATH;
51647 /* Implement TARGET_ATOMIC_ASSIGN_EXPAND_FENV. */
51650 ix86_atomic_assign_expand_fenv (tree *hold, tree *clear, tree *update)
51652 if (!TARGET_80387 && !TARGET_SSE_MATH)
51654 tree exceptions_var = create_tmp_var (integer_type_node);
51657 tree fenv_index_type = build_index_type (size_int (6));
51658 tree fenv_type = build_array_type (unsigned_type_node, fenv_index_type);
51659 tree fenv_var = create_tmp_var (fenv_type);
51660 mark_addressable (fenv_var);
51661 tree fenv_ptr = build_pointer_type (fenv_type);
51662 tree fenv_addr = build1 (ADDR_EXPR, fenv_ptr, fenv_var);
51663 fenv_addr = fold_convert (ptr_type_node, fenv_addr);
51664 tree fnstenv = ix86_builtins[IX86_BUILTIN_FNSTENV];
51665 tree fldenv = ix86_builtins[IX86_BUILTIN_FLDENV];
51666 tree fnstsw = ix86_builtins[IX86_BUILTIN_FNSTSW];
51667 tree fnclex = ix86_builtins[IX86_BUILTIN_FNCLEX];
51668 tree hold_fnstenv = build_call_expr (fnstenv, 1, fenv_addr);
51669 tree hold_fnclex = build_call_expr (fnclex, 0);
51670 *hold = build2 (COMPOUND_EXPR, void_type_node, hold_fnstenv,
51672 *clear = build_call_expr (fnclex, 0);
51673 tree sw_var = create_tmp_var (short_unsigned_type_node);
51674 tree fnstsw_call = build_call_expr (fnstsw, 0);
51675 tree sw_mod = build2 (MODIFY_EXPR, short_unsigned_type_node,
51676 sw_var, fnstsw_call);
51677 tree exceptions_x87 = fold_convert (integer_type_node, sw_var);
51678 tree update_mod = build2 (MODIFY_EXPR, integer_type_node,
51679 exceptions_var, exceptions_x87);
51680 *update = build2 (COMPOUND_EXPR, integer_type_node,
51681 sw_mod, update_mod);
51682 tree update_fldenv = build_call_expr (fldenv, 1, fenv_addr);
51683 *update = build2 (COMPOUND_EXPR, void_type_node, *update, update_fldenv);
51685 if (TARGET_SSE_MATH)
51687 tree mxcsr_orig_var = create_tmp_var (unsigned_type_node);
51688 tree mxcsr_mod_var = create_tmp_var (unsigned_type_node);
51689 tree stmxcsr = ix86_builtins[IX86_BUILTIN_STMXCSR];
51690 tree ldmxcsr = ix86_builtins[IX86_BUILTIN_LDMXCSR];
51691 tree stmxcsr_hold_call = build_call_expr (stmxcsr, 0);
51692 tree hold_assign_orig = build2 (MODIFY_EXPR, unsigned_type_node,
51693 mxcsr_orig_var, stmxcsr_hold_call);
51694 tree hold_mod_val = build2 (BIT_IOR_EXPR, unsigned_type_node,
51696 build_int_cst (unsigned_type_node, 0x1f80));
51697 hold_mod_val = build2 (BIT_AND_EXPR, unsigned_type_node, hold_mod_val,
51698 build_int_cst (unsigned_type_node, 0xffffffc0));
51699 tree hold_assign_mod = build2 (MODIFY_EXPR, unsigned_type_node,
51700 mxcsr_mod_var, hold_mod_val);
51701 tree ldmxcsr_hold_call = build_call_expr (ldmxcsr, 1, mxcsr_mod_var);
51702 tree hold_all = build2 (COMPOUND_EXPR, unsigned_type_node,
51703 hold_assign_orig, hold_assign_mod);
51704 hold_all = build2 (COMPOUND_EXPR, void_type_node, hold_all,
51705 ldmxcsr_hold_call);
51707 *hold = build2 (COMPOUND_EXPR, void_type_node, *hold, hold_all);
51710 tree ldmxcsr_clear_call = build_call_expr (ldmxcsr, 1, mxcsr_mod_var);
51712 *clear = build2 (COMPOUND_EXPR, void_type_node, *clear,
51713 ldmxcsr_clear_call);
51715 *clear = ldmxcsr_clear_call;
51716 tree stxmcsr_update_call = build_call_expr (stmxcsr, 0);
51717 tree exceptions_sse = fold_convert (integer_type_node,
51718 stxmcsr_update_call);
51721 tree exceptions_mod = build2 (BIT_IOR_EXPR, integer_type_node,
51722 exceptions_var, exceptions_sse);
51723 tree exceptions_assign = build2 (MODIFY_EXPR, integer_type_node,
51724 exceptions_var, exceptions_mod);
51725 *update = build2 (COMPOUND_EXPR, integer_type_node, *update,
51726 exceptions_assign);
51729 *update = build2 (MODIFY_EXPR, integer_type_node,
51730 exceptions_var, exceptions_sse);
51731 tree ldmxcsr_update_call = build_call_expr (ldmxcsr, 1, mxcsr_orig_var);
51732 *update = build2 (COMPOUND_EXPR, void_type_node, *update,
51733 ldmxcsr_update_call);
51735 tree atomic_feraiseexcept
51736 = builtin_decl_implicit (BUILT_IN_ATOMIC_FERAISEEXCEPT);
51737 tree atomic_feraiseexcept_call = build_call_expr (atomic_feraiseexcept,
51738 1, exceptions_var);
51739 *update = build2 (COMPOUND_EXPR, void_type_node, *update,
51740 atomic_feraiseexcept_call);
51743 /* Return mode to be used for bounds or VOIDmode
51744 if bounds are not supported. */
51746 static enum machine_mode
51747 ix86_mpx_bound_mode ()
51749 /* Do not support pointer checker if MPX
51753 if (flag_check_pointer_bounds)
51754 warning (0, "Pointer Checker requires MPX support on this target."
51755 " Use -mmpx options to enable MPX.");
51762 /* Return constant used to statically initialize constant bounds.
51764 This function is used to create special bound values. For now
51765 only INIT bounds and NONE bounds are expected. More special
51766 values may be added later. */
51769 ix86_make_bounds_constant (HOST_WIDE_INT lb, HOST_WIDE_INT ub)
51771 tree low = lb ? build_minus_one_cst (pointer_sized_int_node)
51772 : build_zero_cst (pointer_sized_int_node);
51773 tree high = ub ? build_zero_cst (pointer_sized_int_node)
51774 : build_minus_one_cst (pointer_sized_int_node);
51776 /* This function is supposed to be used to create INIT and
51777 NONE bounds only. */
51778 gcc_assert ((lb == 0 && ub == -1)
51779 || (lb == -1 && ub == 0));
51781 return build_complex (NULL, low, high);
51784 /* Generate a list of statements STMTS to initialize pointer bounds
51785 variable VAR with bounds LB and UB. Return the number of generated
51789 ix86_initialize_bounds (tree var, tree lb, tree ub, tree *stmts)
51791 tree bnd_ptr = build_pointer_type (pointer_sized_int_node);
51792 tree lhs, modify, var_p;
51794 ub = build1 (BIT_NOT_EXPR, pointer_sized_int_node, ub);
51795 var_p = fold_convert (bnd_ptr, build_fold_addr_expr (var));
51797 lhs = build1 (INDIRECT_REF, pointer_sized_int_node, var_p);
51798 modify = build2 (MODIFY_EXPR, TREE_TYPE (lhs), lhs, lb);
51799 append_to_statement_list (modify, stmts);
51801 lhs = build1 (INDIRECT_REF, pointer_sized_int_node,
51802 build2 (POINTER_PLUS_EXPR, bnd_ptr, var_p,
51803 TYPE_SIZE_UNIT (pointer_sized_int_node)));
51804 modify = build2 (MODIFY_EXPR, TREE_TYPE (lhs), lhs, ub);
51805 append_to_statement_list (modify, stmts);
51810 #if !TARGET_MACHO && !TARGET_DLLIMPORT_DECL_ATTRIBUTES
51811 /* For i386, common symbol is local only for non-PIE binaries. For
51812 x86-64, common symbol is local only for non-PIE binaries or linker
51813 supports copy reloc in PIE binaries. */
51816 ix86_binds_local_p (const_tree exp)
51818 return default_binds_local_p_3 (exp, flag_shlib != 0, true, true,
51821 && HAVE_LD_PIE_COPYRELOC != 0)));
51825 /* If MEM is in the form of [base+offset], extract the two parts
51826 of address and set to BASE and OFFSET, otherwise return false. */
51829 extract_base_offset_in_addr (rtx mem, rtx *base, rtx *offset)
51833 gcc_assert (MEM_P (mem));
51835 addr = XEXP (mem, 0);
51837 if (GET_CODE (addr) == CONST)
51838 addr = XEXP (addr, 0);
51840 if (REG_P (addr) || GET_CODE (addr) == SYMBOL_REF)
51843 *offset = const0_rtx;
51847 if (GET_CODE (addr) == PLUS
51848 && (REG_P (XEXP (addr, 0))
51849 || GET_CODE (XEXP (addr, 0)) == SYMBOL_REF)
51850 && CONST_INT_P (XEXP (addr, 1)))
51852 *base = XEXP (addr, 0);
51853 *offset = XEXP (addr, 1);
51860 /* Given OPERANDS of consecutive load/store, check if we can merge
51861 them into move multiple. LOAD is true if they are load instructions.
51862 MODE is the mode of memory operands. */
51865 ix86_operands_ok_for_move_multiple (rtx *operands, bool load,
51866 enum machine_mode mode)
51868 HOST_WIDE_INT offval_1, offval_2, msize;
51869 rtx mem_1, mem_2, reg_1, reg_2, base_1, base_2, offset_1, offset_2;
51873 mem_1 = operands[1];
51874 mem_2 = operands[3];
51875 reg_1 = operands[0];
51876 reg_2 = operands[2];
51880 mem_1 = operands[0];
51881 mem_2 = operands[2];
51882 reg_1 = operands[1];
51883 reg_2 = operands[3];
51886 gcc_assert (REG_P (reg_1) && REG_P (reg_2));
51888 if (REGNO (reg_1) != REGNO (reg_2))
51891 /* Check if the addresses are in the form of [base+offset]. */
51892 if (!extract_base_offset_in_addr (mem_1, &base_1, &offset_1))
51894 if (!extract_base_offset_in_addr (mem_2, &base_2, &offset_2))
51897 /* Check if the bases are the same. */
51898 if (!rtx_equal_p (base_1, base_2))
51901 offval_1 = INTVAL (offset_1);
51902 offval_2 = INTVAL (offset_2);
51903 msize = GET_MODE_SIZE (mode);
51904 /* Check if mem_1 is adjacent to mem_2 and mem_1 has lower address. */
51905 if (offval_1 + msize != offval_2)
51911 /* Initialize the GCC target structure. */
51912 #undef TARGET_RETURN_IN_MEMORY
51913 #define TARGET_RETURN_IN_MEMORY ix86_return_in_memory
51915 #undef TARGET_LEGITIMIZE_ADDRESS
51916 #define TARGET_LEGITIMIZE_ADDRESS ix86_legitimize_address
51918 #undef TARGET_ATTRIBUTE_TABLE
51919 #define TARGET_ATTRIBUTE_TABLE ix86_attribute_table
51920 #undef TARGET_FUNCTION_ATTRIBUTE_INLINABLE_P
51921 #define TARGET_FUNCTION_ATTRIBUTE_INLINABLE_P hook_bool_const_tree_true
51922 #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
51923 # undef TARGET_MERGE_DECL_ATTRIBUTES
51924 # define TARGET_MERGE_DECL_ATTRIBUTES merge_dllimport_decl_attributes
51927 #undef TARGET_COMP_TYPE_ATTRIBUTES
51928 #define TARGET_COMP_TYPE_ATTRIBUTES ix86_comp_type_attributes
51930 #undef TARGET_INIT_BUILTINS
51931 #define TARGET_INIT_BUILTINS ix86_init_builtins
51932 #undef TARGET_BUILTIN_DECL
51933 #define TARGET_BUILTIN_DECL ix86_builtin_decl
51934 #undef TARGET_EXPAND_BUILTIN
51935 #define TARGET_EXPAND_BUILTIN ix86_expand_builtin
51937 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION
51938 #define TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION \
51939 ix86_builtin_vectorized_function
51941 #undef TARGET_VECTORIZE_BUILTIN_TM_LOAD
51942 #define TARGET_VECTORIZE_BUILTIN_TM_LOAD ix86_builtin_tm_load
51944 #undef TARGET_VECTORIZE_BUILTIN_TM_STORE
51945 #define TARGET_VECTORIZE_BUILTIN_TM_STORE ix86_builtin_tm_store
51947 #undef TARGET_VECTORIZE_BUILTIN_GATHER
51948 #define TARGET_VECTORIZE_BUILTIN_GATHER ix86_vectorize_builtin_gather
51950 #undef TARGET_BUILTIN_RECIPROCAL
51951 #define TARGET_BUILTIN_RECIPROCAL ix86_builtin_reciprocal
51953 #undef TARGET_ASM_FUNCTION_EPILOGUE
51954 #define TARGET_ASM_FUNCTION_EPILOGUE ix86_output_function_epilogue
51956 #undef TARGET_ENCODE_SECTION_INFO
51957 #ifndef SUBTARGET_ENCODE_SECTION_INFO
51958 #define TARGET_ENCODE_SECTION_INFO ix86_encode_section_info
51960 #define TARGET_ENCODE_SECTION_INFO SUBTARGET_ENCODE_SECTION_INFO
51963 #undef TARGET_ASM_OPEN_PAREN
51964 #define TARGET_ASM_OPEN_PAREN ""
51965 #undef TARGET_ASM_CLOSE_PAREN
51966 #define TARGET_ASM_CLOSE_PAREN ""
51968 #undef TARGET_ASM_BYTE_OP
51969 #define TARGET_ASM_BYTE_OP ASM_BYTE
51971 #undef TARGET_ASM_ALIGNED_HI_OP
51972 #define TARGET_ASM_ALIGNED_HI_OP ASM_SHORT
51973 #undef TARGET_ASM_ALIGNED_SI_OP
51974 #define TARGET_ASM_ALIGNED_SI_OP ASM_LONG
51976 #undef TARGET_ASM_ALIGNED_DI_OP
51977 #define TARGET_ASM_ALIGNED_DI_OP ASM_QUAD
51980 #undef TARGET_PROFILE_BEFORE_PROLOGUE
51981 #define TARGET_PROFILE_BEFORE_PROLOGUE ix86_profile_before_prologue
51983 #undef TARGET_MANGLE_DECL_ASSEMBLER_NAME
51984 #define TARGET_MANGLE_DECL_ASSEMBLER_NAME ix86_mangle_decl_assembler_name
51986 #undef TARGET_ASM_UNALIGNED_HI_OP
51987 #define TARGET_ASM_UNALIGNED_HI_OP TARGET_ASM_ALIGNED_HI_OP
51988 #undef TARGET_ASM_UNALIGNED_SI_OP
51989 #define TARGET_ASM_UNALIGNED_SI_OP TARGET_ASM_ALIGNED_SI_OP
51990 #undef TARGET_ASM_UNALIGNED_DI_OP
51991 #define TARGET_ASM_UNALIGNED_DI_OP TARGET_ASM_ALIGNED_DI_OP
51993 #undef TARGET_PRINT_OPERAND
51994 #define TARGET_PRINT_OPERAND ix86_print_operand
51995 #undef TARGET_PRINT_OPERAND_ADDRESS
51996 #define TARGET_PRINT_OPERAND_ADDRESS ix86_print_operand_address
51997 #undef TARGET_PRINT_OPERAND_PUNCT_VALID_P
51998 #define TARGET_PRINT_OPERAND_PUNCT_VALID_P ix86_print_operand_punct_valid_p
51999 #undef TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA
52000 #define TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA i386_asm_output_addr_const_extra
52002 #undef TARGET_SCHED_INIT_GLOBAL
52003 #define TARGET_SCHED_INIT_GLOBAL ix86_sched_init_global
52004 #undef TARGET_SCHED_ADJUST_COST
52005 #define TARGET_SCHED_ADJUST_COST ix86_adjust_cost
52006 #undef TARGET_SCHED_ISSUE_RATE
52007 #define TARGET_SCHED_ISSUE_RATE ix86_issue_rate
52008 #undef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD
52009 #define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD \
52010 ia32_multipass_dfa_lookahead
52011 #undef TARGET_SCHED_MACRO_FUSION_P
52012 #define TARGET_SCHED_MACRO_FUSION_P ix86_macro_fusion_p
52013 #undef TARGET_SCHED_MACRO_FUSION_PAIR_P
52014 #define TARGET_SCHED_MACRO_FUSION_PAIR_P ix86_macro_fusion_pair_p
52016 #undef TARGET_FUNCTION_OK_FOR_SIBCALL
52017 #define TARGET_FUNCTION_OK_FOR_SIBCALL ix86_function_ok_for_sibcall
52019 #undef TARGET_MEMMODEL_CHECK
52020 #define TARGET_MEMMODEL_CHECK ix86_memmodel_check
52022 #undef TARGET_ATOMIC_ASSIGN_EXPAND_FENV
52023 #define TARGET_ATOMIC_ASSIGN_EXPAND_FENV ix86_atomic_assign_expand_fenv
52026 #undef TARGET_HAVE_TLS
52027 #define TARGET_HAVE_TLS true
52029 #undef TARGET_CANNOT_FORCE_CONST_MEM
52030 #define TARGET_CANNOT_FORCE_CONST_MEM ix86_cannot_force_const_mem
52031 #undef TARGET_USE_BLOCKS_FOR_CONSTANT_P
52032 #define TARGET_USE_BLOCKS_FOR_CONSTANT_P hook_bool_mode_const_rtx_true
52034 #undef TARGET_DELEGITIMIZE_ADDRESS
52035 #define TARGET_DELEGITIMIZE_ADDRESS ix86_delegitimize_address
52037 #undef TARGET_MS_BITFIELD_LAYOUT_P
52038 #define TARGET_MS_BITFIELD_LAYOUT_P ix86_ms_bitfield_layout_p
52041 #undef TARGET_BINDS_LOCAL_P
52042 #define TARGET_BINDS_LOCAL_P darwin_binds_local_p
52044 #undef TARGET_BINDS_LOCAL_P
52045 #define TARGET_BINDS_LOCAL_P ix86_binds_local_p
52047 #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
52048 #undef TARGET_BINDS_LOCAL_P
52049 #define TARGET_BINDS_LOCAL_P i386_pe_binds_local_p
52052 #undef TARGET_ASM_OUTPUT_MI_THUNK
52053 #define TARGET_ASM_OUTPUT_MI_THUNK x86_output_mi_thunk
52054 #undef TARGET_ASM_CAN_OUTPUT_MI_THUNK
52055 #define TARGET_ASM_CAN_OUTPUT_MI_THUNK x86_can_output_mi_thunk
52057 #undef TARGET_ASM_FILE_START
52058 #define TARGET_ASM_FILE_START x86_file_start
52060 #undef TARGET_OPTION_OVERRIDE
52061 #define TARGET_OPTION_OVERRIDE ix86_option_override
52063 #undef TARGET_REGISTER_MOVE_COST
52064 #define TARGET_REGISTER_MOVE_COST ix86_register_move_cost
52065 #undef TARGET_MEMORY_MOVE_COST
52066 #define TARGET_MEMORY_MOVE_COST ix86_memory_move_cost
52067 #undef TARGET_RTX_COSTS
52068 #define TARGET_RTX_COSTS ix86_rtx_costs
52069 #undef TARGET_ADDRESS_COST
52070 #define TARGET_ADDRESS_COST ix86_address_cost
52072 #undef TARGET_FIXED_CONDITION_CODE_REGS
52073 #define TARGET_FIXED_CONDITION_CODE_REGS ix86_fixed_condition_code_regs
52074 #undef TARGET_CC_MODES_COMPATIBLE
52075 #define TARGET_CC_MODES_COMPATIBLE ix86_cc_modes_compatible
52077 #undef TARGET_MACHINE_DEPENDENT_REORG
52078 #define TARGET_MACHINE_DEPENDENT_REORG ix86_reorg
52080 #undef TARGET_BUILTIN_SETJMP_FRAME_VALUE
52081 #define TARGET_BUILTIN_SETJMP_FRAME_VALUE ix86_builtin_setjmp_frame_value
52083 #undef TARGET_BUILD_BUILTIN_VA_LIST
52084 #define TARGET_BUILD_BUILTIN_VA_LIST ix86_build_builtin_va_list
52086 #undef TARGET_FOLD_BUILTIN
52087 #define TARGET_FOLD_BUILTIN ix86_fold_builtin
52089 #undef TARGET_COMPARE_VERSION_PRIORITY
52090 #define TARGET_COMPARE_VERSION_PRIORITY ix86_compare_version_priority
52092 #undef TARGET_GENERATE_VERSION_DISPATCHER_BODY
52093 #define TARGET_GENERATE_VERSION_DISPATCHER_BODY \
52094 ix86_generate_version_dispatcher_body
52096 #undef TARGET_GET_FUNCTION_VERSIONS_DISPATCHER
52097 #define TARGET_GET_FUNCTION_VERSIONS_DISPATCHER \
52098 ix86_get_function_versions_dispatcher
52100 #undef TARGET_ENUM_VA_LIST_P
52101 #define TARGET_ENUM_VA_LIST_P ix86_enum_va_list
52103 #undef TARGET_FN_ABI_VA_LIST
52104 #define TARGET_FN_ABI_VA_LIST ix86_fn_abi_va_list
52106 #undef TARGET_CANONICAL_VA_LIST_TYPE
52107 #define TARGET_CANONICAL_VA_LIST_TYPE ix86_canonical_va_list_type
52109 #undef TARGET_EXPAND_BUILTIN_VA_START
52110 #define TARGET_EXPAND_BUILTIN_VA_START ix86_va_start
52112 #undef TARGET_MD_ASM_ADJUST
52113 #define TARGET_MD_ASM_ADJUST ix86_md_asm_adjust
52115 #undef TARGET_PROMOTE_PROTOTYPES
52116 #define TARGET_PROMOTE_PROTOTYPES hook_bool_const_tree_true
52117 #undef TARGET_SETUP_INCOMING_VARARGS
52118 #define TARGET_SETUP_INCOMING_VARARGS ix86_setup_incoming_varargs
52119 #undef TARGET_MUST_PASS_IN_STACK
52120 #define TARGET_MUST_PASS_IN_STACK ix86_must_pass_in_stack
52121 #undef TARGET_FUNCTION_ARG_ADVANCE
52122 #define TARGET_FUNCTION_ARG_ADVANCE ix86_function_arg_advance
52123 #undef TARGET_FUNCTION_ARG
52124 #define TARGET_FUNCTION_ARG ix86_function_arg
52125 #undef TARGET_INIT_PIC_REG
52126 #define TARGET_INIT_PIC_REG ix86_init_pic_reg
52127 #undef TARGET_USE_PSEUDO_PIC_REG
52128 #define TARGET_USE_PSEUDO_PIC_REG ix86_use_pseudo_pic_reg
52129 #undef TARGET_FUNCTION_ARG_BOUNDARY
52130 #define TARGET_FUNCTION_ARG_BOUNDARY ix86_function_arg_boundary
52131 #undef TARGET_PASS_BY_REFERENCE
52132 #define TARGET_PASS_BY_REFERENCE ix86_pass_by_reference
52133 #undef TARGET_INTERNAL_ARG_POINTER
52134 #define TARGET_INTERNAL_ARG_POINTER ix86_internal_arg_pointer
52135 #undef TARGET_UPDATE_STACK_BOUNDARY
52136 #define TARGET_UPDATE_STACK_BOUNDARY ix86_update_stack_boundary
52137 #undef TARGET_GET_DRAP_RTX
52138 #define TARGET_GET_DRAP_RTX ix86_get_drap_rtx
52139 #undef TARGET_STRICT_ARGUMENT_NAMING
52140 #define TARGET_STRICT_ARGUMENT_NAMING hook_bool_CUMULATIVE_ARGS_true
52141 #undef TARGET_STATIC_CHAIN
52142 #define TARGET_STATIC_CHAIN ix86_static_chain
52143 #undef TARGET_TRAMPOLINE_INIT
52144 #define TARGET_TRAMPOLINE_INIT ix86_trampoline_init
52145 #undef TARGET_RETURN_POPS_ARGS
52146 #define TARGET_RETURN_POPS_ARGS ix86_return_pops_args
52148 #undef TARGET_LEGITIMATE_COMBINED_INSN
52149 #define TARGET_LEGITIMATE_COMBINED_INSN ix86_legitimate_combined_insn
52151 #undef TARGET_ASAN_SHADOW_OFFSET
52152 #define TARGET_ASAN_SHADOW_OFFSET ix86_asan_shadow_offset
52154 #undef TARGET_GIMPLIFY_VA_ARG_EXPR
52155 #define TARGET_GIMPLIFY_VA_ARG_EXPR ix86_gimplify_va_arg
52157 #undef TARGET_SCALAR_MODE_SUPPORTED_P
52158 #define TARGET_SCALAR_MODE_SUPPORTED_P ix86_scalar_mode_supported_p
52160 #undef TARGET_VECTOR_MODE_SUPPORTED_P
52161 #define TARGET_VECTOR_MODE_SUPPORTED_P ix86_vector_mode_supported_p
52163 #undef TARGET_LIBGCC_FLOATING_MODE_SUPPORTED_P
52164 #define TARGET_LIBGCC_FLOATING_MODE_SUPPORTED_P \
52165 ix86_libgcc_floating_mode_supported_p
52167 #undef TARGET_C_MODE_FOR_SUFFIX
52168 #define TARGET_C_MODE_FOR_SUFFIX ix86_c_mode_for_suffix
52171 #undef TARGET_ASM_OUTPUT_DWARF_DTPREL
52172 #define TARGET_ASM_OUTPUT_DWARF_DTPREL i386_output_dwarf_dtprel
52175 #ifdef SUBTARGET_INSERT_ATTRIBUTES
52176 #undef TARGET_INSERT_ATTRIBUTES
52177 #define TARGET_INSERT_ATTRIBUTES SUBTARGET_INSERT_ATTRIBUTES
52180 #undef TARGET_MANGLE_TYPE
52181 #define TARGET_MANGLE_TYPE ix86_mangle_type
52184 #undef TARGET_STACK_PROTECT_FAIL
52185 #define TARGET_STACK_PROTECT_FAIL ix86_stack_protect_fail
52188 #undef TARGET_FUNCTION_VALUE
52189 #define TARGET_FUNCTION_VALUE ix86_function_value
52191 #undef TARGET_FUNCTION_VALUE_REGNO_P
52192 #define TARGET_FUNCTION_VALUE_REGNO_P ix86_function_value_regno_p
52194 #undef TARGET_PROMOTE_FUNCTION_MODE
52195 #define TARGET_PROMOTE_FUNCTION_MODE ix86_promote_function_mode
52197 #undef TARGET_OVERRIDE_OPTIONS_AFTER_CHANGE
52198 #define TARGET_OVERRIDE_OPTIONS_AFTER_CHANGE ix86_override_options_after_change
52200 #undef TARGET_MEMBER_TYPE_FORCES_BLK
52201 #define TARGET_MEMBER_TYPE_FORCES_BLK ix86_member_type_forces_blk
52203 #undef TARGET_INSTANTIATE_DECLS
52204 #define TARGET_INSTANTIATE_DECLS ix86_instantiate_decls
52206 #undef TARGET_SECONDARY_RELOAD
52207 #define TARGET_SECONDARY_RELOAD ix86_secondary_reload
52209 #undef TARGET_CLASS_MAX_NREGS
52210 #define TARGET_CLASS_MAX_NREGS ix86_class_max_nregs
52212 #undef TARGET_PREFERRED_RELOAD_CLASS
52213 #define TARGET_PREFERRED_RELOAD_CLASS ix86_preferred_reload_class
52214 #undef TARGET_PREFERRED_OUTPUT_RELOAD_CLASS
52215 #define TARGET_PREFERRED_OUTPUT_RELOAD_CLASS ix86_preferred_output_reload_class
52216 #undef TARGET_CLASS_LIKELY_SPILLED_P
52217 #define TARGET_CLASS_LIKELY_SPILLED_P ix86_class_likely_spilled_p
52219 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST
52220 #define TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST \
52221 ix86_builtin_vectorization_cost
52222 #undef TARGET_VECTORIZE_VEC_PERM_CONST_OK
52223 #define TARGET_VECTORIZE_VEC_PERM_CONST_OK \
52224 ix86_vectorize_vec_perm_const_ok
52225 #undef TARGET_VECTORIZE_PREFERRED_SIMD_MODE
52226 #define TARGET_VECTORIZE_PREFERRED_SIMD_MODE \
52227 ix86_preferred_simd_mode
52228 #undef TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES
52229 #define TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES \
52230 ix86_autovectorize_vector_sizes
52231 #undef TARGET_VECTORIZE_INIT_COST
52232 #define TARGET_VECTORIZE_INIT_COST ix86_init_cost
52233 #undef TARGET_VECTORIZE_ADD_STMT_COST
52234 #define TARGET_VECTORIZE_ADD_STMT_COST ix86_add_stmt_cost
52235 #undef TARGET_VECTORIZE_FINISH_COST
52236 #define TARGET_VECTORIZE_FINISH_COST ix86_finish_cost
52237 #undef TARGET_VECTORIZE_DESTROY_COST_DATA
52238 #define TARGET_VECTORIZE_DESTROY_COST_DATA ix86_destroy_cost_data
52240 #undef TARGET_SET_CURRENT_FUNCTION
52241 #define TARGET_SET_CURRENT_FUNCTION ix86_set_current_function
52243 #undef TARGET_OPTION_VALID_ATTRIBUTE_P
52244 #define TARGET_OPTION_VALID_ATTRIBUTE_P ix86_valid_target_attribute_p
52246 #undef TARGET_OPTION_SAVE
52247 #define TARGET_OPTION_SAVE ix86_function_specific_save
52249 #undef TARGET_OPTION_RESTORE
52250 #define TARGET_OPTION_RESTORE ix86_function_specific_restore
52252 #undef TARGET_OPTION_POST_STREAM_IN
52253 #define TARGET_OPTION_POST_STREAM_IN ix86_function_specific_post_stream_in
52255 #undef TARGET_OPTION_PRINT
52256 #define TARGET_OPTION_PRINT ix86_function_specific_print
52258 #undef TARGET_OPTION_FUNCTION_VERSIONS
52259 #define TARGET_OPTION_FUNCTION_VERSIONS ix86_function_versions
52261 #undef TARGET_CAN_INLINE_P
52262 #define TARGET_CAN_INLINE_P ix86_can_inline_p
52264 #undef TARGET_EXPAND_TO_RTL_HOOK
52265 #define TARGET_EXPAND_TO_RTL_HOOK ix86_maybe_switch_abi
52267 #undef TARGET_LEGITIMATE_ADDRESS_P
52268 #define TARGET_LEGITIMATE_ADDRESS_P ix86_legitimate_address_p
52270 #undef TARGET_LRA_P
52271 #define TARGET_LRA_P hook_bool_void_true
52273 #undef TARGET_REGISTER_PRIORITY
52274 #define TARGET_REGISTER_PRIORITY ix86_register_priority
52276 #undef TARGET_REGISTER_USAGE_LEVELING_P
52277 #define TARGET_REGISTER_USAGE_LEVELING_P hook_bool_void_true
52279 #undef TARGET_LEGITIMATE_CONSTANT_P
52280 #define TARGET_LEGITIMATE_CONSTANT_P ix86_legitimate_constant_p
52282 #undef TARGET_FRAME_POINTER_REQUIRED
52283 #define TARGET_FRAME_POINTER_REQUIRED ix86_frame_pointer_required
52285 #undef TARGET_CAN_ELIMINATE
52286 #define TARGET_CAN_ELIMINATE ix86_can_eliminate
52288 #undef TARGET_EXTRA_LIVE_ON_ENTRY
52289 #define TARGET_EXTRA_LIVE_ON_ENTRY ix86_live_on_entry
52291 #undef TARGET_ASM_CODE_END
52292 #define TARGET_ASM_CODE_END ix86_code_end
52294 #undef TARGET_CONDITIONAL_REGISTER_USAGE
52295 #define TARGET_CONDITIONAL_REGISTER_USAGE ix86_conditional_register_usage
52298 #undef TARGET_INIT_LIBFUNCS
52299 #define TARGET_INIT_LIBFUNCS darwin_rename_builtins
52302 #undef TARGET_LOOP_UNROLL_ADJUST
52303 #define TARGET_LOOP_UNROLL_ADJUST ix86_loop_unroll_adjust
52305 #undef TARGET_SPILL_CLASS
52306 #define TARGET_SPILL_CLASS ix86_spill_class
52308 #undef TARGET_SIMD_CLONE_COMPUTE_VECSIZE_AND_SIMDLEN
52309 #define TARGET_SIMD_CLONE_COMPUTE_VECSIZE_AND_SIMDLEN \
52310 ix86_simd_clone_compute_vecsize_and_simdlen
52312 #undef TARGET_SIMD_CLONE_ADJUST
52313 #define TARGET_SIMD_CLONE_ADJUST \
52314 ix86_simd_clone_adjust
52316 #undef TARGET_SIMD_CLONE_USABLE
52317 #define TARGET_SIMD_CLONE_USABLE \
52318 ix86_simd_clone_usable
52320 #undef TARGET_FLOAT_EXCEPTIONS_ROUNDING_SUPPORTED_P
52321 #define TARGET_FLOAT_EXCEPTIONS_ROUNDING_SUPPORTED_P \
52322 ix86_float_exceptions_rounding_supported_p
52324 #undef TARGET_MODE_EMIT
52325 #define TARGET_MODE_EMIT ix86_emit_mode_set
52327 #undef TARGET_MODE_NEEDED
52328 #define TARGET_MODE_NEEDED ix86_mode_needed
52330 #undef TARGET_MODE_AFTER
52331 #define TARGET_MODE_AFTER ix86_mode_after
52333 #undef TARGET_MODE_ENTRY
52334 #define TARGET_MODE_ENTRY ix86_mode_entry
52336 #undef TARGET_MODE_EXIT
52337 #define TARGET_MODE_EXIT ix86_mode_exit
52339 #undef TARGET_MODE_PRIORITY
52340 #define TARGET_MODE_PRIORITY ix86_mode_priority
52342 #undef TARGET_CALL_FUSAGE_CONTAINS_NON_CALLEE_CLOBBERS
52343 #define TARGET_CALL_FUSAGE_CONTAINS_NON_CALLEE_CLOBBERS true
52345 #undef TARGET_LOAD_BOUNDS_FOR_ARG
52346 #define TARGET_LOAD_BOUNDS_FOR_ARG ix86_load_bounds
52348 #undef TARGET_STORE_BOUNDS_FOR_ARG
52349 #define TARGET_STORE_BOUNDS_FOR_ARG ix86_store_bounds
52351 #undef TARGET_LOAD_RETURNED_BOUNDS
52352 #define TARGET_LOAD_RETURNED_BOUNDS ix86_load_returned_bounds
52354 #undef TARGET_STORE_RETURNED_BOUNDS
52355 #define TARGET_STORE_RETURNED_BOUNDS ix86_store_returned_bounds
52357 #undef TARGET_CHKP_BOUND_MODE
52358 #define TARGET_CHKP_BOUND_MODE ix86_mpx_bound_mode
52360 #undef TARGET_BUILTIN_CHKP_FUNCTION
52361 #define TARGET_BUILTIN_CHKP_FUNCTION ix86_builtin_mpx_function
52363 #undef TARGET_CHKP_FUNCTION_VALUE_BOUNDS
52364 #define TARGET_CHKP_FUNCTION_VALUE_BOUNDS ix86_function_value_bounds
52366 #undef TARGET_CHKP_MAKE_BOUNDS_CONSTANT
52367 #define TARGET_CHKP_MAKE_BOUNDS_CONSTANT ix86_make_bounds_constant
52369 #undef TARGET_CHKP_INITIALIZE_BOUNDS
52370 #define TARGET_CHKP_INITIALIZE_BOUNDS ix86_initialize_bounds
52372 #undef TARGET_SETUP_INCOMING_VARARG_BOUNDS
52373 #define TARGET_SETUP_INCOMING_VARARG_BOUNDS ix86_setup_incoming_vararg_bounds
52375 #undef TARGET_OFFLOAD_OPTIONS
52376 #define TARGET_OFFLOAD_OPTIONS \
52377 ix86_offload_options
52379 #undef TARGET_ABSOLUTE_BIGGEST_ALIGNMENT
52380 #define TARGET_ABSOLUTE_BIGGEST_ALIGNMENT 512
52382 struct gcc_target targetm = TARGET_INITIALIZER;
52384 #include "gt-i386.h"
projects / platform / upstream / gcc.git / blob