/
inst.isle
2976 lines (2546 loc) · 105 KB
/
inst.isle
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
;; Extern type definitions and constructors for the x64 `MachInst` type.
;;;; `MInst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Don't build `MInst` variants directly, in general. Instead, use the
;; instruction-emitting helpers defined further down.
(type MInst nodebug
(enum
;; Nops of various sizes, including zero.
(Nop (len u8))
;; =========================================
;; Integer instructions.
;; Integer arithmetic/bit-twiddling.
(AluRmiR (size OperandSize) ;; 4 or 8
(op AluRmiROpcode)
(src1 Gpr)
(src2 GprMemImm)
(dst WritableGpr))
;; Integer arithmetic read-modify-write on memory.
(AluRM (size OperandSize) ;; 4 or 8
(op AluRmiROpcode)
(src1_dst SyntheticAmode)
(src2 Gpr))
;; Instructions on general-purpose registers that only read src and
;; defines dst (dst is not modified). `bsr`, etc.
(UnaryRmR (size OperandSize) ;; 2, 4, or 8
(op UnaryRmROpcode)
(src GprMem)
(dst WritableGpr))
;; Bitwise not.
(Not (size OperandSize) ;; 1, 2, 4, or 8
(src Gpr)
(dst WritableGpr))
;; Integer negation.
(Neg (size OperandSize) ;; 1, 2, 4, or 8
(src Gpr)
(dst WritableGpr))
;; Integer quotient and remainder: (div idiv) $rax $rdx (reg addr)
(Div (size OperandSize) ;; 1, 2, 4, or 8
(signed bool)
(divisor GprMem)
(dividend_lo Gpr)
(dividend_hi Gpr)
(dst_quotient WritableGpr)
(dst_remainder WritableGpr))
;; The high (and low) bits of a (un)signed multiply: `RDX:RAX := RAX *
;; rhs`.
(MulHi (size OperandSize)
(signed bool)
(src1 Gpr)
(src2 GprMem)
(dst_lo WritableGpr)
(dst_hi WritableGpr))
;; A synthetic sequence to implement the right inline checks for
;; remainder and division, assuming the dividend is in %rax.
;;
;; Puts the result back into %rax if is_div, %rdx if !is_div, to mimic
;; what the div instruction does.
;;
;; The generated code sequence is described in the emit's function match
;; arm for this instruction.
;;
;; Note: %rdx is marked as modified by this instruction, to avoid an
;; early clobber problem with the temporary and divisor registers. Make
;; sure to zero %rdx right before this instruction, or you might run into
;; regalloc failures where %rdx is live before its first def!
(CheckedDivOrRemSeq (kind DivOrRemKind)
(size OperandSize)
(dividend_lo Gpr)
(dividend_hi Gpr)
;; The divisor operand. Note it's marked as modified
;; so that it gets assigned a register different from
;; the temporary.
(divisor WritableGpr)
(dst_quotient WritableGpr)
(dst_remainder WritableGpr)
(tmp OptionWritableGpr))
;; Do a sign-extend based on the sign of the value in rax into rdx: (cwd
;; cdq cqo) or al into ah: (cbw)
(SignExtendData (size OperandSize) ;; 1, 2, 4, or 8
(src Gpr)
(dst WritableGpr))
;; Constant materialization: (imm32 imm64) reg.
;;
;; Either: movl $imm32, %reg32 or movabsq $imm64, %reg32.
(Imm (dst_size OperandSize) ;; 4 or 8
(simm64 u64)
(dst WritableGpr))
;; GPR to GPR move: mov (64 32) reg reg.
(MovRR (size OperandSize) ;; 4 or 8
(src Gpr)
(dst WritableGpr))
;; Zero-extended loads, except for 64 bits: movz (bl bq wl wq lq) addr
;; reg.
;;
;; Note that the lq variant doesn't really exist since the default
;; zero-extend rule makes it unnecessary. For that case we emit the
;; equivalent "movl AM, reg32".
(MovzxRmR (ext_mode ExtMode)
(src GprMem)
(dst WritableGpr))
;; A plain 64-bit integer load, since MovZX_RM_R can't represent that.
(Mov64MR (src SyntheticAmode)
(dst WritableGpr))
;; Loads the memory address of addr into dst.
(LoadEffectiveAddress (addr SyntheticAmode)
(dst WritableGpr))
;; Sign-extended loads and moves: movs (bl bq wl wq lq) addr reg.
(MovsxRmR (ext_mode ExtMode)
(src GprMem)
(dst WritableGpr))
;; Integer stores: mov (b w l q) reg addr.
(MovRM (size OperandSize) ;; 1, 2, 4, or 8
(src Gpr)
(dst SyntheticAmode))
;; Arithmetic shifts: (shl shr sar) (b w l q) imm reg.
(ShiftR (size OperandSize) ;; 1, 2, 4, or 8
(kind ShiftKind)
(src Gpr)
;; shift count: `Imm8Gpr::Imm8(0 .. #bits-in-type - 1)` or
;; `Imm8Reg::Gpr(r)` where `r` get's move mitosis'd into `%cl`.
(num_bits Imm8Gpr)
(dst WritableGpr))
;; Arithmetic SIMD shifts.
(XmmRmiReg (opcode SseOpcode)
(src1 Xmm)
(src2 XmmMemImm)
(dst WritableXmm))
;; Integer comparisons/tests: cmp or test (b w l q) (reg addr imm) reg.
(CmpRmiR (size OperandSize) ;; 1, 2, 4, or 8
(opcode CmpOpcode)
(src GprMemImm)
(dst Gpr))
;; Materializes the requested condition code in the destinaton reg.
(Setcc (cc CC)
(dst WritableGpr))
;; =========================================
;; Conditional moves.
;; GPR conditional move; overwrites the destination register.
(Cmove (size OperandSize)
(cc CC)
(consequent GprMem)
(alternative Gpr)
(dst WritableGpr))
;; XMM conditional move; overwrites the destination register.
(XmmCmove (ty Type)
(cc CC)
(consequent XmmMem)
(alternative Xmm)
(dst WritableXmm))
;; =========================================
;; Stack manipulation.
;; pushq (reg addr imm)
(Push64 (src GprMemImm))
;; popq reg
(Pop64 (dst WritableGpr))
;; =========================================
;; Floating-point operations.
;; XMM (scalar or vector) binary op: (add sub and or xor mul adc? sbb?)
;; (32 64) (reg addr) reg
(XmmRmR (op SseOpcode)
(src1 Xmm)
(src2 XmmMem)
(dst WritableXmm))
;; XMM (scalar or vector) binary op that relies on the EVEX prefix.
(XmmRmREvex (op Avx512Opcode)
(src1 XmmMem)
(src2 Xmm)
(dst WritableXmm))
;; XMM (scalar or vector) unary op: mov between XMM registers (32 64)
;; (reg addr) reg, sqrt, etc.
;;
;; This differs from XMM_RM_R in that the dst register of XmmUnaryRmR is
;; not used in the computation of the instruction dst value and so does
;; not have to be a previously valid value. This is characteristic of mov
;; instructions.
(XmmUnaryRmR (op SseOpcode)
(src XmmMem)
(dst WritableXmm))
;; XMM (scalar or vector) unary op that relies on the EVEX prefix.
(XmmUnaryRmREvex (op Avx512Opcode)
(src XmmMem)
(dst WritableXmm))
;; XMM (scalar or vector) unary op (from xmm to reg/mem): stores, movd,
;; movq
(XmmMovRM (op SseOpcode)
(src Reg)
(dst SyntheticAmode))
;; XMM (vector) unary op (to move a constant value into an xmm register):
;; movups
(XmmLoadConst (src VCodeConstant)
(dst WritableReg)
(ty Type))
;; XMM (scalar) unary op (from xmm to integer reg): movd, movq,
;; cvtts{s,d}2si
(XmmToGpr (op SseOpcode)
(src Xmm)
(dst WritableGpr)
(dst_size OperandSize))
;; XMM (scalar) unary op (from integer to float reg): movd, movq,
;; cvtsi2s{s,d}
(GprToXmm (op SseOpcode)
(src GprMem)
(dst WritableXmm)
(src_size OperandSize))
;; Converts an unsigned int64 to a float32/float64.
(CvtUint64ToFloatSeq (dst_size OperandSize) ;; 4 or 8
;; A copy of the source register, fed by
;; lowering. It is marked as modified during
;; register allocation to make sure that the
;; temporary registers differ from the src register,
;; since both registers are live at the same time in
;; the generated code sequence.
(src WritableGpr)
(dst WritableXmm)
(tmp_gpr1 WritableGpr)
(tmp_gpr2 WritableGpr))
;; Converts a scalar xmm to a signed int32/int64.
(CvtFloatToSintSeq (dst_size OperandSize)
(src_size OperandSize)
(is_saturating bool)
;; A copy of the source register, fed by
;; lowering. It is marked as modified during
;; register allocation to make sure that the
;; temporary registers differ from the src register,
;; since both registers are live at the same time in
;; the generated code sequence.
(src WritableXmm)
(dst WritableGpr)
(tmp_gpr WritableGpr)
(tmp_xmm WritableXmm))
;; Converts a scalar xmm to an unsigned int32/int64.
(CvtFloatToUintSeq (dst_size OperandSize)
(src_size OperandSize)
(is_saturating bool)
;; A copy of the source register, fed by
;; lowering. It is marked as modified during
;; register allocation to make sure that the
;; temporary registers differ from the src register,
;; since both registers are live at the same time in
;; the generated code sequence.
(src WritableXmm)
(dst WritableGpr)
(tmp_gpr WritableGpr)
(tmp_xmm WritableXmm))
;; A sequence to compute min/max with the proper NaN semantics for xmm
;; registers.
(XmmMinMaxSeq (size OperandSize)
(is_min bool)
(lhs Xmm)
(rhs Xmm)
(dst WritableXmm))
;; Float comparisons/tests: cmp (b w l q) (reg addr imm) reg.
(XmmCmpRmR (op SseOpcode)
(src XmmMem)
(dst Xmm))
;; A binary XMM instruction with an 8-bit immediate: e.g. cmp (ps pd) imm
;; (reg addr) reg
;;
;; Note: this has to use `Reg*`, not `Xmm*`, operands because it is used
;; in various lane insertion and extraction instructions that move
;; between XMMs and GPRs.
(XmmRmRImm (op SseOpcode)
(src1 Reg)
(src2 RegMem)
(dst WritableReg)
(imm u8)
(size OperandSize))
;; =========================================
;; Control flow instructions.
;; Direct call: call simm32.
(CallKnown (dest ExternalName)
(uses VecReg)
(defs VecWritableReg)
(opcode Opcode))
;; Indirect call: callq (reg mem)
(CallUnknown (dest RegMem)
(uses VecReg)
(defs VecWritableReg)
(opcode Opcode))
;; Return.
(Ret (rets VecReg))
;; A placeholder instruction, generating no code, meaning that a function
;; epilogue must be inserted there.
(EpiloguePlaceholder)
;; Jump to a known target: jmp simm32.
(JmpKnown (dst MachLabel))
;; One-way conditional branch: jcond cond target.
;;
;; This instruction is useful when we have conditional jumps depending on
;; more than two conditions, see for instance the lowering of Brz/brnz
;; with Fcmp inputs.
;;
;; A note of caution: in contexts where the branch target is another
;; block, this has to be the same successor as the one specified in the
;; terminator branch of the current block. Otherwise, this might confuse
;; register allocation by creating new invisible edges.
(JmpIf (cc CC)
(taken MachLabel))
;; Two-way conditional branch: jcond cond target target.
;;
;; Emitted as a compound sequence; the MachBuffer will shrink it as
;; appropriate.
(JmpCond (cc CC)
(taken MachLabel)
(not_taken MachLabel))
;; Jump-table sequence, as one compound instruction (see note in lower.rs
;; for rationale).
;;
;; The generated code sequence is described in the emit's function match
;; arm for this instruction.
;;
;; See comment in lowering about the temporaries signedness.
(JmpTableSeq (idx Reg)
(tmp1 WritableReg)
(tmp2 WritableReg)
(default_target MachLabel)
(targets VecMachLabel)
(targets_for_term VecMachLabel))
;; Indirect jump: jmpq (reg mem).
(JmpUnknown (target RegMem))
;; Traps if the condition code is set.
(TrapIf (cc CC)
(trap_code TrapCode))
;; A debug trap.
(Hlt)
;; An instruction that will always trigger the illegal instruction
;; exception.
(Ud2 (trap_code TrapCode))
;; Loads an external symbol in a register, with a relocation:
;;
;; movq $name@GOTPCREL(%rip), dst if PIC is enabled, or
;; movabsq $name, dst otherwise.
(LoadExtName (dst WritableReg)
(name BoxExternalName)
(offset i64))
;; =========================================
;; Instructions pertaining to atomic memory accesses.
;; A standard (native) `lock cmpxchg src, (amode)`, with register
;; conventions:
;;
;; `mem` (read) address
;; `replacement` (read) replacement value
;; %rax (modified) in: expected value, out: value that was actually at `dst`
;; %rflags is written. Do not assume anything about it after the instruction.
;;
;; The instruction "succeeded" iff the lowest `ty` bits of %rax
;; afterwards are the same as they were before.
(LockCmpxchg (ty Type) ;; I8, I16, I32, or I64
(replacement Reg)
(expected Reg)
(mem SyntheticAmode)
(dst_old WritableReg))
;; A synthetic instruction, based on a loop around a native `lock
;; cmpxchg` instruction.
;;
;; This atomically modifies a value in memory and returns the old value.
;; The sequence consists of an initial "normal" load from `dst`, followed
;; by a loop which computes the new value and tries to compare-and-swap
;; ("CAS") it into `dst`, using the native instruction `lock
;; cmpxchg{b,w,l,q}` . The loop iterates until the CAS is successful.
;; If there is no contention, there will be only one pass through the
;; loop body. The sequence does *not* perform any explicit memory fence
;; instructions (mfence/sfence/lfence).
;;
;; Note that the transaction is atomic in the sense that, as observed by
;; some other thread, `dst` either has the initial or final value, but no
;; other. It isn't atomic in the sense of guaranteeing that no other
;; thread writes to `dst` in between the initial load and the CAS -- but
;; that would cause the CAS to fail unless the other thread's last write
;; before the CAS wrote the same value that was already there. In other
;; words, this implementation suffers (unavoidably) from the A-B-A
;; problem.
;;
;; This instruction sequence has fixed register uses as follows:
;;
;; %r9 (read) address
;; %r10 (read) second operand for `op`
;; %r11 (written) scratch reg; value afterwards has no meaning
;; %rax (written) the old value at %r9
;; %rflags is written. Do not assume anything about it after the instruction.
(AtomicRmwSeq (ty Type) ;; I8, I16, I32, or I64
(op AtomicRmwOp)
(address Reg)
(operand Reg)
(temp WritableReg)
(dst_old WritableReg))
;; A memory fence (mfence, lfence or sfence).
(Fence (kind FenceKind))
;; =========================================
;; Meta-instructions generating no code.
;; Marker, no-op in generated code: SP "virtual offset" is adjusted.
;;
;; This controls how `MemArg::NominalSPOffset` args are lowered.
(VirtualSPOffsetAdj (offset i64))
;; Provides a way to tell the register allocator that the upcoming
;; sequence of instructions will overwrite `dst` so it should be
;; considered as a `def`; use this with care.
;;
;; This is useful when we have a sequence of instructions whose register
;; usages are nominally `mod`s, but such that the combination of
;; operations creates a result that is independent of the initial
;; register value. It's thus semantically a `def`, not a `mod`, when all
;; the instructions are taken together, so we want to ensure the register
;; is defined (its live-range starts) prior to the sequence to keep
;; analyses happy.
;;
;; One alternative would be a compound instruction that somehow
;; encapsulates the others and reports its own `def`s/`use`s/`mod`s; this
;; adds complexity (the instruction list is no longer flat) and requires
;; knowledge about semantics and initial-value independence anyway.
(XmmUninitializedValue (dst WritableXmm))
;; A call to the `ElfTlsGetAddr` libcall. Returns address of TLS symbol
;; in `rax`.
(ElfTlsGetAddr (symbol ExternalName))
;; A Mach-O TLS symbol access. Returns address of the TLS symbol in
;; `rax`.
(MachOTlsGetAddr (symbol ExternalName))
;; An unwind pseudoinstruction describing the state of the machine at
;; this program point.
(Unwind (inst UnwindInst))
;; A pseudoinstruction that just keeps a value alive.
(DummyUse (reg Reg))))
(type OperandSize extern
(enum Size8
Size16
Size32
Size64))
(type FenceKind extern
(enum MFence
LFence
SFence))
;; Get the `OperandSize` for a given `Type`, rounding smaller types up to 32 bits.
(decl operand_size_of_type_32_64 (Type) OperandSize)
(extern constructor operand_size_of_type_32_64 operand_size_of_type_32_64)
;; Get the true `OperandSize` for a given `Type`, with no rounding.
(decl raw_operand_size_of_type (Type) OperandSize)
(extern constructor raw_operand_size_of_type raw_operand_size_of_type)
;; Get the bit width of an `OperandSize`.
(decl operand_size_bits (OperandSize) u16)
(rule (operand_size_bits (OperandSize.Size8)) 8)
(rule (operand_size_bits (OperandSize.Size16)) 16)
(rule (operand_size_bits (OperandSize.Size32)) 32)
(rule (operand_size_bits (OperandSize.Size64)) 64)
(type AluRmiROpcode extern
(enum Add
Adc
Sub
Sbb
And
Or
Xor
Mul
And8
Or8))
(type UnaryRmROpcode extern
(enum Bsr
Bsf
Lzcnt
Tzcnt
Popcnt))
(type DivOrRemKind extern
(enum SignedDiv
UnsignedDiv
SignedRem
UnsignedRem))
(type SseOpcode extern
(enum Addps
Addpd
Addss
Addsd
Andps
Andpd
Andnps
Andnpd
Blendvpd
Blendvps
Comiss
Comisd
Cmpps
Cmppd
Cmpss
Cmpsd
Cvtdq2ps
Cvtdq2pd
Cvtpd2ps
Cvtps2pd
Cvtsd2ss
Cvtsd2si
Cvtsi2ss
Cvtsi2sd
Cvtss2si
Cvtss2sd
Cvttpd2dq
Cvttps2dq
Cvttss2si
Cvttsd2si
Divps
Divpd
Divss
Divsd
Insertps
Maxps
Maxpd
Maxss
Maxsd
Minps
Minpd
Minss
Minsd
Movaps
Movapd
Movd
Movdqa
Movdqu
Movlhps
Movmskps
Movmskpd
Movq
Movss
Movsd
Movups
Movupd
Mulps
Mulpd
Mulss
Mulsd
Orps
Orpd
Pabsb
Pabsw
Pabsd
Packssdw
Packsswb
Packusdw
Packuswb
Paddb
Paddd
Paddq
Paddw
Paddsb
Paddsw
Paddusb
Paddusw
Palignr
Pand
Pandn
Pavgb
Pavgw
Pblendvb
Pcmpeqb
Pcmpeqw
Pcmpeqd
Pcmpeqq
Pcmpgtb
Pcmpgtw
Pcmpgtd
Pcmpgtq
Pextrb
Pextrw
Pextrd
Pinsrb
Pinsrw
Pinsrd
Pmaddubsw
Pmaddwd
Pmaxsb
Pmaxsw
Pmaxsd
Pmaxub
Pmaxuw
Pmaxud
Pminsb
Pminsw
Pminsd
Pminub
Pminuw
Pminud
Pmovmskb
Pmovsxbd
Pmovsxbw
Pmovsxbq
Pmovsxwd
Pmovsxwq
Pmovsxdq
Pmovzxbd
Pmovzxbw
Pmovzxbq
Pmovzxwd
Pmovzxwq
Pmovzxdq
Pmuldq
Pmulhw
Pmulhuw
Pmulhrsw
Pmulld
Pmullw
Pmuludq
Por
Pshufb
Pshufd
Psllw
Pslld
Psllq
Psraw
Psrad
Psrlw
Psrld
Psrlq
Psubb
Psubd
Psubq
Psubw
Psubsb
Psubsw
Psubusb
Psubusw
Ptest
Punpckhbw
Punpckhwd
Punpcklbw
Punpcklwd
Pxor
Rcpss
Roundps
Roundpd
Roundss
Roundsd
Rsqrtss
Shufps
Sqrtps
Sqrtpd
Sqrtss
Sqrtsd
Subps
Subpd
Subss
Subsd
Ucomiss
Ucomisd
Unpcklps
Xorps
Xorpd))
(type CmpOpcode extern
(enum Cmp
Test))
(type RegMemImm extern
(enum
(Reg (reg Reg))
(Mem (addr SyntheticAmode))
(Imm (simm32 u32))))
;; Put the given clif value into a `RegMemImm` operand.
;;
;; Asserts that the value fits into a single register, and doesn't require
;; multiple registers for its representation (like `i128` for example).
;;
;; As a side effect, this marks the value as used.
(decl put_in_reg_mem_imm (Value) RegMemImm)
(extern constructor put_in_reg_mem_imm put_in_reg_mem_imm)
(type RegMem extern
(enum
(Reg (reg Reg))
(Mem (addr SyntheticAmode))))
;; Put the given clif value into a `RegMem` operand.
;;
;; Asserts that the value fits into a single register, and doesn't require
;; multiple registers for its representation (like `i128` for example).
;;
;; As a side effect, this marks the value as used.
(decl put_in_reg_mem (Value) RegMem)
(extern constructor put_in_reg_mem put_in_reg_mem)
;; Addressing modes.
(type SyntheticAmode extern (enum))
(decl synthetic_amode_to_reg_mem (SyntheticAmode) RegMem)
(extern constructor synthetic_amode_to_reg_mem synthetic_amode_to_reg_mem)
(decl amode_to_synthetic_amode (Amode) SyntheticAmode)
(extern constructor amode_to_synthetic_amode amode_to_synthetic_amode)
;; An `Amode` represents a possible addressing mode that can be used
;; in instructions. These denote a 64-bit value only.
(type Amode (enum
;; Immediate sign-extended and a register
(ImmReg (simm32 u32)
(base Reg)
(flags MemFlags))
;; Sign-extend-32-to-64(simm32) + base + (index << shift)
(ImmRegRegShift (simm32 u32)
(base Gpr)
(index Gpr)
(shift u8)
(flags MemFlags))
;; Sign-extend-32-to-64(immediate) + RIP (instruction
;; pointer). The appropriate relocation is emitted so
;; that the resulting immediate makes this Amode refer to
;; the given MachLabel.
(RipRelative (target MachLabel))))
;; Some Amode constructor helpers.
(decl amode_with_flags (Amode MemFlags) Amode)
(extern constructor amode_with_flags amode_with_flags)
(decl amode_imm_reg (u32 Gpr) Amode)
(extern constructor amode_imm_reg amode_imm_reg)
(decl amode_imm_reg_flags (u32 Gpr MemFlags) Amode)
(rule (amode_imm_reg_flags offset base flags)
(amode_with_flags (amode_imm_reg offset base) flags))
(decl amode_imm_reg_reg_shift (u32 Gpr Gpr u8) Amode)
(extern constructor amode_imm_reg_reg_shift amode_imm_reg_reg_shift)
(decl amode_imm_reg_reg_shift_flags (u32 Gpr Gpr u8 MemFlags) Amode)
(rule (amode_imm_reg_reg_shift_flags offset base index shift flags)
(amode_with_flags (amode_imm_reg_reg_shift offset base index shift) flags))
;; A helper to both check that the `Imm64` and `Offset32` values sum to less
;; than 32-bits AND return this summed `u32` value. Also, the `Imm64` will be
;; zero-extended from `Type` up to 64 bits. This is useful for `to_amode`.
(decl pure sum_extend_fits_in_32_bits (Type Imm64 Offset32) u32)
(extern constructor sum_extend_fits_in_32_bits sum_extend_fits_in_32_bits)
;;;; Amode lowering ;;;;
;; To generate an address for a memory access, we can pattern-match
;; various CLIF sub-trees to x64's complex addressing modes (`Amode`).
;;
;; Information about available addressing modes is available in
;; Intel's Software Developer's Manual, volume 2, section 2.1.5,
;; "Addressing-Mode Encoding of ModR/M and SIB Bytes."
;;
;; The general strategy to build an `Amode` is to traverse over the
;; input expression's addends, recursively deconstructing a tree of
;; `iadd` operators that add up parts of the address, updating the
;; `Amode` in an incremental fashion as we add in each piece.
;;
;; We start with an "immediate + register" form that encapsulates the
;; load/store's built-in `Offset32` and `invalid_reg` as the
;; register. This is given by `amode_initial`. Then we add `Value`s
;; one at a time with `amode_add`. (Why start with `invalid_reg` at
;; all? Because we don't want to special-case the first input and
;; duplicate rules; this lets us use the "add a value" logic even for
;; the first value.)
;;
;; It is always valid to use `amode_add` to add the one single
;; `address` input to the load/store (i.e., the `Value` given to
;; `to_amode`). In the fallback case, this is what we do. Then we get
;; an `Amode.ImmReg` with the `Offset32` and `Value` below and nothing
;; else; this always works and is not *that* bad.
;;
;; But we can often do better. The toplevel rule for `iadd` below will
;; turn an `(amode_add amode (iadd a b))` into two invocations of
;; `amode_add`, for each operand of the `iadd`. This is what allows us
;; to handle sums of many parts.
;;
;; Then we "just" need to work out how we can incorporate a new
;; component into an existing addressing mode:
;;
;; - Case 1: When we have an `ImmReg` and the register is
;; `invalid_reg` (the initial `Amode` above), we can put the new
;; addend into a register and insert it into the `ImmReg`.
;;
;; - Case 2: When we have an `ImmReg` with a valid register already,
;; and we have another register to add, we can transition to an
;; `ImmRegRegShift`.
;;
;; - Case 3: When we're adding an `ishl`, we can refine the above rule
;; and use the built-in multiplier of 1, 2, 4, 8 to implement a
;; left-shift by 0, 1, 2, 3.
;;
;; - Case 4: When we are adding another constant offset, we can fold
;; it into the existing offset, as long as the sum still fits into
;; the signed 32-bit field.
;;
;; - Case 5: And as a general fallback, we can generate a new `add`
;; instruction and add the new addend to an existing component of
;; the `Amode`.
(decl to_amode (MemFlags Value Offset32) Amode)
;; Initial step in amode processing: create an ImmReg with
;; (invalid_reg) and encapsulating the flags and offset from the
;; load/store.
(decl amode_initial (MemFlags Offset32) Amode)
(rule (amode_initial flags (offset32 off))
(Amode.ImmReg off (invalid_reg) flags))
;; One step in amode processing: take an existing amode and add
;; another value to it.
(decl amode_add (Amode Value) Amode)
;; -- Top-level driver: pull apart the addends.
;;
;; Any amode can absorb an `iadd` by absorbing first the LHS of the
;; add, then the RHS.
;;
;; Priority 2 to take this above fallbacks and ensure we traverse the
;; `iadd` tree fully.
(rule 2 (amode_add amode (iadd x y))
(let ((amode1 Amode (amode_add amode x))
(amode2 Amode (amode_add amode1 y)))
amode2))
;; -- Case 1 (adding a register to the initial Amode with invalid_reg).
;;
;; An Amode.ImmReg with invalid_reg (initial state) can absorb a
;; register as the base register.
(rule (amode_add (Amode.ImmReg off (invalid_reg) flags) value)
(Amode.ImmReg off value flags))
;; -- Case 2 (adding a register to an Amode with a register already).
;;
;; An Amode.ImmReg can absorb another register as the index register.
(rule (amode_add (Amode.ImmReg off base flags) value)
(if-let (valid_reg) base)
;; Shift of 0 --> base + 1*value.
(Amode.ImmRegRegShift off base value 0 flags))
;; -- Case 3 (adding a shifted value to an Amode).
;;
;; An Amode.ImmReg can absorb a shift of another register as the index register.
;;
;; Priority 2 to take these rules above generic case.
(rule 2 (amode_add (Amode.ImmReg off base flags) (ishl index (iconst (uimm8 shift))))
(if-let (valid_reg) base)
(if (u32_lteq (u8_as_u32 shift) 3))
(Amode.ImmRegRegShift off base index shift flags))
(rule 2 (amode_add (Amode.ImmReg off base flags) (uextend (ishl index (iconst (uimm8 shift)))))
(if-let (valid_reg) base)
(if (u32_lteq (u8_as_u32 shift) 3))
(Amode.ImmRegRegShift off base (extend_to_gpr index $I64 (ExtendKind.Zero)) shift flags))
;; Same, but with a uextend of a shift of a 32-bit add. This is valid
;; because we know our lowering of a narrower-than-64-bit `iadd` will
;; always write the full register width, so we can effectively ignore
;; the `uextend` and look through it to the `ishl`.
;;
;; Priority 2 to take this case above generic rules.
(rule 2 (amode_add (Amode.ImmReg off base flags)
(uextend (ishl index @ (iadd _ _) (iconst (uimm8 shift)))))
(if-let (valid_reg) base)
(if (u32_lteq (u8_as_u32 shift) 3))
(Amode.ImmRegRegShift off base index shift flags))
;; -- Case 4 (absorbing constant offsets).
;;
;; An Amode can absorb a constant (i64, or extended i32) as long as
;; the sum still fits in the signed-32-bit offset.
;;
;; Priority 3 in order to take this option above the fallback
;; (immediate in register). Two rules, for imm+reg and
;; imm+reg+scale*reg cases.
(rule 3 (amode_add (Amode.ImmReg off base flags)
(iconst (simm32 c)))
(if-let sum (s32_add_fallible off c))
(Amode.ImmReg sum base flags))
(rule 3 (amode_add (Amode.ImmRegRegShift off base index shift flags)
(iconst (simm32 c)))
(if-let sum (s32_add_fallible off c))
(Amode.ImmRegRegShift sum base index shift flags))
;; Likewise for a zero-extended i32 const, as long as the constant
;; wasn't negative. (Why nonnegative? Because adding a
;; non-sign-extended negative to a 64-bit address is not the same as
;; adding in simm32-space.)
(rule 3 (amode_add (Amode.ImmReg off base flags)
(uextend (iconst (simm32 (u32_nonnegative c)))))
(if-let sum (s32_add_fallible off c))
(Amode.ImmReg sum base flags))
(rule 3 (amode_add (Amode.ImmRegRegShift off base index shift flags)
(uextend (iconst (simm32 (u32_nonnegative c)))))
(if-let sum (s32_add_fallible off c))
(Amode.ImmRegRegShift sum base index shift flags))
;; Likewise for a sign-extended i32 const.
(rule 3 (amode_add (Amode.ImmReg off base flags)
(sextend (iconst (simm32 c))))
(if-let sum (s32_add_fallible off c))
(Amode.ImmReg sum base flags))
(rule 3 (amode_add (Amode.ImmRegRegShift off base index shift flags)
(sextend (iconst (simm32 c))))
(if-let sum (s32_add_fallible off c))
(Amode.ImmRegRegShift sum base index shift flags))
;; -- Case 5 (fallback to add a new value to an imm+reg+scale*reg).
;;
;; An Amode.ImmRegRegShift can absorb any other value by creating a
;; new add instruction and replacing the base with
;; (base+value).
(rule (amode_add (Amode.ImmRegRegShift off base index shift flags) value)
(let ((sum Gpr (x64_add $I64 base value)))
(Amode.ImmRegRegShift off sum index shift flags)))
;; Finally, define the toplevel `to_amode`.
(rule (to_amode flags base offset)
(amode_finalize (amode_add (amode_initial flags offset) base)))
;; If an amode has no registers at all and only offsets (a constant
;; value), we need to "finalize" it by sticking in a zero'd reg in
;; place of the (invalid_reg) produced by (amode_initial).
(decl amode_finalize (Amode) Amode)
(rule 1 (amode_finalize (Amode.ImmReg off (invalid_reg) flags))
(Amode.ImmReg off (imm $I64 0) flags))
(rule 0 (amode_finalize amode)
amode)
;; Offsetting an Amode. Used when we need to do consecutive
;; loads/stores to adjacent addresses.
(decl amode_offset (Amode u32) Amode)
(extern constructor amode_offset amode_offset)
;; Return a zero offset as an `Offset32`.