1// Copyright 2016 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5// Lowering arithmetic 6(Add(64|Ptr) ...) => (ADD ...) 7(Add(32|16|8) ...) => (ADDW ...) 8(Add32F x y) => (Select0 (FADDS x y)) 9(Add64F x y) => (Select0 (FADD x y)) 10 11(Sub(64|Ptr) ...) => (SUB ...) 12(Sub(32|16|8) ...) => (SUBW ...) 13(Sub32F x y) => (Select0 (FSUBS x y)) 14(Sub64F x y) => (Select0 (FSUB x y)) 15 16(Mul64 ...) => (MULLD ...) 17(Mul(32|16|8) ...) => (MULLW ...) 18(Mul32F ...) => (FMULS ...) 19(Mul64F ...) => (FMUL ...) 20(Mul64uhilo ...) => (MLGR ...) 21 22(Div32F ...) => (FDIVS ...) 23(Div64F ...) => (FDIV ...) 24 25(Div64 x y) => (DIVD x y) 26(Div64u ...) => (DIVDU ...) 27// DIVW/DIVWU has a 64-bit dividend and a 32-bit divisor, 28// so a sign/zero extension of the dividend is required. 29(Div32 x y) => (DIVW (MOVWreg x) y) 30(Div32u x y) => (DIVWU (MOVWZreg x) y) 31(Div16 x y) => (DIVW (MOVHreg x) (MOVHreg y)) 32(Div16u x y) => (DIVWU (MOVHZreg x) (MOVHZreg y)) 33(Div8 x y) => (DIVW (MOVBreg x) (MOVBreg y)) 34(Div8u x y) => (DIVWU (MOVBZreg x) (MOVBZreg y)) 35 36(Hmul(64|64u) ...) => (MULH(D|DU) ...) 37(Hmul32 x y) => (SRDconst [32] (MULLD (MOVWreg x) (MOVWreg y))) 38(Hmul32u x y) => (SRDconst [32] (MULLD (MOVWZreg x) (MOVWZreg y))) 39 40(Mod64 x y) => (MODD x y) 41(Mod64u ...) => (MODDU ...) 42// MODW/MODWU has a 64-bit dividend and a 32-bit divisor, 43// so a sign/zero extension of the dividend is required. 44(Mod32 x y) => (MODW (MOVWreg x) y) 45(Mod32u x y) => (MODWU (MOVWZreg x) y) 46(Mod16 x y) => (MODW (MOVHreg x) (MOVHreg y)) 47(Mod16u x y) => (MODWU (MOVHZreg x) (MOVHZreg y)) 48(Mod8 x y) => (MODW (MOVBreg x) (MOVBreg y)) 49(Mod8u x y) => (MODWU (MOVBZreg x) (MOVBZreg y)) 50 51// (x + y) / 2 with x>=y -> (x - y) / 2 + y 52(Avg64u <t> x y) => (ADD (SRDconst <t> (SUB <t> x y) [1]) y) 53 54(And64 ...) => (AND ...) 55(And(32|16|8) ...) => (ANDW ...) 56 57(Or64 ...) => (OR ...) 58(Or(32|16|8) ...) => (ORW ...) 59 60(Xor64 ...) => (XOR ...) 61(Xor(32|16|8) ...) => (XORW ...) 62 63(Neg64 ...) => (NEG ...) 64(Neg(32|16|8) ...) => (NEGW ...) 65(Neg32F ...) => (FNEGS ...) 66(Neg64F ...) => (FNEG ...) 67 68(Com64 ...) => (NOT ...) 69(Com(32|16|8) ...) => (NOTW ...) 70(NOT x) => (XOR (MOVDconst [-1]) x) 71(NOTW x) => (XORWconst [-1] x) 72 73// Lowering boolean ops 74(AndB ...) => (ANDW ...) 75(OrB ...) => (ORW ...) 76(Not x) => (XORWconst [1] x) 77 78// Lowering pointer arithmetic 79(OffPtr [off] ptr:(SP)) => (MOVDaddr [int32(off)] ptr) 80(OffPtr [off] ptr) && is32Bit(off) => (ADDconst [int32(off)] ptr) 81(OffPtr [off] ptr) => (ADD (MOVDconst [off]) ptr) 82 83// TODO: optimize these cases? 84(Ctz64NonZero ...) => (Ctz64 ...) 85(Ctz32NonZero ...) => (Ctz32 ...) 86 87// Ctz(x) = 64 - findLeftmostOne((x-1)&^x) 88(Ctz64 <t> x) => (SUB (MOVDconst [64]) (FLOGR (AND <t> (SUBconst <t> [1] x) (NOT <t> x)))) 89(Ctz32 <t> x) => (SUB (MOVDconst [64]) (FLOGR (MOVWZreg (ANDW <t> (SUBWconst <t> [1] x) (NOTW <t> x))))) 90 91(BitLen64 x) => (SUB (MOVDconst [64]) (FLOGR x)) 92 93// POPCNT treats the input register as a vector of 8 bytes, producing 94// a population count for each individual byte. For inputs larger than 95// a single byte we therefore need to sum the individual bytes produced 96// by the POPCNT instruction. For example, the following instruction 97// sequence could be used to calculate the population count of a 4-byte 98// value: 99// 100// MOVD $0x12345678, R1 // R1=0x12345678 <-- input 101// POPCNT R1, R2 // R2=0x02030404 102// SRW $16, R2, R3 // R3=0x00000203 103// ADDW R2, R3, R4 // R4=0x02030607 104// SRW $8, R4, R5 // R5=0x00020306 105// ADDW R4, R5, R6 // R6=0x0205090d 106// MOVBZ R6, R7 // R7=0x0000000d <-- result is 13 107// 108(PopCount8 x) => (POPCNT (MOVBZreg x)) 109(PopCount16 x) => (MOVBZreg (SumBytes2 (POPCNT <typ.UInt16> x))) 110(PopCount32 x) => (MOVBZreg (SumBytes4 (POPCNT <typ.UInt32> x))) 111(PopCount64 x) => (MOVBZreg (SumBytes8 (POPCNT <typ.UInt64> x))) 112 113// SumBytes{2,4,8} pseudo operations sum the values of the rightmost 114// 2, 4 or 8 bytes respectively. The result is a single byte however 115// other bytes might contain junk so a zero extension is required if 116// the desired output type is larger than 1 byte. 117(SumBytes2 x) => (ADDW (SRWconst <typ.UInt8> x [8]) x) 118(SumBytes4 x) => (SumBytes2 (ADDW <typ.UInt16> (SRWconst <typ.UInt16> x [16]) x)) 119(SumBytes8 x) => (SumBytes4 (ADDW <typ.UInt32> (SRDconst <typ.UInt32> x [32]) x)) 120 121(Bswap64 ...) => (MOVDBR ...) 122(Bswap32 ...) => (MOVWBR ...) 123 124// add with carry 125(Select0 (Add64carry x y c)) 126 => (Select0 <typ.UInt64> (ADDE x y (Select1 <types.TypeFlags> (ADDCconst c [-1])))) 127(Select1 (Add64carry x y c)) 128 => (Select0 <typ.UInt64> (ADDE (MOVDconst [0]) (MOVDconst [0]) (Select1 <types.TypeFlags> (ADDE x y (Select1 <types.TypeFlags> (ADDCconst c [-1])))))) 129 130// subtract with borrow 131(Select0 (Sub64borrow x y c)) 132 => (Select0 <typ.UInt64> (SUBE x y (Select1 <types.TypeFlags> (SUBC (MOVDconst [0]) c)))) 133(Select1 (Sub64borrow x y c)) 134 => (NEG (Select0 <typ.UInt64> (SUBE (MOVDconst [0]) (MOVDconst [0]) (Select1 <types.TypeFlags> (SUBE x y (Select1 <types.TypeFlags> (SUBC (MOVDconst [0]) c))))))) 135 136// math package intrinsics 137(Sqrt ...) => (FSQRT ...) 138(Floor x) => (FIDBR [7] x) 139(Ceil x) => (FIDBR [6] x) 140(Trunc x) => (FIDBR [5] x) 141(RoundToEven x) => (FIDBR [4] x) 142(Round x) => (FIDBR [1] x) 143(FMA x y z) => (FMADD z x y) 144 145(Sqrt32 ...) => (FSQRTS ...) 146 147// Atomic loads and stores. 148// The SYNC instruction (fast-BCR-serialization) prevents store-load 149// reordering. Other sequences of memory operations (load-load, 150// store-store and load-store) are already guaranteed not to be reordered. 151(AtomicLoad(8|32|Acq32|64|Ptr) ptr mem) => (MOV(BZ|WZ|WZ|D|D)atomicload ptr mem) 152(AtomicStore(8|32|64|PtrNoWB) ptr val mem) => (SYNC (MOV(B|W|D|D)atomicstore ptr val mem)) 153 154// Store-release doesn't require store-load ordering. 155(AtomicStoreRel32 ptr val mem) => (MOVWatomicstore ptr val mem) 156 157// Atomic adds. 158(AtomicAdd32 ptr val mem) => (AddTupleFirst32 val (LAA ptr val mem)) 159(AtomicAdd64 ptr val mem) => (AddTupleFirst64 val (LAAG ptr val mem)) 160(Select0 <t> (AddTupleFirst32 val tuple)) => (ADDW val (Select0 <t> tuple)) 161(Select1 (AddTupleFirst32 _ tuple)) => (Select1 tuple) 162(Select0 <t> (AddTupleFirst64 val tuple)) => (ADD val (Select0 <t> tuple)) 163(Select1 (AddTupleFirst64 _ tuple)) => (Select1 tuple) 164 165// Atomic exchanges. 166(AtomicExchange32 ptr val mem) => (LoweredAtomicExchange32 ptr val mem) 167(AtomicExchange64 ptr val mem) => (LoweredAtomicExchange64 ptr val mem) 168 169// Atomic compare and swap. 170(AtomicCompareAndSwap32 ptr old new_ mem) => (LoweredAtomicCas32 ptr old new_ mem) 171(AtomicCompareAndSwap64 ptr old new_ mem) => (LoweredAtomicCas64 ptr old new_ mem) 172 173// Atomic and: *(*uint8)(ptr) &= val 174// 175// Round pointer down to nearest word boundary and pad value with ones before 176// applying atomic AND operation to target word. 177// 178// *(*uint32)(ptr &^ 3) &= rotateleft(uint32(val) | 0xffffff00, ((3 << 3) ^ ((ptr & 3) << 3)) 179// 180(AtomicAnd8 ptr val mem) 181 => (LANfloor 182 ptr 183 (RLL <typ.UInt32> 184 (ORWconst <typ.UInt32> val [-1<<8]) 185 (RXSBG <typ.UInt32> {s390x.NewRotateParams(59, 60, 3)} (MOVDconst [3<<3]) ptr)) 186 mem) 187 188// Atomic or: *(*uint8)(ptr) |= val 189// 190// Round pointer down to nearest word boundary and pad value with zeros before 191// applying atomic OR operation to target word. 192// 193// *(*uint32)(ptr &^ 3) |= uint32(val) << ((3 << 3) ^ ((ptr & 3) << 3)) 194// 195(AtomicOr8 ptr val mem) 196 => (LAOfloor 197 ptr 198 (SLW <typ.UInt32> 199 (MOVBZreg <typ.UInt32> val) 200 (RXSBG <typ.UInt32> {s390x.NewRotateParams(59, 60, 3)} (MOVDconst [3<<3]) ptr)) 201 mem) 202 203(AtomicAnd32 ...) => (LAN ...) 204(AtomicOr32 ...) => (LAO ...) 205 206// Lowering extension 207// Note: we always extend to 64 bits even though some ops don't need that many result bits. 208(SignExt8to(16|32|64) ...) => (MOVBreg ...) 209(SignExt16to(32|64) ...) => (MOVHreg ...) 210(SignExt32to64 ...) => (MOVWreg ...) 211 212(ZeroExt8to(16|32|64) ...) => (MOVBZreg ...) 213(ZeroExt16to(32|64) ...) => (MOVHZreg ...) 214(ZeroExt32to64 ...) => (MOVWZreg ...) 215 216(Slicemask <t> x) => (SRADconst (NEG <t> x) [63]) 217 218// Lowering truncation 219// Because we ignore high parts of registers, truncates are just copies. 220(Trunc(16|32|64)to8 ...) => (Copy ...) 221(Trunc(32|64)to16 ...) => (Copy ...) 222(Trunc64to32 ...) => (Copy ...) 223 224// Lowering float <-> int 225(Cvt32to32F ...) => (CEFBRA ...) 226(Cvt32to64F ...) => (CDFBRA ...) 227(Cvt64to32F ...) => (CEGBRA ...) 228(Cvt64to64F ...) => (CDGBRA ...) 229 230(Cvt32Fto32 ...) => (CFEBRA ...) 231(Cvt32Fto64 ...) => (CGEBRA ...) 232(Cvt64Fto32 ...) => (CFDBRA ...) 233(Cvt64Fto64 ...) => (CGDBRA ...) 234 235// Lowering float <-> uint 236(Cvt32Uto32F ...) => (CELFBR ...) 237(Cvt32Uto64F ...) => (CDLFBR ...) 238(Cvt64Uto32F ...) => (CELGBR ...) 239(Cvt64Uto64F ...) => (CDLGBR ...) 240 241(Cvt32Fto32U ...) => (CLFEBR ...) 242(Cvt32Fto64U ...) => (CLGEBR ...) 243(Cvt64Fto32U ...) => (CLFDBR ...) 244(Cvt64Fto64U ...) => (CLGDBR ...) 245 246// Lowering float32 <-> float64 247(Cvt32Fto64F ...) => (LDEBR ...) 248(Cvt64Fto32F ...) => (LEDBR ...) 249 250(CvtBoolToUint8 ...) => (Copy ...) 251 252(Round(32|64)F ...) => (LoweredRound(32|64)F ...) 253 254// Lowering shifts 255 256// Lower bounded shifts first. No need to check shift value. 257(Lsh64x(64|32|16|8) x y) && shiftIsBounded(v) => (SLD x y) 258(Lsh32x(64|32|16|8) x y) && shiftIsBounded(v) => (SLW x y) 259(Lsh16x(64|32|16|8) x y) && shiftIsBounded(v) => (SLW x y) 260(Lsh8x(64|32|16|8) x y) && shiftIsBounded(v) => (SLW x y) 261(Rsh64Ux(64|32|16|8) x y) && shiftIsBounded(v) => (SRD x y) 262(Rsh32Ux(64|32|16|8) x y) && shiftIsBounded(v) => (SRW x y) 263(Rsh16Ux(64|32|16|8) x y) && shiftIsBounded(v) => (SRW (MOVHZreg x) y) 264(Rsh8Ux(64|32|16|8) x y) && shiftIsBounded(v) => (SRW (MOVBZreg x) y) 265(Rsh64x(64|32|16|8) x y) && shiftIsBounded(v) => (SRAD x y) 266(Rsh32x(64|32|16|8) x y) && shiftIsBounded(v) => (SRAW x y) 267(Rsh16x(64|32|16|8) x y) && shiftIsBounded(v) => (SRAW (MOVHreg x) y) 268(Rsh8x(64|32|16|8) x y) && shiftIsBounded(v) => (SRAW (MOVBreg x) y) 269 270// Unsigned shifts need to return 0 if shift amount is >= width of shifted value. 271// result = shift >= 64 ? 0 : arg << shift 272(Lsh(64|32|16|8)x64 <t> x y) => (LOCGR {s390x.GreaterOrEqual} <t> (SL(D|W|W|W) <t> x y) (MOVDconst [0]) (CMPUconst y [64])) 273(Lsh(64|32|16|8)x32 <t> x y) => (LOCGR {s390x.GreaterOrEqual} <t> (SL(D|W|W|W) <t> x y) (MOVDconst [0]) (CMPWUconst y [64])) 274(Lsh(64|32|16|8)x16 <t> x y) => (LOCGR {s390x.GreaterOrEqual} <t> (SL(D|W|W|W) <t> x y) (MOVDconst [0]) (CMPWUconst (MOVHZreg y) [64])) 275(Lsh(64|32|16|8)x8 <t> x y) => (LOCGR {s390x.GreaterOrEqual} <t> (SL(D|W|W|W) <t> x y) (MOVDconst [0]) (CMPWUconst (MOVBZreg y) [64])) 276 277(Rsh(64|32)Ux64 <t> x y) => (LOCGR {s390x.GreaterOrEqual} <t> (SR(D|W) <t> x y) (MOVDconst [0]) (CMPUconst y [64])) 278(Rsh(64|32)Ux32 <t> x y) => (LOCGR {s390x.GreaterOrEqual} <t> (SR(D|W) <t> x y) (MOVDconst [0]) (CMPWUconst y [64])) 279(Rsh(64|32)Ux16 <t> x y) => (LOCGR {s390x.GreaterOrEqual} <t> (SR(D|W) <t> x y) (MOVDconst [0]) (CMPWUconst (MOVHZreg y) [64])) 280(Rsh(64|32)Ux8 <t> x y) => (LOCGR {s390x.GreaterOrEqual} <t> (SR(D|W) <t> x y) (MOVDconst [0]) (CMPWUconst (MOVBZreg y) [64])) 281 282(Rsh(16|8)Ux64 <t> x y) => (LOCGR {s390x.GreaterOrEqual} <t> (SRW <t> (MOV(H|B)Zreg x) y) (MOVDconst [0]) (CMPUconst y [64])) 283(Rsh(16|8)Ux32 <t> x y) => (LOCGR {s390x.GreaterOrEqual} <t> (SRW <t> (MOV(H|B)Zreg x) y) (MOVDconst [0]) (CMPWUconst y [64])) 284(Rsh(16|8)Ux16 <t> x y) => (LOCGR {s390x.GreaterOrEqual} <t> (SRW <t> (MOV(H|B)Zreg x) y) (MOVDconst [0]) (CMPWUconst (MOVHZreg y) [64])) 285(Rsh(16|8)Ux8 <t> x y) => (LOCGR {s390x.GreaterOrEqual} <t> (SRW <t> (MOV(H|B)Zreg x) y) (MOVDconst [0]) (CMPWUconst (MOVBZreg y) [64])) 286 287// Signed right shift needs to return 0/-1 if shift amount is >= width of shifted value. 288// We implement this by setting the shift value to 63 (all ones) if the shift value is more than 63. 289// result = arg >> (shift >= 64 ? 63 : shift) 290(Rsh(64|32)x64 x y) => (SRA(D|W) x (LOCGR {s390x.GreaterOrEqual} <y.Type> y (MOVDconst <y.Type> [63]) (CMPUconst y [64]))) 291(Rsh(64|32)x32 x y) => (SRA(D|W) x (LOCGR {s390x.GreaterOrEqual} <y.Type> y (MOVDconst <y.Type> [63]) (CMPWUconst y [64]))) 292(Rsh(64|32)x16 x y) => (SRA(D|W) x (LOCGR {s390x.GreaterOrEqual} <y.Type> y (MOVDconst <y.Type> [63]) (CMPWUconst (MOVHZreg y) [64]))) 293(Rsh(64|32)x8 x y) => (SRA(D|W) x (LOCGR {s390x.GreaterOrEqual} <y.Type> y (MOVDconst <y.Type> [63]) (CMPWUconst (MOVBZreg y) [64]))) 294 295(Rsh(16|8)x64 x y) => (SRAW (MOV(H|B)reg x) (LOCGR {s390x.GreaterOrEqual} <y.Type> y (MOVDconst <y.Type> [63]) (CMPUconst y [64]))) 296(Rsh(16|8)x32 x y) => (SRAW (MOV(H|B)reg x) (LOCGR {s390x.GreaterOrEqual} <y.Type> y (MOVDconst <y.Type> [63]) (CMPWUconst y [64]))) 297(Rsh(16|8)x16 x y) => (SRAW (MOV(H|B)reg x) (LOCGR {s390x.GreaterOrEqual} <y.Type> y (MOVDconst <y.Type> [63]) (CMPWUconst (MOVHZreg y) [64]))) 298(Rsh(16|8)x8 x y) => (SRAW (MOV(H|B)reg x) (LOCGR {s390x.GreaterOrEqual} <y.Type> y (MOVDconst <y.Type> [63]) (CMPWUconst (MOVBZreg y) [64]))) 299 300// Lowering rotates 301(RotateLeft8 <t> x (MOVDconst [c])) => (Or8 (Lsh8x64 <t> x (MOVDconst [c&7])) (Rsh8Ux64 <t> x (MOVDconst [-c&7]))) 302(RotateLeft16 <t> x (MOVDconst [c])) => (Or16 (Lsh16x64 <t> x (MOVDconst [c&15])) (Rsh16Ux64 <t> x (MOVDconst [-c&15]))) 303(RotateLeft32 ...) => (RLL ...) 304(RotateLeft64 ...) => (RLLG ...) 305 306// Lowering comparisons 307(Less64 x y) => (LOCGR {s390x.Less} (MOVDconst [0]) (MOVDconst [1]) (CMP x y)) 308(Less32 x y) => (LOCGR {s390x.Less} (MOVDconst [0]) (MOVDconst [1]) (CMPW x y)) 309(Less(16|8) x y) => (LOCGR {s390x.Less} (MOVDconst [0]) (MOVDconst [1]) (CMPW (MOV(H|B)reg x) (MOV(H|B)reg y))) 310(Less64U x y) => (LOCGR {s390x.Less} (MOVDconst [0]) (MOVDconst [1]) (CMPU x y)) 311(Less32U x y) => (LOCGR {s390x.Less} (MOVDconst [0]) (MOVDconst [1]) (CMPWU x y)) 312(Less(16|8)U x y) => (LOCGR {s390x.Less} (MOVDconst [0]) (MOVDconst [1]) (CMPWU (MOV(H|B)Zreg x) (MOV(H|B)Zreg y))) 313(Less64F x y) => (LOCGR {s390x.Less} (MOVDconst [0]) (MOVDconst [1]) (FCMP x y)) 314(Less32F x y) => (LOCGR {s390x.Less} (MOVDconst [0]) (MOVDconst [1]) (FCMPS x y)) 315 316(Leq64 x y) => (LOCGR {s390x.LessOrEqual} (MOVDconst [0]) (MOVDconst [1]) (CMP x y)) 317(Leq32 x y) => (LOCGR {s390x.LessOrEqual} (MOVDconst [0]) (MOVDconst [1]) (CMPW x y)) 318(Leq(16|8) x y) => (LOCGR {s390x.LessOrEqual} (MOVDconst [0]) (MOVDconst [1]) (CMPW (MOV(H|B)reg x) (MOV(H|B)reg y))) 319(Leq64U x y) => (LOCGR {s390x.LessOrEqual} (MOVDconst [0]) (MOVDconst [1]) (CMPU x y)) 320(Leq32U x y) => (LOCGR {s390x.LessOrEqual} (MOVDconst [0]) (MOVDconst [1]) (CMPWU x y)) 321(Leq(16|8)U x y) => (LOCGR {s390x.LessOrEqual} (MOVDconst [0]) (MOVDconst [1]) (CMPWU (MOV(H|B)Zreg x) (MOV(H|B)Zreg y))) 322(Leq64F x y) => (LOCGR {s390x.LessOrEqual} (MOVDconst [0]) (MOVDconst [1]) (FCMP x y)) 323(Leq32F x y) => (LOCGR {s390x.LessOrEqual} (MOVDconst [0]) (MOVDconst [1]) (FCMPS x y)) 324 325(Eq(64|Ptr) x y) => (LOCGR {s390x.Equal} (MOVDconst [0]) (MOVDconst [1]) (CMP x y)) 326(Eq32 x y) => (LOCGR {s390x.Equal} (MOVDconst [0]) (MOVDconst [1]) (CMPW x y)) 327(Eq(16|8|B) x y) => (LOCGR {s390x.Equal} (MOVDconst [0]) (MOVDconst [1]) (CMPW (MOV(H|B|B)reg x) (MOV(H|B|B)reg y))) 328(Eq64F x y) => (LOCGR {s390x.Equal} (MOVDconst [0]) (MOVDconst [1]) (FCMP x y)) 329(Eq32F x y) => (LOCGR {s390x.Equal} (MOVDconst [0]) (MOVDconst [1]) (FCMPS x y)) 330 331(Neq(64|Ptr) x y) => (LOCGR {s390x.NotEqual} (MOVDconst [0]) (MOVDconst [1]) (CMP x y)) 332(Neq32 x y) => (LOCGR {s390x.NotEqual} (MOVDconst [0]) (MOVDconst [1]) (CMPW x y)) 333(Neq(16|8|B) x y) => (LOCGR {s390x.NotEqual} (MOVDconst [0]) (MOVDconst [1]) (CMPW (MOV(H|B|B)reg x) (MOV(H|B|B)reg y))) 334(Neq64F x y) => (LOCGR {s390x.NotEqual} (MOVDconst [0]) (MOVDconst [1]) (FCMP x y)) 335(Neq32F x y) => (LOCGR {s390x.NotEqual} (MOVDconst [0]) (MOVDconst [1]) (FCMPS x y)) 336 337// Lowering loads 338(Load <t> ptr mem) && (is64BitInt(t) || isPtr(t)) => (MOVDload ptr mem) 339(Load <t> ptr mem) && is32BitInt(t) && t.IsSigned() => (MOVWload ptr mem) 340(Load <t> ptr mem) && is32BitInt(t) && !t.IsSigned() => (MOVWZload ptr mem) 341(Load <t> ptr mem) && is16BitInt(t) && t.IsSigned() => (MOVHload ptr mem) 342(Load <t> ptr mem) && is16BitInt(t) && !t.IsSigned() => (MOVHZload ptr mem) 343(Load <t> ptr mem) && is8BitInt(t) && t.IsSigned() => (MOVBload ptr mem) 344(Load <t> ptr mem) && (t.IsBoolean() || (is8BitInt(t) && !t.IsSigned())) => (MOVBZload ptr mem) 345(Load <t> ptr mem) && is32BitFloat(t) => (FMOVSload ptr mem) 346(Load <t> ptr mem) && is64BitFloat(t) => (FMOVDload ptr mem) 347 348// Lowering stores 349(Store {t} ptr val mem) && t.Size() == 8 && t.IsFloat() => (FMOVDstore ptr val mem) 350(Store {t} ptr val mem) && t.Size() == 4 && t.IsFloat() => (FMOVSstore ptr val mem) 351(Store {t} ptr val mem) && t.Size() == 8 && !t.IsFloat() => (MOVDstore ptr val mem) 352(Store {t} ptr val mem) && t.Size() == 4 && !t.IsFloat() => (MOVWstore ptr val mem) 353(Store {t} ptr val mem) && t.Size() == 2 => (MOVHstore ptr val mem) 354(Store {t} ptr val mem) && t.Size() == 1 => (MOVBstore ptr val mem) 355 356// Lowering moves 357 358// Load and store for small copies. 359(Move [0] _ _ mem) => mem 360(Move [1] dst src mem) => (MOVBstore dst (MOVBZload src mem) mem) 361(Move [2] dst src mem) => (MOVHstore dst (MOVHZload src mem) mem) 362(Move [4] dst src mem) => (MOVWstore dst (MOVWZload src mem) mem) 363(Move [8] dst src mem) => (MOVDstore dst (MOVDload src mem) mem) 364(Move [16] dst src mem) => 365 (MOVDstore [8] dst (MOVDload [8] src mem) 366 (MOVDstore dst (MOVDload src mem) mem)) 367(Move [24] dst src mem) => 368 (MOVDstore [16] dst (MOVDload [16] src mem) 369 (MOVDstore [8] dst (MOVDload [8] src mem) 370 (MOVDstore dst (MOVDload src mem) mem))) 371(Move [3] dst src mem) => 372 (MOVBstore [2] dst (MOVBZload [2] src mem) 373 (MOVHstore dst (MOVHZload src mem) mem)) 374(Move [5] dst src mem) => 375 (MOVBstore [4] dst (MOVBZload [4] src mem) 376 (MOVWstore dst (MOVWZload src mem) mem)) 377(Move [6] dst src mem) => 378 (MOVHstore [4] dst (MOVHZload [4] src mem) 379 (MOVWstore dst (MOVWZload src mem) mem)) 380(Move [7] dst src mem) => 381 (MOVBstore [6] dst (MOVBZload [6] src mem) 382 (MOVHstore [4] dst (MOVHZload [4] src mem) 383 (MOVWstore dst (MOVWZload src mem) mem))) 384 385// MVC for other moves. Use up to 4 instructions (sizes up to 1024 bytes). 386(Move [s] dst src mem) && s > 0 && s <= 256 && logLargeCopy(v, s) => 387 (MVC [makeValAndOff(int32(s), 0)] dst src mem) 388(Move [s] dst src mem) && s > 256 && s <= 512 && logLargeCopy(v, s) => 389 (MVC [makeValAndOff(int32(s)-256, 256)] dst src (MVC [makeValAndOff(256, 0)] dst src mem)) 390(Move [s] dst src mem) && s > 512 && s <= 768 && logLargeCopy(v, s) => 391 (MVC [makeValAndOff(int32(s)-512, 512)] dst src (MVC [makeValAndOff(256, 256)] dst src (MVC [makeValAndOff(256, 0)] dst src mem))) 392(Move [s] dst src mem) && s > 768 && s <= 1024 && logLargeCopy(v, s) => 393 (MVC [makeValAndOff(int32(s)-768, 768)] dst src (MVC [makeValAndOff(256, 512)] dst src (MVC [makeValAndOff(256, 256)] dst src (MVC [makeValAndOff(256, 0)] dst src mem)))) 394 395// Move more than 1024 bytes using a loop. 396(Move [s] dst src mem) && s > 1024 && logLargeCopy(v, s) => 397 (LoweredMove [s%256] dst src (ADD <src.Type> src (MOVDconst [(s/256)*256])) mem) 398 399// Lowering Zero instructions 400(Zero [0] _ mem) => mem 401(Zero [1] destptr mem) => (MOVBstoreconst [0] destptr mem) 402(Zero [2] destptr mem) => (MOVHstoreconst [0] destptr mem) 403(Zero [4] destptr mem) => (MOVWstoreconst [0] destptr mem) 404(Zero [8] destptr mem) => (MOVDstoreconst [0] destptr mem) 405(Zero [3] destptr mem) => 406 (MOVBstoreconst [makeValAndOff(0,2)] destptr 407 (MOVHstoreconst [0] destptr mem)) 408(Zero [5] destptr mem) => 409 (MOVBstoreconst [makeValAndOff(0,4)] destptr 410 (MOVWstoreconst [0] destptr mem)) 411(Zero [6] destptr mem) => 412 (MOVHstoreconst [makeValAndOff(0,4)] destptr 413 (MOVWstoreconst [0] destptr mem)) 414(Zero [7] destptr mem) => 415 (MOVWstoreconst [makeValAndOff(0,3)] destptr 416 (MOVWstoreconst [0] destptr mem)) 417 418(Zero [s] destptr mem) && s > 0 && s <= 1024 => 419 (CLEAR [makeValAndOff(int32(s), 0)] destptr mem) 420 421// Zero more than 1024 bytes using a loop. 422(Zero [s] destptr mem) && s > 1024 => 423 (LoweredZero [s%256] destptr (ADDconst <destptr.Type> destptr [(int32(s)/256)*256]) mem) 424 425// Lowering constants 426(Const(64|32|16|8) [val]) => (MOVDconst [int64(val)]) 427(Const(32|64)F ...) => (FMOV(S|D)const ...) 428(ConstNil) => (MOVDconst [0]) 429(ConstBool [t]) => (MOVDconst [b2i(t)]) 430 431// Lowering calls 432(StaticCall ...) => (CALLstatic ...) 433(ClosureCall ...) => (CALLclosure ...) 434(InterCall ...) => (CALLinter ...) 435(TailCall ...) => (CALLtail ...) 436 437// Miscellaneous 438(IsNonNil p) => (LOCGR {s390x.NotEqual} (MOVDconst [0]) (MOVDconst [1]) (CMPconst p [0])) 439(IsInBounds idx len) => (LOCGR {s390x.Less} (MOVDconst [0]) (MOVDconst [1]) (CMPU idx len)) 440(IsSliceInBounds idx len) => (LOCGR {s390x.LessOrEqual} (MOVDconst [0]) (MOVDconst [1]) (CMPU idx len)) 441(NilCheck ...) => (LoweredNilCheck ...) 442(GetG ...) => (LoweredGetG ...) 443(GetClosurePtr ...) => (LoweredGetClosurePtr ...) 444(GetCallerSP ...) => (LoweredGetCallerSP ...) 445(GetCallerPC ...) => (LoweredGetCallerPC ...) 446(Addr {sym} base) => (MOVDaddr {sym} base) 447(LocalAddr <t> {sym} base mem) && t.Elem().HasPointers() => (MOVDaddr {sym} (SPanchored base mem)) 448(LocalAddr <t> {sym} base _) && !t.Elem().HasPointers() => (MOVDaddr {sym} base) 449(ITab (Load ptr mem)) => (MOVDload ptr mem) 450 451// block rewrites 452(If cond yes no) => (CLIJ {s390x.LessOrGreater} (MOVBZreg <typ.Bool> cond) [0] yes no) 453 454// Write barrier. 455(WB ...) => (LoweredWB ...) 456 457(PanicBounds [kind] x y mem) && boundsABI(kind) == 0 => (LoweredPanicBoundsA [kind] x y mem) 458(PanicBounds [kind] x y mem) && boundsABI(kind) == 1 => (LoweredPanicBoundsB [kind] x y mem) 459(PanicBounds [kind] x y mem) && boundsABI(kind) == 2 => (LoweredPanicBoundsC [kind] x y mem) 460 461// *************************** 462// Above: lowering rules 463// Below: optimizations 464// *************************** 465// TODO: Should the optimizations be a separate pass? 466 467// Note: when removing unnecessary sign/zero extensions. 468// 469// After a value is spilled it is restored using a sign- or zero-extension 470// to register-width as appropriate for its type. For example, a uint8 will 471// be restored using a MOVBZ (llgc) instruction which will zero extend the 472// 8-bit value to 64-bits. 473// 474// This is a hazard when folding sign- and zero-extensions since we need to 475// ensure not only that the value in the argument register is correctly 476// extended but also that it will still be correctly extended if it is 477// spilled and restored. 478// 479// In general this means we need type checks when the RHS of a rule is an 480// OpCopy (i.e. "(... x:(...) ...) -> x"). 481 482// Merge double extensions. 483(MOV(H|HZ)reg e:(MOV(B|BZ)reg x)) && clobberIfDead(e) => (MOV(B|BZ)reg x) 484(MOV(W|WZ)reg e:(MOV(B|BZ)reg x)) && clobberIfDead(e) => (MOV(B|BZ)reg x) 485(MOV(W|WZ)reg e:(MOV(H|HZ)reg x)) && clobberIfDead(e) => (MOV(H|HZ)reg x) 486 487// Bypass redundant sign extensions. 488(MOV(B|BZ)reg e:(MOVBreg x)) && clobberIfDead(e) => (MOV(B|BZ)reg x) 489(MOV(B|BZ)reg e:(MOVHreg x)) && clobberIfDead(e) => (MOV(B|BZ)reg x) 490(MOV(B|BZ)reg e:(MOVWreg x)) && clobberIfDead(e) => (MOV(B|BZ)reg x) 491(MOV(H|HZ)reg e:(MOVHreg x)) && clobberIfDead(e) => (MOV(H|HZ)reg x) 492(MOV(H|HZ)reg e:(MOVWreg x)) && clobberIfDead(e) => (MOV(H|HZ)reg x) 493(MOV(W|WZ)reg e:(MOVWreg x)) && clobberIfDead(e) => (MOV(W|WZ)reg x) 494 495// Bypass redundant zero extensions. 496(MOV(B|BZ)reg e:(MOVBZreg x)) && clobberIfDead(e) => (MOV(B|BZ)reg x) 497(MOV(B|BZ)reg e:(MOVHZreg x)) && clobberIfDead(e) => (MOV(B|BZ)reg x) 498(MOV(B|BZ)reg e:(MOVWZreg x)) && clobberIfDead(e) => (MOV(B|BZ)reg x) 499(MOV(H|HZ)reg e:(MOVHZreg x)) && clobberIfDead(e) => (MOV(H|HZ)reg x) 500(MOV(H|HZ)reg e:(MOVWZreg x)) && clobberIfDead(e) => (MOV(H|HZ)reg x) 501(MOV(W|WZ)reg e:(MOVWZreg x)) && clobberIfDead(e) => (MOV(W|WZ)reg x) 502 503// Remove zero extensions after zero extending load. 504// Note: take care that if x is spilled it is restored correctly. 505(MOV(B|H|W)Zreg x:(MOVBZload _ _)) && (!x.Type.IsSigned() || x.Type.Size() > 1) => x 506(MOV(H|W)Zreg x:(MOVHZload _ _)) && (!x.Type.IsSigned() || x.Type.Size() > 2) => x 507(MOVWZreg x:(MOVWZload _ _)) && (!x.Type.IsSigned() || x.Type.Size() > 4) => x 508 509// Remove sign extensions after sign extending load. 510// Note: take care that if x is spilled it is restored correctly. 511(MOV(B|H|W)reg x:(MOVBload _ _)) && (x.Type.IsSigned() || x.Type.Size() == 8) => x 512(MOV(H|W)reg x:(MOVHload _ _)) && (x.Type.IsSigned() || x.Type.Size() == 8) => x 513(MOVWreg x:(MOVWload _ _)) && (x.Type.IsSigned() || x.Type.Size() == 8) => x 514 515// Remove sign extensions after zero extending load. 516// These type checks are probably unnecessary but do them anyway just in case. 517(MOV(H|W)reg x:(MOVBZload _ _)) && (!x.Type.IsSigned() || x.Type.Size() > 1) => x 518(MOVWreg x:(MOVHZload _ _)) && (!x.Type.IsSigned() || x.Type.Size() > 2) => x 519 520// Fold sign and zero extensions into loads. 521// 522// Note: The combined instruction must end up in the same block 523// as the original load. If not, we end up making a value with 524// memory type live in two different blocks, which can lead to 525// multiple memory values alive simultaneously. 526// 527// Make sure we don't combine these ops if the load has another use. 528// This prevents a single load from being split into multiple loads 529// which then might return different values. See test/atomicload.go. 530(MOV(B|H|W)Zreg <t> x:(MOV(B|H|W)load [o] {s} p mem)) 531 && x.Uses == 1 532 && clobber(x) 533 => @x.Block (MOV(B|H|W)Zload <t> [o] {s} p mem) 534(MOV(B|H|W)reg <t> x:(MOV(B|H|W)Zload [o] {s} p mem)) 535 && x.Uses == 1 536 && clobber(x) 537 => @x.Block (MOV(B|H|W)load <t> [o] {s} p mem) 538 539// Remove zero extensions after argument load. 540(MOVBZreg x:(Arg <t>)) && !t.IsSigned() && t.Size() == 1 => x 541(MOVHZreg x:(Arg <t>)) && !t.IsSigned() && t.Size() <= 2 => x 542(MOVWZreg x:(Arg <t>)) && !t.IsSigned() && t.Size() <= 4 => x 543 544// Remove sign extensions after argument load. 545(MOVBreg x:(Arg <t>)) && t.IsSigned() && t.Size() == 1 => x 546(MOVHreg x:(Arg <t>)) && t.IsSigned() && t.Size() <= 2 => x 547(MOVWreg x:(Arg <t>)) && t.IsSigned() && t.Size() <= 4 => x 548 549// Fold zero extensions into constants. 550(MOVBZreg (MOVDconst [c])) => (MOVDconst [int64( uint8(c))]) 551(MOVHZreg (MOVDconst [c])) => (MOVDconst [int64(uint16(c))]) 552(MOVWZreg (MOVDconst [c])) => (MOVDconst [int64(uint32(c))]) 553 554// Fold sign extensions into constants. 555(MOVBreg (MOVDconst [c])) => (MOVDconst [int64( int8(c))]) 556(MOVHreg (MOVDconst [c])) => (MOVDconst [int64(int16(c))]) 557(MOVWreg (MOVDconst [c])) => (MOVDconst [int64(int32(c))]) 558 559// Remove zero extension of conditional move. 560// Note: only for MOVBZreg for now since it is added as part of 'if' statement lowering. 561(MOVBZreg x:(LOCGR (MOVDconst [c]) (MOVDconst [d]) _)) 562 && int64(uint8(c)) == c 563 && int64(uint8(d)) == d 564 && (!x.Type.IsSigned() || x.Type.Size() > 1) 565 => x 566 567// Fold boolean tests into blocks. 568// Note: this must match If statement lowering. 569(CLIJ {s390x.LessOrGreater} (LOCGR {d} (MOVDconst [0]) (MOVDconst [x]) cmp) [0] yes no) 570 && int32(x) != 0 571 => (BRC {d} cmp yes no) 572 573// Canonicalize BRC condition code mask by removing impossible conditions. 574// Integer comparisons cannot generate the unordered condition. 575(BRC {c} x:((CMP|CMPW|CMPU|CMPWU) _ _) yes no) && c&s390x.Unordered != 0 => (BRC {c&^s390x.Unordered} x yes no) 576(BRC {c} x:((CMP|CMPW|CMPU|CMPWU)const _) yes no) && c&s390x.Unordered != 0 => (BRC {c&^s390x.Unordered} x yes no) 577 578// Compare-and-branch. 579// Note: bit 3 (unordered) must not be set so we mask out s390x.Unordered. 580(BRC {c} (CMP x y) yes no) => (CGRJ {c&^s390x.Unordered} x y yes no) 581(BRC {c} (CMPW x y) yes no) => (CRJ {c&^s390x.Unordered} x y yes no) 582(BRC {c} (CMPU x y) yes no) => (CLGRJ {c&^s390x.Unordered} x y yes no) 583(BRC {c} (CMPWU x y) yes no) => (CLRJ {c&^s390x.Unordered} x y yes no) 584 585// Compare-and-branch (immediate). 586// Note: bit 3 (unordered) must not be set so we mask out s390x.Unordered. 587(BRC {c} (CMPconst x [y]) yes no) && y == int32( int8(y)) => (CGIJ {c&^s390x.Unordered} x [ int8(y)] yes no) 588(BRC {c} (CMPWconst x [y]) yes no) && y == int32( int8(y)) => (CIJ {c&^s390x.Unordered} x [ int8(y)] yes no) 589(BRC {c} (CMPUconst x [y]) yes no) && y == int32(uint8(y)) => (CLGIJ {c&^s390x.Unordered} x [uint8(y)] yes no) 590(BRC {c} (CMPWUconst x [y]) yes no) && y == int32(uint8(y)) => (CLIJ {c&^s390x.Unordered} x [uint8(y)] yes no) 591 592// Absorb immediate into compare-and-branch. 593(C(R|GR)J {c} x (MOVDconst [y]) yes no) && is8Bit(y) => (C(I|GI)J {c} x [ int8(y)] yes no) 594(CL(R|GR)J {c} x (MOVDconst [y]) yes no) && isU8Bit(y) => (CL(I|GI)J {c} x [uint8(y)] yes no) 595(C(R|GR)J {c} (MOVDconst [x]) y yes no) && is8Bit(x) => (C(I|GI)J {c.ReverseComparison()} y [ int8(x)] yes no) 596(CL(R|GR)J {c} (MOVDconst [x]) y yes no) && isU8Bit(x) => (CL(I|GI)J {c.ReverseComparison()} y [uint8(x)] yes no) 597 598// Prefer comparison with immediate to compare-and-branch. 599(CGRJ {c} x (MOVDconst [y]) yes no) && !is8Bit(y) && is32Bit(y) => (BRC {c} (CMPconst x [int32(y)]) yes no) 600(CRJ {c} x (MOVDconst [y]) yes no) && !is8Bit(y) && is32Bit(y) => (BRC {c} (CMPWconst x [int32(y)]) yes no) 601(CLGRJ {c} x (MOVDconst [y]) yes no) && !isU8Bit(y) && isU32Bit(y) => (BRC {c} (CMPUconst x [int32(y)]) yes no) 602(CLRJ {c} x (MOVDconst [y]) yes no) && !isU8Bit(y) && isU32Bit(y) => (BRC {c} (CMPWUconst x [int32(y)]) yes no) 603(CGRJ {c} (MOVDconst [x]) y yes no) && !is8Bit(x) && is32Bit(x) => (BRC {c.ReverseComparison()} (CMPconst y [int32(x)]) yes no) 604(CRJ {c} (MOVDconst [x]) y yes no) && !is8Bit(x) && is32Bit(x) => (BRC {c.ReverseComparison()} (CMPWconst y [int32(x)]) yes no) 605(CLGRJ {c} (MOVDconst [x]) y yes no) && !isU8Bit(x) && isU32Bit(x) => (BRC {c.ReverseComparison()} (CMPUconst y [int32(x)]) yes no) 606(CLRJ {c} (MOVDconst [x]) y yes no) && !isU8Bit(x) && isU32Bit(x) => (BRC {c.ReverseComparison()} (CMPWUconst y [int32(x)]) yes no) 607 608// Absorb sign/zero extensions into 32-bit compare-and-branch. 609(CIJ {c} (MOV(W|WZ)reg x) [y] yes no) => (CIJ {c} x [y] yes no) 610(CLIJ {c} (MOV(W|WZ)reg x) [y] yes no) => (CLIJ {c} x [y] yes no) 611 612// Bring out-of-range signed immediates into range by varying branch condition. 613(BRC {s390x.Less} (CMPconst x [ 128]) yes no) => (CGIJ {s390x.LessOrEqual} x [ 127] yes no) 614(BRC {s390x.Less} (CMPWconst x [ 128]) yes no) => (CIJ {s390x.LessOrEqual} x [ 127] yes no) 615(BRC {s390x.LessOrEqual} (CMPconst x [-129]) yes no) => (CGIJ {s390x.Less} x [-128] yes no) 616(BRC {s390x.LessOrEqual} (CMPWconst x [-129]) yes no) => (CIJ {s390x.Less} x [-128] yes no) 617(BRC {s390x.Greater} (CMPconst x [-129]) yes no) => (CGIJ {s390x.GreaterOrEqual} x [-128] yes no) 618(BRC {s390x.Greater} (CMPWconst x [-129]) yes no) => (CIJ {s390x.GreaterOrEqual} x [-128] yes no) 619(BRC {s390x.GreaterOrEqual} (CMPconst x [ 128]) yes no) => (CGIJ {s390x.Greater} x [ 127] yes no) 620(BRC {s390x.GreaterOrEqual} (CMPWconst x [ 128]) yes no) => (CIJ {s390x.Greater} x [ 127] yes no) 621 622// Bring out-of-range unsigned immediates into range by varying branch condition. 623(BRC {s390x.Less} (CMP(WU|U)const x [256]) yes no) => (C(L|LG)IJ {s390x.LessOrEqual} x [255] yes no) 624(BRC {s390x.GreaterOrEqual} (CMP(WU|U)const x [256]) yes no) => (C(L|LG)IJ {s390x.Greater} x [255] yes no) 625 626// Bring out-of-range immediates into range by switching signedness (only == and !=). 627(BRC {c} (CMPconst x [y]) yes no) && y == int32(uint8(y)) && (c == s390x.Equal || c == s390x.LessOrGreater) => (CLGIJ {c} x [uint8(y)] yes no) 628(BRC {c} (CMPWconst x [y]) yes no) && y == int32(uint8(y)) && (c == s390x.Equal || c == s390x.LessOrGreater) => (CLIJ {c} x [uint8(y)] yes no) 629(BRC {c} (CMPUconst x [y]) yes no) && y == int32( int8(y)) && (c == s390x.Equal || c == s390x.LessOrGreater) => (CGIJ {c} x [ int8(y)] yes no) 630(BRC {c} (CMPWUconst x [y]) yes no) && y == int32( int8(y)) && (c == s390x.Equal || c == s390x.LessOrGreater) => (CIJ {c} x [ int8(y)] yes no) 631 632// Fold constants into instructions. 633(ADD x (MOVDconst <t> [c])) && is32Bit(c) && !t.IsPtr() => (ADDconst [int32(c)] x) 634(ADDW x (MOVDconst [c])) => (ADDWconst [int32(c)] x) 635 636(SUB x (MOVDconst [c])) && is32Bit(c) => (SUBconst x [int32(c)]) 637(SUB (MOVDconst [c]) x) && is32Bit(c) => (NEG (SUBconst <v.Type> x [int32(c)])) 638(SUBW x (MOVDconst [c])) => (SUBWconst x [int32(c)]) 639(SUBW (MOVDconst [c]) x) => (NEGW (SUBWconst <v.Type> x [int32(c)])) 640 641(MULLD x (MOVDconst [c])) && is32Bit(c) => (MULLDconst [int32(c)] x) 642(MULLW x (MOVDconst [c])) => (MULLWconst [int32(c)] x) 643 644// NILF instructions leave the high 32 bits unchanged which is 645// equivalent to the leftmost 32 bits being set. 646// TODO(mundaym): modify the assembler to accept 64-bit values 647// and use isU32Bit(^c). 648(AND x (MOVDconst [c])) 649 && s390x.NewRotateParams(0, 63, 0).OutMerge(uint64(c)) != nil 650 => (RISBGZ x {*s390x.NewRotateParams(0, 63, 0).OutMerge(uint64(c))}) 651(AND x (MOVDconst [c])) 652 && is32Bit(c) 653 && c < 0 654 => (ANDconst [c] x) 655(AND x (MOVDconst [c])) 656 && is32Bit(c) 657 && c >= 0 658 => (MOVWZreg (ANDWconst <typ.UInt32> [int32(c)] x)) 659 660(ANDW x (MOVDconst [c])) => (ANDWconst [int32(c)] x) 661 662((AND|ANDW)const [c] ((AND|ANDW)const [d] x)) => ((AND|ANDW)const [c&d] x) 663 664((OR|XOR) x (MOVDconst [c])) && isU32Bit(c) => ((OR|XOR)const [c] x) 665((OR|XOR)W x (MOVDconst [c])) => ((OR|XOR)Wconst [int32(c)] x) 666 667// Constant shifts. 668(S(LD|RD|RAD) x (MOVDconst [c])) => (S(LD|RD|RAD)const x [uint8(c&63)]) 669(S(LW|RW|RAW) x (MOVDconst [c])) && c&32 == 0 => (S(LW|RW|RAW)const x [uint8(c&31)]) 670(S(LW|RW) _ (MOVDconst [c])) && c&32 != 0 => (MOVDconst [0]) 671(SRAW x (MOVDconst [c])) && c&32 != 0 => (SRAWconst x [31]) 672 673// Shifts only use the rightmost 6 bits of the shift value. 674(S(LD|RD|RAD|LW|RW|RAW) x (RISBGZ y {r})) 675 && r.Amount == 0 676 && r.OutMask()&63 == 63 677 => (S(LD|RD|RAD|LW|RW|RAW) x y) 678(S(LD|RD|RAD|LW|RW|RAW) x (AND (MOVDconst [c]) y)) 679 => (S(LD|RD|RAD|LW|RW|RAW) x (ANDWconst <typ.UInt32> [int32(c&63)] y)) 680(S(LD|RD|RAD|LW|RW|RAW) x (ANDWconst [c] y)) && c&63 == 63 681 => (S(LD|RD|RAD|LW|RW|RAW) x y) 682(SLD x (MOV(W|H|B|WZ|HZ|BZ)reg y)) => (SLD x y) 683(SRD x (MOV(W|H|B|WZ|HZ|BZ)reg y)) => (SRD x y) 684(SRAD x (MOV(W|H|B|WZ|HZ|BZ)reg y)) => (SRAD x y) 685(SLW x (MOV(W|H|B|WZ|HZ|BZ)reg y)) => (SLW x y) 686(SRW x (MOV(W|H|B|WZ|HZ|BZ)reg y)) => (SRW x y) 687(SRAW x (MOV(W|H|B|WZ|HZ|BZ)reg y)) => (SRAW x y) 688 689// Match rotate by constant. 690(RLLG x (MOVDconst [c])) => (RISBGZ x {s390x.NewRotateParams(0, 63, uint8(c&63))}) 691(RLL x (MOVDconst [c])) => (RLLconst x [uint8(c&31)]) 692 693// Signed 64-bit comparison with immediate. 694(CMP x (MOVDconst [c])) && is32Bit(c) => (CMPconst x [int32(c)]) 695(CMP (MOVDconst [c]) x) && is32Bit(c) => (InvertFlags (CMPconst x [int32(c)])) 696 697// Unsigned 64-bit comparison with immediate. 698(CMPU x (MOVDconst [c])) && isU32Bit(c) => (CMPUconst x [int32(c)]) 699(CMPU (MOVDconst [c]) x) && isU32Bit(c) => (InvertFlags (CMPUconst x [int32(c)])) 700 701// Signed and unsigned 32-bit comparison with immediate. 702(CMP(W|WU) x (MOVDconst [c])) => (CMP(W|WU)const x [int32(c)]) 703(CMP(W|WU) (MOVDconst [c]) x) => (InvertFlags (CMP(W|WU)const x [int32(c)])) 704 705// Match (x >> c) << d to 'rotate then insert selected bits [into zero]'. 706(SLDconst (SRDconst x [c]) [d]) => (RISBGZ x {s390x.NewRotateParams(uint8(max8(0, int8(c-d))), 63-d, uint8(int8(d-c)&63))}) 707 708// Match (x << c) >> d to 'rotate then insert selected bits [into zero]'. 709(SRDconst (SLDconst x [c]) [d]) => (RISBGZ x {s390x.NewRotateParams(d, uint8(min8(63, int8(63-c+d))), uint8(int8(c-d)&63))}) 710 711// Absorb input zero extension into 'rotate then insert selected bits [into zero]'. 712(RISBGZ (MOVWZreg x) {r}) && r.InMerge(0xffffffff) != nil => (RISBGZ x {*r.InMerge(0xffffffff)}) 713(RISBGZ (MOVHZreg x) {r}) && r.InMerge(0x0000ffff) != nil => (RISBGZ x {*r.InMerge(0x0000ffff)}) 714(RISBGZ (MOVBZreg x) {r}) && r.InMerge(0x000000ff) != nil => (RISBGZ x {*r.InMerge(0x000000ff)}) 715 716// Absorb 'rotate then insert selected bits [into zero]' into zero extension. 717(MOVWZreg (RISBGZ x {r})) && r.OutMerge(0xffffffff) != nil => (RISBGZ x {*r.OutMerge(0xffffffff)}) 718(MOVHZreg (RISBGZ x {r})) && r.OutMerge(0x0000ffff) != nil => (RISBGZ x {*r.OutMerge(0x0000ffff)}) 719(MOVBZreg (RISBGZ x {r})) && r.OutMerge(0x000000ff) != nil => (RISBGZ x {*r.OutMerge(0x000000ff)}) 720 721// Absorb shift into 'rotate then insert selected bits [into zero]'. 722// 723// Any unsigned shift can be represented as a rotate and mask operation: 724// 725// x << c => RotateLeft64(x, c) & (^uint64(0) << c) 726// x >> c => RotateLeft64(x, -c) & (^uint64(0) >> c) 727// 728// Therefore when a shift is used as the input to a rotate then insert 729// selected bits instruction we can merge the two together. We just have 730// to be careful that the resultant mask is representable (non-zero and 731// contiguous). For example, assuming that x is variable and c, y and m 732// are constants, a shift followed by a rotate then insert selected bits 733// could be represented as: 734// 735// RotateLeft64(RotateLeft64(x, c) & (^uint64(0) << c), y) & m 736// 737// We can split the rotation by y into two, one rotate for x and one for 738// the mask: 739// 740// RotateLeft64(RotateLeft64(x, c), y) & (RotateLeft64(^uint64(0) << c, y)) & m 741// 742// The rotations of x by c followed by y can then be combined: 743// 744// RotateLeft64(x, c+y) & (RotateLeft64(^uint64(0) << c, y)) & m 745// ^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 746// rotate mask 747// 748// To perform this optimization we therefore just need to check that it 749// is valid to merge the shift mask (^(uint64(0)<<c)) into the selected 750// bits mask (i.e. that the resultant mask is non-zero and contiguous). 751// 752(RISBGZ (SLDconst x [c]) {r}) && r.InMerge(^uint64(0)<<c) != nil => (RISBGZ x {(*r.InMerge(^uint64(0)<<c)).RotateLeft(c)}) 753(RISBGZ (SRDconst x [c]) {r}) && r.InMerge(^uint64(0)>>c) != nil => (RISBGZ x {(*r.InMerge(^uint64(0)>>c)).RotateLeft(-c)}) 754 755// Absorb 'rotate then insert selected bits [into zero]' into left shift. 756(SLDconst (RISBGZ x {r}) [c]) 757 && s390x.NewRotateParams(0, 63-c, c).InMerge(r.OutMask()) != nil 758 => (RISBGZ x {(*s390x.NewRotateParams(0, 63-c, c).InMerge(r.OutMask())).RotateLeft(r.Amount)}) 759 760// Absorb 'rotate then insert selected bits [into zero]' into right shift. 761(SRDconst (RISBGZ x {r}) [c]) 762 && s390x.NewRotateParams(c, 63, -c&63).InMerge(r.OutMask()) != nil 763 => (RISBGZ x {(*s390x.NewRotateParams(c, 63, -c&63).InMerge(r.OutMask())).RotateLeft(r.Amount)}) 764 765// Merge 'rotate then insert selected bits [into zero]' instructions together. 766(RISBGZ (RISBGZ x {y}) {z}) 767 && z.InMerge(y.OutMask()) != nil 768 => (RISBGZ x {(*z.InMerge(y.OutMask())).RotateLeft(y.Amount)}) 769 770// Convert RISBGZ into 64-bit shift (helps CSE). 771(RISBGZ x {r}) && r.End == 63 && r.Start == -r.Amount&63 => (SRDconst x [-r.Amount&63]) 772(RISBGZ x {r}) && r.Start == 0 && r.End == 63-r.Amount => (SLDconst x [r.Amount]) 773 774// Optimize single bit isolation when it is known to be equivalent to 775// the most significant bit due to mask produced by arithmetic shift. 776// Simply isolate the most significant bit itself and place it in the 777// correct position. 778// 779// Example: (int64(x) >> 63) & 0x8 -> RISBGZ $60, $60, $4, Rsrc, Rdst 780(RISBGZ (SRADconst x [c]) {r}) 781 && r.Start == r.End // single bit selected 782 && (r.Start+r.Amount)&63 <= c // equivalent to most significant bit of x 783 => (RISBGZ x {s390x.NewRotateParams(r.Start, r.Start, -r.Start&63)}) 784 785// Canonicalize the order of arguments to comparisons - helps with CSE. 786((CMP|CMPW|CMPU|CMPWU) x y) && canonLessThan(x,y) => (InvertFlags ((CMP|CMPW|CMPU|CMPWU) y x)) 787 788// Use sign/zero extend instead of RISBGZ. 789(RISBGZ x {r}) && r == s390x.NewRotateParams(56, 63, 0) => (MOVBZreg x) 790(RISBGZ x {r}) && r == s390x.NewRotateParams(48, 63, 0) => (MOVHZreg x) 791(RISBGZ x {r}) && r == s390x.NewRotateParams(32, 63, 0) => (MOVWZreg x) 792 793// Use sign/zero extend instead of ANDW. 794(ANDWconst [0x00ff] x) => (MOVBZreg x) 795(ANDWconst [0xffff] x) => (MOVHZreg x) 796 797// Strength reduce multiplication to the sum (or difference) of two powers of two. 798// 799// Examples: 800// 5x -> 4x + 1x 801// 10x -> 8x + 2x 802// 120x -> 128x - 8x 803// -120x -> 8x - 128x 804// 805// We know that the rightmost bit of any positive value, once isolated, must either 806// be a power of 2 (because it is a single bit) or 0 (if the original value is 0). 807// In all of these rules we use a rightmost bit calculation to determine one operand 808// for the addition or subtraction. We then just need to calculate if the other 809// operand is a valid power of 2 before we can match the rule. 810// 811// Notes: 812// - the generic rules have already matched single powers of two so we ignore them here 813// - isPowerOfTwo32 asserts that its argument is greater than 0 814// - c&(c-1) = clear rightmost bit 815// - c&^(c-1) = isolate rightmost bit 816 817// c = 2ˣ + 2ʸ => c - 2ˣ = 2ʸ 818(MULL(D|W)const <t> x [c]) && isPowerOfTwo32(c&(c-1)) 819 => ((ADD|ADDW) (SL(D|W)const <t> x [uint8(log32(c&(c-1)))]) 820 (SL(D|W)const <t> x [uint8(log32(c&^(c-1)))])) 821 822// c = 2ʸ - 2ˣ => c + 2ˣ = 2ʸ 823(MULL(D|W)const <t> x [c]) && isPowerOfTwo32(c+(c&^(c-1))) 824 => ((SUB|SUBW) (SL(D|W)const <t> x [uint8(log32(c+(c&^(c-1))))]) 825 (SL(D|W)const <t> x [uint8(log32(c&^(c-1)))])) 826 827// c = 2ˣ - 2ʸ => -c + 2ˣ = 2ʸ 828(MULL(D|W)const <t> x [c]) && isPowerOfTwo32(-c+(-c&^(-c-1))) 829 => ((SUB|SUBW) (SL(D|W)const <t> x [uint8(log32(-c&^(-c-1)))]) 830 (SL(D|W)const <t> x [uint8(log32(-c+(-c&^(-c-1))))])) 831 832// Fold ADD into MOVDaddr. Odd offsets from SB shouldn't be folded (LARL can't handle them). 833(ADDconst [c] (MOVDaddr [d] {s} x:(SB))) && ((c+d)&1 == 0) && is32Bit(int64(c)+int64(d)) => (MOVDaddr [c+d] {s} x) 834(ADDconst [c] (MOVDaddr [d] {s} x)) && x.Op != OpSB && is20Bit(int64(c)+int64(d)) => (MOVDaddr [c+d] {s} x) 835(ADD idx (MOVDaddr [c] {s} ptr)) && ptr.Op != OpSB => (MOVDaddridx [c] {s} ptr idx) 836 837// fold ADDconst into MOVDaddrx 838(ADDconst [c] (MOVDaddridx [d] {s} x y)) && is20Bit(int64(c)+int64(d)) => (MOVDaddridx [c+d] {s} x y) 839(MOVDaddridx [c] {s} (ADDconst [d] x) y) && is20Bit(int64(c)+int64(d)) => (MOVDaddridx [c+d] {s} x y) 840(MOVDaddridx [c] {s} x (ADDconst [d] y)) && is20Bit(int64(c)+int64(d)) => (MOVDaddridx [c+d] {s} x y) 841 842// reverse ordering of compare instruction 843(LOCGR {c} x y (InvertFlags cmp)) => (LOCGR {c.ReverseComparison()} x y cmp) 844 845// replace load from same location as preceding store with copy 846(MOVDload [off] {sym} ptr1 (MOVDstore [off] {sym} ptr2 x _)) && isSamePtr(ptr1, ptr2) => x 847(MOVWload [off] {sym} ptr1 (MOVWstore [off] {sym} ptr2 x _)) && isSamePtr(ptr1, ptr2) => (MOVWreg x) 848(MOVHload [off] {sym} ptr1 (MOVHstore [off] {sym} ptr2 x _)) && isSamePtr(ptr1, ptr2) => (MOVHreg x) 849(MOVBload [off] {sym} ptr1 (MOVBstore [off] {sym} ptr2 x _)) && isSamePtr(ptr1, ptr2) => (MOVBreg x) 850(MOVWZload [off] {sym} ptr1 (MOVWstore [off] {sym} ptr2 x _)) && isSamePtr(ptr1, ptr2) => (MOVWZreg x) 851(MOVHZload [off] {sym} ptr1 (MOVHstore [off] {sym} ptr2 x _)) && isSamePtr(ptr1, ptr2) => (MOVHZreg x) 852(MOVBZload [off] {sym} ptr1 (MOVBstore [off] {sym} ptr2 x _)) && isSamePtr(ptr1, ptr2) => (MOVBZreg x) 853(MOVDload [off] {sym} ptr1 (FMOVDstore [off] {sym} ptr2 x _)) && isSamePtr(ptr1, ptr2) => (LGDR x) 854(FMOVDload [off] {sym} ptr1 (MOVDstore [off] {sym} ptr2 x _)) && isSamePtr(ptr1, ptr2) => (LDGR x) 855(FMOVDload [off] {sym} ptr1 (FMOVDstore [off] {sym} ptr2 x _)) && isSamePtr(ptr1, ptr2) => x 856(FMOVSload [off] {sym} ptr1 (FMOVSstore [off] {sym} ptr2 x _)) && isSamePtr(ptr1, ptr2) => x 857 858// prefer FPR <-> GPR moves over combined load ops 859(MULLDload <t> [off] {sym} x ptr1 (FMOVDstore [off] {sym} ptr2 y _)) && isSamePtr(ptr1, ptr2) => (MULLD x (LGDR <t> y)) 860(ADDload <t> [off] {sym} x ptr1 (FMOVDstore [off] {sym} ptr2 y _)) && isSamePtr(ptr1, ptr2) => (ADD x (LGDR <t> y)) 861(SUBload <t> [off] {sym} x ptr1 (FMOVDstore [off] {sym} ptr2 y _)) && isSamePtr(ptr1, ptr2) => (SUB x (LGDR <t> y)) 862(ORload <t> [off] {sym} x ptr1 (FMOVDstore [off] {sym} ptr2 y _)) && isSamePtr(ptr1, ptr2) => (OR x (LGDR <t> y)) 863(ANDload <t> [off] {sym} x ptr1 (FMOVDstore [off] {sym} ptr2 y _)) && isSamePtr(ptr1, ptr2) => (AND x (LGDR <t> y)) 864(XORload <t> [off] {sym} x ptr1 (FMOVDstore [off] {sym} ptr2 y _)) && isSamePtr(ptr1, ptr2) => (XOR x (LGDR <t> y)) 865 866// detect attempts to set/clear the sign bit 867// may need to be reworked when NIHH/OIHH are added 868(RISBGZ (LGDR <t> x) {r}) && r == s390x.NewRotateParams(1, 63, 0) => (LGDR <t> (LPDFR <x.Type> x)) 869(LDGR <t> (RISBGZ x {r})) && r == s390x.NewRotateParams(1, 63, 0) => (LPDFR (LDGR <t> x)) 870(OR (MOVDconst [-1<<63]) (LGDR <t> x)) => (LGDR <t> (LNDFR <x.Type> x)) 871(LDGR <t> (OR (MOVDconst [-1<<63]) x)) => (LNDFR (LDGR <t> x)) 872 873// detect attempts to set the sign bit with load 874(LDGR <t> x:(ORload <t1> [off] {sym} (MOVDconst [-1<<63]) ptr mem)) && x.Uses == 1 && clobber(x) => @x.Block (LNDFR <t> (LDGR <t> (MOVDload <t1> [off] {sym} ptr mem))) 875 876// detect copysign 877(OR (RISBGZ (LGDR x) {r}) (LGDR (LPDFR <t> y))) 878 && r == s390x.NewRotateParams(0, 0, 0) 879 => (LGDR (CPSDR <t> y x)) 880(OR (RISBGZ (LGDR x) {r}) (MOVDconst [c])) 881 && c >= 0 882 && r == s390x.NewRotateParams(0, 0, 0) 883 => (LGDR (CPSDR <x.Type> (FMOVDconst <x.Type> [math.Float64frombits(uint64(c))]) x)) 884(CPSDR y (FMOVDconst [c])) && !math.Signbit(c) => (LPDFR y) 885(CPSDR y (FMOVDconst [c])) && math.Signbit(c) => (LNDFR y) 886 887// absorb negations into set/clear sign bit 888(FNEG (LPDFR x)) => (LNDFR x) 889(FNEG (LNDFR x)) => (LPDFR x) 890(FNEGS (LPDFR x)) => (LNDFR x) 891(FNEGS (LNDFR x)) => (LPDFR x) 892 893// no need to convert float32 to float64 to set/clear sign bit 894(LEDBR (LPDFR (LDEBR x))) => (LPDFR x) 895(LEDBR (LNDFR (LDEBR x))) => (LNDFR x) 896 897// remove unnecessary FPR <-> GPR moves 898(LDGR (LGDR x)) => x 899(LGDR (LDGR x)) => x 900 901// Don't extend before storing 902(MOVWstore [off] {sym} ptr (MOVWreg x) mem) => (MOVWstore [off] {sym} ptr x mem) 903(MOVHstore [off] {sym} ptr (MOVHreg x) mem) => (MOVHstore [off] {sym} ptr x mem) 904(MOVBstore [off] {sym} ptr (MOVBreg x) mem) => (MOVBstore [off] {sym} ptr x mem) 905(MOVWstore [off] {sym} ptr (MOVWZreg x) mem) => (MOVWstore [off] {sym} ptr x mem) 906(MOVHstore [off] {sym} ptr (MOVHZreg x) mem) => (MOVHstore [off] {sym} ptr x mem) 907(MOVBstore [off] {sym} ptr (MOVBZreg x) mem) => (MOVBstore [off] {sym} ptr x mem) 908 909// Fold constants into memory operations. 910// Note that this is not always a good idea because if not all the uses of 911// the ADDconst get eliminated, we still have to compute the ADDconst and we now 912// have potentially two live values (ptr and (ADDconst [off] ptr)) instead of one. 913// Nevertheless, let's do it! 914(MOVDload [off1] {sym} (ADDconst [off2] ptr) mem) && is20Bit(int64(off1)+int64(off2)) => (MOVDload [off1+off2] {sym} ptr mem) 915(MOVWload [off1] {sym} (ADDconst [off2] ptr) mem) && is20Bit(int64(off1)+int64(off2)) => (MOVWload [off1+off2] {sym} ptr mem) 916(MOVHload [off1] {sym} (ADDconst [off2] ptr) mem) && is20Bit(int64(off1)+int64(off2)) => (MOVHload [off1+off2] {sym} ptr mem) 917(MOVBload [off1] {sym} (ADDconst [off2] ptr) mem) && is20Bit(int64(off1)+int64(off2)) => (MOVBload [off1+off2] {sym} ptr mem) 918(MOVWZload [off1] {sym} (ADDconst [off2] ptr) mem) && is20Bit(int64(off1)+int64(off2)) => (MOVWZload [off1+off2] {sym} ptr mem) 919(MOVHZload [off1] {sym} (ADDconst [off2] ptr) mem) && is20Bit(int64(off1)+int64(off2)) => (MOVHZload [off1+off2] {sym} ptr mem) 920(MOVBZload [off1] {sym} (ADDconst [off2] ptr) mem) && is20Bit(int64(off1)+int64(off2)) => (MOVBZload [off1+off2] {sym} ptr mem) 921(FMOVSload [off1] {sym} (ADDconst [off2] ptr) mem) && is20Bit(int64(off1)+int64(off2)) => (FMOVSload [off1+off2] {sym} ptr mem) 922(FMOVDload [off1] {sym} (ADDconst [off2] ptr) mem) && is20Bit(int64(off1)+int64(off2)) => (FMOVDload [off1+off2] {sym} ptr mem) 923 924(MOVDstore [off1] {sym} (ADDconst [off2] ptr) val mem) && is20Bit(int64(off1)+int64(off2)) => (MOVDstore [off1+off2] {sym} ptr val mem) 925(MOVWstore [off1] {sym} (ADDconst [off2] ptr) val mem) && is20Bit(int64(off1)+int64(off2)) => (MOVWstore [off1+off2] {sym} ptr val mem) 926(MOVHstore [off1] {sym} (ADDconst [off2] ptr) val mem) && is20Bit(int64(off1)+int64(off2)) => (MOVHstore [off1+off2] {sym} ptr val mem) 927(MOVBstore [off1] {sym} (ADDconst [off2] ptr) val mem) && is20Bit(int64(off1)+int64(off2)) => (MOVBstore [off1+off2] {sym} ptr val mem) 928(FMOVSstore [off1] {sym} (ADDconst [off2] ptr) val mem) && is20Bit(int64(off1)+int64(off2)) => (FMOVSstore [off1+off2] {sym} ptr val mem) 929(FMOVDstore [off1] {sym} (ADDconst [off2] ptr) val mem) && is20Bit(int64(off1)+int64(off2)) => (FMOVDstore [off1+off2] {sym} ptr val mem) 930 931(ADDload [off1] {sym} x (ADDconst [off2] ptr) mem) && ptr.Op != OpSB && is20Bit(int64(off1)+int64(off2)) => (ADDload [off1+off2] {sym} x ptr mem) 932(ADDWload [off1] {sym} x (ADDconst [off2] ptr) mem) && ptr.Op != OpSB && is20Bit(int64(off1)+int64(off2)) => (ADDWload [off1+off2] {sym} x ptr mem) 933(MULLDload [off1] {sym} x (ADDconst [off2] ptr) mem) && ptr.Op != OpSB && is20Bit(int64(off1)+int64(off2)) => (MULLDload [off1+off2] {sym} x ptr mem) 934(MULLWload [off1] {sym} x (ADDconst [off2] ptr) mem) && ptr.Op != OpSB && is20Bit(int64(off1)+int64(off2)) => (MULLWload [off1+off2] {sym} x ptr mem) 935(SUBload [off1] {sym} x (ADDconst [off2] ptr) mem) && ptr.Op != OpSB && is20Bit(int64(off1)+int64(off2)) => (SUBload [off1+off2] {sym} x ptr mem) 936(SUBWload [off1] {sym} x (ADDconst [off2] ptr) mem) && ptr.Op != OpSB && is20Bit(int64(off1)+int64(off2)) => (SUBWload [off1+off2] {sym} x ptr mem) 937 938(ANDload [off1] {sym} x (ADDconst [off2] ptr) mem) && ptr.Op != OpSB && is20Bit(int64(off1)+int64(off2)) => (ANDload [off1+off2] {sym} x ptr mem) 939(ANDWload [off1] {sym} x (ADDconst [off2] ptr) mem) && ptr.Op != OpSB && is20Bit(int64(off1)+int64(off2)) => (ANDWload [off1+off2] {sym} x ptr mem) 940(ORload [off1] {sym} x (ADDconst [off2] ptr) mem) && ptr.Op != OpSB && is20Bit(int64(off1)+int64(off2)) => (ORload [off1+off2] {sym} x ptr mem) 941(ORWload [off1] {sym} x (ADDconst [off2] ptr) mem) && ptr.Op != OpSB && is20Bit(int64(off1)+int64(off2)) => (ORWload [off1+off2] {sym} x ptr mem) 942(XORload [off1] {sym} x (ADDconst [off2] ptr) mem) && ptr.Op != OpSB && is20Bit(int64(off1)+int64(off2)) => (XORload [off1+off2] {sym} x ptr mem) 943(XORWload [off1] {sym} x (ADDconst [off2] ptr) mem) && ptr.Op != OpSB && is20Bit(int64(off1)+int64(off2)) => (XORWload [off1+off2] {sym} x ptr mem) 944 945// Fold constants into stores. 946(MOVDstore [off] {sym} ptr (MOVDconst [c]) mem) && is16Bit(c) && isU12Bit(int64(off)) && ptr.Op != OpSB => 947 (MOVDstoreconst [makeValAndOff(int32(c),off)] {sym} ptr mem) 948(MOVWstore [off] {sym} ptr (MOVDconst [c]) mem) && is16Bit(c) && isU12Bit(int64(off)) && ptr.Op != OpSB => 949 (MOVWstoreconst [makeValAndOff(int32(c),off)] {sym} ptr mem) 950(MOVHstore [off] {sym} ptr (MOVDconst [c]) mem) && isU12Bit(int64(off)) && ptr.Op != OpSB => 951 (MOVHstoreconst [makeValAndOff(int32(int16(c)),off)] {sym} ptr mem) 952(MOVBstore [off] {sym} ptr (MOVDconst [c]) mem) && is20Bit(int64(off)) && ptr.Op != OpSB => 953 (MOVBstoreconst [makeValAndOff(int32(int8(c)),off)] {sym} ptr mem) 954 955// Fold address offsets into constant stores. 956(MOVDstoreconst [sc] {s} (ADDconst [off] ptr) mem) && isU12Bit(sc.Off64()+int64(off)) => 957 (MOVDstoreconst [sc.addOffset32(off)] {s} ptr mem) 958(MOVWstoreconst [sc] {s} (ADDconst [off] ptr) mem) && isU12Bit(sc.Off64()+int64(off)) => 959 (MOVWstoreconst [sc.addOffset32(off)] {s} ptr mem) 960(MOVHstoreconst [sc] {s} (ADDconst [off] ptr) mem) && isU12Bit(sc.Off64()+int64(off)) => 961 (MOVHstoreconst [sc.addOffset32(off)] {s} ptr mem) 962(MOVBstoreconst [sc] {s} (ADDconst [off] ptr) mem) && is20Bit(sc.Off64()+int64(off)) => 963 (MOVBstoreconst [sc.addOffset32(off)] {s} ptr mem) 964 965// Merge address calculations into loads and stores. 966// Offsets from SB must not be merged into unaligned memory accesses because 967// loads/stores using PC-relative addressing directly must be aligned to the 968// size of the target. 969(MOVDload [off1] {sym1} (MOVDaddr <t> [off2] {sym2} base) mem) && is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2) && (base.Op != OpSB || (t.IsPtr() && t.Elem().Alignment()%8 == 0 && (off1+off2)%8 == 0)) => 970 (MOVDload [off1+off2] {mergeSym(sym1,sym2)} base mem) 971(MOVWZload [off1] {sym1} (MOVDaddr <t> [off2] {sym2} base) mem) && is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2) && (base.Op != OpSB || (t.IsPtr() && t.Elem().Alignment()%4 == 0 && (off1+off2)%4 == 0)) => 972 (MOVWZload [off1+off2] {mergeSym(sym1,sym2)} base mem) 973(MOVHZload [off1] {sym1} (MOVDaddr <t> [off2] {sym2} base) mem) && is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2) && (base.Op != OpSB || (t.IsPtr() && t.Elem().Alignment()%2 == 0 && (off1+off2)%2 == 0)) => 974 (MOVHZload [off1+off2] {mergeSym(sym1,sym2)} base mem) 975(MOVBZload [off1] {sym1} (MOVDaddr [off2] {sym2} base) mem) && is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2) => 976 (MOVBZload [off1+off2] {mergeSym(sym1,sym2)} base mem) 977(FMOVSload [off1] {sym1} (MOVDaddr [off2] {sym2} base) mem) && is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2) => 978 (FMOVSload [off1+off2] {mergeSym(sym1,sym2)} base mem) 979(FMOVDload [off1] {sym1} (MOVDaddr [off2] {sym2} base) mem) && is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2) => 980 (FMOVDload [off1+off2] {mergeSym(sym1,sym2)} base mem) 981 982(MOVWload [off1] {sym1} (MOVDaddr <t> [off2] {sym2} base) mem) && is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2) && (base.Op != OpSB || (t.IsPtr() && t.Elem().Alignment()%4 == 0 && (off1+off2)%4 == 0)) => 983 (MOVWload [off1+off2] {mergeSym(sym1,sym2)} base mem) 984(MOVHload [off1] {sym1} (MOVDaddr <t> [off2] {sym2} base) mem) && is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2) && (base.Op != OpSB || (t.IsPtr() && t.Elem().Alignment()%2 == 0 && (off1+off2)%2 == 0)) => 985 (MOVHload [off1+off2] {mergeSym(sym1,sym2)} base mem) 986(MOVBload [off1] {sym1} (MOVDaddr [off2] {sym2} base) mem) && is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2) => 987 (MOVBload [off1+off2] {mergeSym(sym1,sym2)} base mem) 988 989(MOVDstore [off1] {sym1} (MOVDaddr <t> [off2] {sym2} base) val mem) && is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2) && (base.Op != OpSB || (t.IsPtr() && t.Elem().Alignment()%8 == 0 && (off1+off2)%8 == 0)) => 990 (MOVDstore [off1+off2] {mergeSym(sym1,sym2)} base val mem) 991(MOVWstore [off1] {sym1} (MOVDaddr <t> [off2] {sym2} base) val mem) && is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2) && (base.Op != OpSB || (t.IsPtr() && t.Elem().Alignment()%4 == 0 && (off1+off2)%4 == 0)) => 992 (MOVWstore [off1+off2] {mergeSym(sym1,sym2)} base val mem) 993(MOVHstore [off1] {sym1} (MOVDaddr <t> [off2] {sym2} base) val mem) && is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2) && (base.Op != OpSB || (t.IsPtr() && t.Elem().Alignment()%2 == 0 && (off1+off2)%2 == 0)) => 994 (MOVHstore [off1+off2] {mergeSym(sym1,sym2)} base val mem) 995(MOVBstore [off1] {sym1} (MOVDaddr [off2] {sym2} base) val mem) && is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2) => 996 (MOVBstore [off1+off2] {mergeSym(sym1,sym2)} base val mem) 997(FMOVSstore [off1] {sym1} (MOVDaddr [off2] {sym2} base) val mem) && is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2) => 998 (FMOVSstore [off1+off2] {mergeSym(sym1,sym2)} base val mem) 999(FMOVDstore [off1] {sym1} (MOVDaddr [off2] {sym2} base) val mem) && is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2) => 1000 (FMOVDstore [off1+off2] {mergeSym(sym1,sym2)} base val mem) 1001 1002(ADDload [o1] {s1} x (MOVDaddr [o2] {s2} ptr) mem) && ptr.Op != OpSB && is20Bit(int64(o1)+int64(o2)) && canMergeSym(s1, s2) => (ADDload [o1+o2] {mergeSym(s1, s2)} x ptr mem) 1003(ADDWload [o1] {s1} x (MOVDaddr [o2] {s2} ptr) mem) && ptr.Op != OpSB && is20Bit(int64(o1)+int64(o2)) && canMergeSym(s1, s2) => (ADDWload [o1+o2] {mergeSym(s1, s2)} x ptr mem) 1004(MULLDload [o1] {s1} x (MOVDaddr [o2] {s2} ptr) mem) && ptr.Op != OpSB && is20Bit(int64(o1)+int64(o2)) && canMergeSym(s1, s2) => (MULLDload [o1+o2] {mergeSym(s1, s2)} x ptr mem) 1005(MULLWload [o1] {s1} x (MOVDaddr [o2] {s2} ptr) mem) && ptr.Op != OpSB && is20Bit(int64(o1)+int64(o2)) && canMergeSym(s1, s2) => (MULLWload [o1+o2] {mergeSym(s1, s2)} x ptr mem) 1006(SUBload [o1] {s1} x (MOVDaddr [o2] {s2} ptr) mem) && ptr.Op != OpSB && is20Bit(int64(o1)+int64(o2)) && canMergeSym(s1, s2) => (SUBload [o1+o2] {mergeSym(s1, s2)} x ptr mem) 1007(SUBWload [o1] {s1} x (MOVDaddr [o2] {s2} ptr) mem) && ptr.Op != OpSB && is20Bit(int64(o1)+int64(o2)) && canMergeSym(s1, s2) => (SUBWload [o1+o2] {mergeSym(s1, s2)} x ptr mem) 1008 1009(ANDload [o1] {s1} x (MOVDaddr [o2] {s2} ptr) mem) && ptr.Op != OpSB && is20Bit(int64(o1)+int64(o2)) && canMergeSym(s1, s2) => (ANDload [o1+o2] {mergeSym(s1, s2)} x ptr mem) 1010(ANDWload [o1] {s1} x (MOVDaddr [o2] {s2} ptr) mem) && ptr.Op != OpSB && is20Bit(int64(o1)+int64(o2)) && canMergeSym(s1, s2) => (ANDWload [o1+o2] {mergeSym(s1, s2)} x ptr mem) 1011(ORload [o1] {s1} x (MOVDaddr [o2] {s2} ptr) mem) && ptr.Op != OpSB && is20Bit(int64(o1)+int64(o2)) && canMergeSym(s1, s2) => (ORload [o1+o2] {mergeSym(s1, s2)} x ptr mem) 1012(ORWload [o1] {s1} x (MOVDaddr [o2] {s2} ptr) mem) && ptr.Op != OpSB && is20Bit(int64(o1)+int64(o2)) && canMergeSym(s1, s2) => (ORWload [o1+o2] {mergeSym(s1, s2)} x ptr mem) 1013(XORload [o1] {s1} x (MOVDaddr [o2] {s2} ptr) mem) && ptr.Op != OpSB && is20Bit(int64(o1)+int64(o2)) && canMergeSym(s1, s2) => (XORload [o1+o2] {mergeSym(s1, s2)} x ptr mem) 1014(XORWload [o1] {s1} x (MOVDaddr [o2] {s2} ptr) mem) && ptr.Op != OpSB && is20Bit(int64(o1)+int64(o2)) && canMergeSym(s1, s2) => (XORWload [o1+o2] {mergeSym(s1, s2)} x ptr mem) 1015 1016// Cannot store constant to SB directly (no 'move relative long immediate' instructions). 1017(MOVDstoreconst [sc] {sym1} (MOVDaddr [off] {sym2} ptr) mem) && ptr.Op != OpSB && canMergeSym(sym1, sym2) && sc.canAdd32(off) => 1018 (MOVDstoreconst [sc.addOffset32(off)] {mergeSym(sym1, sym2)} ptr mem) 1019(MOVWstoreconst [sc] {sym1} (MOVDaddr [off] {sym2} ptr) mem) && ptr.Op != OpSB && canMergeSym(sym1, sym2) && sc.canAdd32(off) => 1020 (MOVWstoreconst [sc.addOffset32(off)] {mergeSym(sym1, sym2)} ptr mem) 1021(MOVHstoreconst [sc] {sym1} (MOVDaddr [off] {sym2} ptr) mem) && ptr.Op != OpSB && canMergeSym(sym1, sym2) && sc.canAdd32(off) => 1022 (MOVHstoreconst [sc.addOffset32(off)] {mergeSym(sym1, sym2)} ptr mem) 1023(MOVBstoreconst [sc] {sym1} (MOVDaddr [off] {sym2} ptr) mem) && ptr.Op != OpSB && canMergeSym(sym1, sym2) && sc.canAdd32(off) => 1024 (MOVBstoreconst [sc.addOffset32(off)] {mergeSym(sym1, sym2)} ptr mem) 1025 1026// MOVDaddr into MOVDaddridx 1027(MOVDaddridx [off1] {sym1} (MOVDaddr [off2] {sym2} x) y) && is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2) && x.Op != OpSB => 1028 (MOVDaddridx [off1+off2] {mergeSym(sym1,sym2)} x y) 1029(MOVDaddridx [off1] {sym1} x (MOVDaddr [off2] {sym2} y)) && is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2) && y.Op != OpSB => 1030 (MOVDaddridx [off1+off2] {mergeSym(sym1,sym2)} x y) 1031 1032// Absorb InvertFlags into branches. 1033(BRC {c} (InvertFlags cmp) yes no) => (BRC {c.ReverseComparison()} cmp yes no) 1034 1035// Constant comparisons. 1036(CMPconst (MOVDconst [x]) [y]) && x==int64(y) => (FlagEQ) 1037(CMPconst (MOVDconst [x]) [y]) && x<int64(y) => (FlagLT) 1038(CMPconst (MOVDconst [x]) [y]) && x>int64(y) => (FlagGT) 1039(CMPUconst (MOVDconst [x]) [y]) && uint64(x)==uint64(y) => (FlagEQ) 1040(CMPUconst (MOVDconst [x]) [y]) && uint64(x)<uint64(y) => (FlagLT) 1041(CMPUconst (MOVDconst [x]) [y]) && uint64(x)>uint64(y) => (FlagGT) 1042 1043(CMPWconst (MOVDconst [x]) [y]) && int32(x)==int32(y) => (FlagEQ) 1044(CMPWconst (MOVDconst [x]) [y]) && int32(x)<int32(y) => (FlagLT) 1045(CMPWconst (MOVDconst [x]) [y]) && int32(x)>int32(y) => (FlagGT) 1046(CMPWUconst (MOVDconst [x]) [y]) && uint32(x)==uint32(y) => (FlagEQ) 1047(CMPWUconst (MOVDconst [x]) [y]) && uint32(x)<uint32(y) => (FlagLT) 1048(CMPWUconst (MOVDconst [x]) [y]) && uint32(x)>uint32(y) => (FlagGT) 1049 1050(CMP(W|WU)const (MOVBZreg _) [c]) && 0xff < c => (FlagLT) 1051(CMP(W|WU)const (MOVHZreg _) [c]) && 0xffff < c => (FlagLT) 1052 1053(CMPconst (SRDconst _ [c]) [n]) && c > 0 && n < 0 => (FlagGT) 1054(CMPWconst (SRWconst _ [c]) [n]) && c > 0 && n < 0 => (FlagGT) 1055 1056(CMPUconst (SRDconst _ [c]) [n]) && c > 0 && c < 64 && (1<<uint(64-c)) <= uint64(n) => (FlagLT) 1057(CMPWUconst (SRWconst _ [c]) [n]) && c > 0 && c < 32 && (1<<uint(32-c)) <= uint32(n) => (FlagLT) 1058 1059(CMPWconst (ANDWconst _ [m]) [n]) && int32(m) >= 0 && int32(m) < int32(n) => (FlagLT) 1060(CMPWUconst (ANDWconst _ [m]) [n]) && uint32(m) < uint32(n) => (FlagLT) 1061 1062(CMPconst (RISBGZ x {r}) [c]) && c > 0 && r.OutMask() < uint64(c) => (FlagLT) 1063(CMPUconst (RISBGZ x {r}) [c]) && r.OutMask() < uint64(uint32(c)) => (FlagLT) 1064 1065// Constant compare-and-branch with immediate. 1066(CGIJ {c} (MOVDconst [x]) [y] yes no) && c&s390x.Equal != 0 && int64(x) == int64(y) => (First yes no) 1067(CGIJ {c} (MOVDconst [x]) [y] yes no) && c&s390x.Less != 0 && int64(x) < int64(y) => (First yes no) 1068(CGIJ {c} (MOVDconst [x]) [y] yes no) && c&s390x.Greater != 0 && int64(x) > int64(y) => (First yes no) 1069(CIJ {c} (MOVDconst [x]) [y] yes no) && c&s390x.Equal != 0 && int32(x) == int32(y) => (First yes no) 1070(CIJ {c} (MOVDconst [x]) [y] yes no) && c&s390x.Less != 0 && int32(x) < int32(y) => (First yes no) 1071(CIJ {c} (MOVDconst [x]) [y] yes no) && c&s390x.Greater != 0 && int32(x) > int32(y) => (First yes no) 1072(CLGIJ {c} (MOVDconst [x]) [y] yes no) && c&s390x.Equal != 0 && uint64(x) == uint64(y) => (First yes no) 1073(CLGIJ {c} (MOVDconst [x]) [y] yes no) && c&s390x.Less != 0 && uint64(x) < uint64(y) => (First yes no) 1074(CLGIJ {c} (MOVDconst [x]) [y] yes no) && c&s390x.Greater != 0 && uint64(x) > uint64(y) => (First yes no) 1075(CLIJ {c} (MOVDconst [x]) [y] yes no) && c&s390x.Equal != 0 && uint32(x) == uint32(y) => (First yes no) 1076(CLIJ {c} (MOVDconst [x]) [y] yes no) && c&s390x.Less != 0 && uint32(x) < uint32(y) => (First yes no) 1077(CLIJ {c} (MOVDconst [x]) [y] yes no) && c&s390x.Greater != 0 && uint32(x) > uint32(y) => (First yes no) 1078(CGIJ {c} (MOVDconst [x]) [y] yes no) && c&s390x.Equal == 0 && int64(x) == int64(y) => (First no yes) 1079(CGIJ {c} (MOVDconst [x]) [y] yes no) && c&s390x.Less == 0 && int64(x) < int64(y) => (First no yes) 1080(CGIJ {c} (MOVDconst [x]) [y] yes no) && c&s390x.Greater == 0 && int64(x) > int64(y) => (First no yes) 1081(CIJ {c} (MOVDconst [x]) [y] yes no) && c&s390x.Equal == 0 && int32(x) == int32(y) => (First no yes) 1082(CIJ {c} (MOVDconst [x]) [y] yes no) && c&s390x.Less == 0 && int32(x) < int32(y) => (First no yes) 1083(CIJ {c} (MOVDconst [x]) [y] yes no) && c&s390x.Greater == 0 && int32(x) > int32(y) => (First no yes) 1084(CLGIJ {c} (MOVDconst [x]) [y] yes no) && c&s390x.Equal == 0 && uint64(x) == uint64(y) => (First no yes) 1085(CLGIJ {c} (MOVDconst [x]) [y] yes no) && c&s390x.Less == 0 && uint64(x) < uint64(y) => (First no yes) 1086(CLGIJ {c} (MOVDconst [x]) [y] yes no) && c&s390x.Greater == 0 && uint64(x) > uint64(y) => (First no yes) 1087(CLIJ {c} (MOVDconst [x]) [y] yes no) && c&s390x.Equal == 0 && uint32(x) == uint32(y) => (First no yes) 1088(CLIJ {c} (MOVDconst [x]) [y] yes no) && c&s390x.Less == 0 && uint32(x) < uint32(y) => (First no yes) 1089(CLIJ {c} (MOVDconst [x]) [y] yes no) && c&s390x.Greater == 0 && uint32(x) > uint32(y) => (First no yes) 1090 1091// Constant compare-and-branch with immediate when unsigned comparison with zero. 1092(C(L|LG)IJ {s390x.GreaterOrEqual} _ [0] yes no) => (First yes no) 1093(C(L|LG)IJ {s390x.Less} _ [0] yes no) => (First no yes) 1094 1095// Constant compare-and-branch when operands match. 1096(C(GR|R|LGR|LR)J {c} x y yes no) && x == y && c&s390x.Equal != 0 => (First yes no) 1097(C(GR|R|LGR|LR)J {c} x y yes no) && x == y && c&s390x.Equal == 0 => (First no yes) 1098 1099// Convert 64-bit comparisons to 32-bit comparisons and signed comparisons 1100// to unsigned comparisons. 1101// Helps simplify constant comparison detection. 1102(CM(P|PU)const (MOV(W|WZ)reg x) [c]) => (CMP(W|WU)const x [c]) 1103(CM(P|P|PU|PU)const x:(MOV(H|HZ|H|HZ)reg _) [c]) => (CMP(W|W|WU|WU)const x [c]) 1104(CM(P|P|PU|PU)const x:(MOV(B|BZ|B|BZ)reg _) [c]) => (CMP(W|W|WU|WU)const x [c]) 1105(CMPconst (MOV(WZ|W)reg x:(ANDWconst [m] _)) [c]) && int32(m) >= 0 && c >= 0 => (CMPWUconst x [c]) 1106(CMPUconst (MOV(WZ|W)reg x:(ANDWconst [m] _)) [c]) && int32(m) >= 0 => (CMPWUconst x [c]) 1107(CMPconst x:(SRDconst _ [c]) [n]) && c > 0 && n >= 0 => (CMPUconst x [n]) 1108(CMPWconst x:(SRWconst _ [c]) [n]) && c > 0 && n >= 0 => (CMPWUconst x [n]) 1109 1110// Absorb sign and zero extensions into 32-bit comparisons. 1111(CMP(W|W|WU|WU) x (MOV(W|WZ|W|WZ)reg y)) => (CMP(W|W|WU|WU) x y) 1112(CMP(W|W|WU|WU) (MOV(W|WZ|W|WZ)reg x) y) => (CMP(W|W|WU|WU) x y) 1113(CMP(W|W|WU|WU)const (MOV(W|WZ|W|WZ)reg x) [c]) => (CMP(W|W|WU|WU)const x [c]) 1114 1115// Absorb flag constants into branches. 1116(BRC {c} (FlagEQ) yes no) && c&s390x.Equal != 0 => (First yes no) 1117(BRC {c} (FlagLT) yes no) && c&s390x.Less != 0 => (First yes no) 1118(BRC {c} (FlagGT) yes no) && c&s390x.Greater != 0 => (First yes no) 1119(BRC {c} (FlagOV) yes no) && c&s390x.Unordered != 0 => (First yes no) 1120 1121(BRC {c} (FlagEQ) yes no) && c&s390x.Equal == 0 => (First no yes) 1122(BRC {c} (FlagLT) yes no) && c&s390x.Less == 0 => (First no yes) 1123(BRC {c} (FlagGT) yes no) && c&s390x.Greater == 0 => (First no yes) 1124(BRC {c} (FlagOV) yes no) && c&s390x.Unordered == 0 => (First no yes) 1125 1126// Absorb flag constants into SETxx ops. 1127(LOCGR {c} _ x (FlagEQ)) && c&s390x.Equal != 0 => x 1128(LOCGR {c} _ x (FlagLT)) && c&s390x.Less != 0 => x 1129(LOCGR {c} _ x (FlagGT)) && c&s390x.Greater != 0 => x 1130(LOCGR {c} _ x (FlagOV)) && c&s390x.Unordered != 0 => x 1131 1132(LOCGR {c} x _ (FlagEQ)) && c&s390x.Equal == 0 => x 1133(LOCGR {c} x _ (FlagLT)) && c&s390x.Less == 0 => x 1134(LOCGR {c} x _ (FlagGT)) && c&s390x.Greater == 0 => x 1135(LOCGR {c} x _ (FlagOV)) && c&s390x.Unordered == 0 => x 1136 1137// Remove redundant *const ops 1138(ADDconst [0] x) => x 1139(ADDWconst [c] x) && int32(c)==0 => x 1140(SUBconst [0] x) => x 1141(SUBWconst [c] x) && int32(c) == 0 => x 1142(ANDconst [0] _) => (MOVDconst [0]) 1143(ANDWconst [c] _) && int32(c)==0 => (MOVDconst [0]) 1144(ANDconst [-1] x) => x 1145(ANDWconst [c] x) && int32(c)==-1 => x 1146(ORconst [0] x) => x 1147(ORWconst [c] x) && int32(c)==0 => x 1148(ORconst [-1] _) => (MOVDconst [-1]) 1149(ORWconst [c] _) && int32(c)==-1 => (MOVDconst [-1]) 1150(XORconst [0] x) => x 1151(XORWconst [c] x) && int32(c)==0 => x 1152 1153// Shifts by zero (may be inserted during multiplication strength reduction). 1154((SLD|SLW|SRD|SRW|SRAD|SRAW)const x [0]) => x 1155 1156// Convert constant subtracts to constant adds. 1157(SUBconst [c] x) && c != -(1<<31) => (ADDconst [-c] x) 1158(SUBWconst [c] x) => (ADDWconst [-int32(c)] x) 1159 1160// generic constant folding 1161// TODO: more of this 1162(ADDconst [c] (MOVDconst [d])) => (MOVDconst [int64(c)+d]) 1163(ADDWconst [c] (MOVDconst [d])) => (MOVDconst [int64(c)+d]) 1164(ADDconst [c] (ADDconst [d] x)) && is32Bit(int64(c)+int64(d)) => (ADDconst [c+d] x) 1165(ADDWconst [c] (ADDWconst [d] x)) => (ADDWconst [int32(c+d)] x) 1166(SUBconst (MOVDconst [d]) [c]) => (MOVDconst [d-int64(c)]) 1167(SUBconst (SUBconst x [d]) [c]) && is32Bit(-int64(c)-int64(d)) => (ADDconst [-c-d] x) 1168(SRADconst [c] (MOVDconst [d])) => (MOVDconst [d>>uint64(c)]) 1169(SRAWconst [c] (MOVDconst [d])) => (MOVDconst [int64(int32(d))>>uint64(c)]) 1170(NEG (MOVDconst [c])) => (MOVDconst [-c]) 1171(NEGW (MOVDconst [c])) => (MOVDconst [int64(int32(-c))]) 1172(MULLDconst [c] (MOVDconst [d])) => (MOVDconst [int64(c)*d]) 1173(MULLWconst [c] (MOVDconst [d])) => (MOVDconst [int64(c*int32(d))]) 1174(AND (MOVDconst [c]) (MOVDconst [d])) => (MOVDconst [c&d]) 1175(ANDconst [c] (MOVDconst [d])) => (MOVDconst [c&d]) 1176(ANDWconst [c] (MOVDconst [d])) => (MOVDconst [int64(c)&d]) 1177(OR (MOVDconst [c]) (MOVDconst [d])) => (MOVDconst [c|d]) 1178(ORconst [c] (MOVDconst [d])) => (MOVDconst [c|d]) 1179(ORWconst [c] (MOVDconst [d])) => (MOVDconst [int64(c)|d]) 1180(XOR (MOVDconst [c]) (MOVDconst [d])) => (MOVDconst [c^d]) 1181(XORconst [c] (MOVDconst [d])) => (MOVDconst [c^d]) 1182(XORWconst [c] (MOVDconst [d])) => (MOVDconst [int64(c)^d]) 1183(LoweredRound32F x:(FMOVSconst)) => x 1184(LoweredRound64F x:(FMOVDconst)) => x 1185 1186// generic simplifications 1187// TODO: more of this 1188(ADD x (NEG y)) => (SUB x y) 1189(ADDW x (NEGW y)) => (SUBW x y) 1190(SUB x x) => (MOVDconst [0]) 1191(SUBW x x) => (MOVDconst [0]) 1192(AND x x) => x 1193(ANDW x x) => x 1194(OR x x) => x 1195(ORW x x) => x 1196(XOR x x) => (MOVDconst [0]) 1197(XORW x x) => (MOVDconst [0]) 1198(NEG (ADDconst [c] (NEG x))) && c != -(1<<31) => (ADDconst [-c] x) 1199(MOVBZreg (ANDWconst [m] x)) => (MOVWZreg (ANDWconst <typ.UInt32> [int32( uint8(m))] x)) 1200(MOVHZreg (ANDWconst [m] x)) => (MOVWZreg (ANDWconst <typ.UInt32> [int32(uint16(m))] x)) 1201(MOVBreg (ANDWconst [m] x)) && int8(m) >= 0 => (MOVWZreg (ANDWconst <typ.UInt32> [int32( uint8(m))] x)) 1202(MOVHreg (ANDWconst [m] x)) && int16(m) >= 0 => (MOVWZreg (ANDWconst <typ.UInt32> [int32(uint16(m))] x)) 1203 1204// carry flag generation 1205// (only constant fold carry of zero) 1206(Select1 (ADDCconst (MOVDconst [c]) [d])) 1207 && uint64(c+int64(d)) >= uint64(c) && c+int64(d) == 0 1208 => (FlagEQ) 1209(Select1 (ADDCconst (MOVDconst [c]) [d])) 1210 && uint64(c+int64(d)) >= uint64(c) && c+int64(d) != 0 1211 => (FlagLT) 1212 1213// borrow flag generation 1214// (only constant fold borrow of zero) 1215(Select1 (SUBC (MOVDconst [c]) (MOVDconst [d]))) 1216 && uint64(d) <= uint64(c) && c-d == 0 1217 => (FlagGT) 1218(Select1 (SUBC (MOVDconst [c]) (MOVDconst [d]))) 1219 && uint64(d) <= uint64(c) && c-d != 0 1220 => (FlagOV) 1221 1222// add with carry 1223(ADDE x y (FlagEQ)) => (ADDC x y) 1224(ADDE x y (FlagLT)) => (ADDC x y) 1225(ADDC x (MOVDconst [c])) && is16Bit(c) => (ADDCconst x [int16(c)]) 1226(Select0 (ADDCconst (MOVDconst [c]) [d])) => (MOVDconst [c+int64(d)]) 1227 1228// subtract with borrow 1229(SUBE x y (FlagGT)) => (SUBC x y) 1230(SUBE x y (FlagOV)) => (SUBC x y) 1231(Select0 (SUBC (MOVDconst [c]) (MOVDconst [d]))) => (MOVDconst [c-d]) 1232 1233// collapse carry chain 1234(ADDE x y (Select1 (ADDCconst [-1] (Select0 (ADDE (MOVDconst [0]) (MOVDconst [0]) c))))) 1235 => (ADDE x y c) 1236 1237// collapse borrow chain 1238(SUBE x y (Select1 (SUBC (MOVDconst [0]) (NEG (Select0 (SUBE (MOVDconst [0]) (MOVDconst [0]) c)))))) 1239 => (SUBE x y c) 1240 1241// branch on carry 1242(C(G|LG)IJ {s390x.Equal} (Select0 (ADDE (MOVDconst [0]) (MOVDconst [0]) carry)) [0]) => (BRC {s390x.NoCarry} carry) 1243(C(G|LG)IJ {s390x.Equal} (Select0 (ADDE (MOVDconst [0]) (MOVDconst [0]) carry)) [1]) => (BRC {s390x.Carry} carry) 1244(C(G|LG)IJ {s390x.LessOrGreater} (Select0 (ADDE (MOVDconst [0]) (MOVDconst [0]) carry)) [0]) => (BRC {s390x.Carry} carry) 1245(C(G|LG)IJ {s390x.LessOrGreater} (Select0 (ADDE (MOVDconst [0]) (MOVDconst [0]) carry)) [1]) => (BRC {s390x.NoCarry} carry) 1246(C(G|LG)IJ {s390x.Greater} (Select0 (ADDE (MOVDconst [0]) (MOVDconst [0]) carry)) [0]) => (BRC {s390x.Carry} carry) 1247 1248// branch on borrow 1249(C(G|LG)IJ {s390x.Equal} (NEG (Select0 (SUBE (MOVDconst [0]) (MOVDconst [0]) borrow))) [0]) => (BRC {s390x.NoBorrow} borrow) 1250(C(G|LG)IJ {s390x.Equal} (NEG (Select0 (SUBE (MOVDconst [0]) (MOVDconst [0]) borrow))) [1]) => (BRC {s390x.Borrow} borrow) 1251(C(G|LG)IJ {s390x.LessOrGreater} (NEG (Select0 (SUBE (MOVDconst [0]) (MOVDconst [0]) borrow))) [0]) => (BRC {s390x.Borrow} borrow) 1252(C(G|LG)IJ {s390x.LessOrGreater} (NEG (Select0 (SUBE (MOVDconst [0]) (MOVDconst [0]) borrow))) [1]) => (BRC {s390x.NoBorrow} borrow) 1253(C(G|LG)IJ {s390x.Greater} (NEG (Select0 (SUBE (MOVDconst [0]) (MOVDconst [0]) borrow))) [0]) => (BRC {s390x.Borrow} borrow) 1254 1255// fused multiply-add 1256(Select0 (F(ADD|SUB) (FMUL y z) x)) && x.Block.Func.useFMA(v) => (FM(ADD|SUB) x y z) 1257(Select0 (F(ADDS|SUBS) (FMULS y z) x)) && x.Block.Func.useFMA(v) => (FM(ADDS|SUBS) x y z) 1258 1259// Convert floating point comparisons against zero into 'load and test' instructions. 1260(F(CMP|CMPS) x (FMOV(D|S)const [0.0])) => (LT(D|E)BR x) 1261(F(CMP|CMPS) (FMOV(D|S)const [0.0]) x) => (InvertFlags (LT(D|E)BR <v.Type> x)) 1262 1263// FSUB, FSUBS, FADD, FADDS now produce a condition code representing the 1264// comparison of the result with 0.0. If a compare with zero instruction 1265// (e.g. LTDBR) is following one of those instructions, we can use the 1266// generated flag and remove the comparison instruction. 1267// Note: when inserting Select1 ops we need to ensure they are in the 1268// same block as their argument. We could also use @x.Block for this 1269// but moving the flag generating value to a different block seems to 1270// increase the likelihood that the flags value will have to be regenerated 1271// by flagalloc which is not what we want. 1272(LTDBR (Select0 x:(F(ADD|SUB) _ _))) && b == x.Block => (Select1 x) 1273(LTEBR (Select0 x:(F(ADDS|SUBS) _ _))) && b == x.Block => (Select1 x) 1274 1275// Fold memory operations into operations. 1276// Exclude global data (SB) because these instructions cannot handle relative addresses. 1277// TODO(mundaym): indexed versions of these? 1278((ADD|SUB|MULLD|AND|OR|XOR) <t> x g:(MOVDload [off] {sym} ptr mem)) 1279 && ptr.Op != OpSB 1280 && is20Bit(int64(off)) 1281 && canMergeLoadClobber(v, g, x) 1282 && clobber(g) 1283 => ((ADD|SUB|MULLD|AND|OR|XOR)load <t> [off] {sym} x ptr mem) 1284((ADD|SUB|MULL|AND|OR|XOR)W <t> x g:(MOVWload [off] {sym} ptr mem)) 1285 && ptr.Op != OpSB 1286 && is20Bit(int64(off)) 1287 && canMergeLoadClobber(v, g, x) 1288 && clobber(g) 1289 => ((ADD|SUB|MULL|AND|OR|XOR)Wload <t> [off] {sym} x ptr mem) 1290((ADD|SUB|MULL|AND|OR|XOR)W <t> x g:(MOVWZload [off] {sym} ptr mem)) 1291 && ptr.Op != OpSB 1292 && is20Bit(int64(off)) 1293 && canMergeLoadClobber(v, g, x) 1294 && clobber(g) 1295 => ((ADD|SUB|MULL|AND|OR|XOR)Wload <t> [off] {sym} x ptr mem) 1296 1297// Combine stores into store multiples. 1298// 32-bit 1299(MOVWstore [i] {s} p w1 x:(MOVWstore [i-4] {s} p w0 mem)) 1300 && p.Op != OpSB 1301 && x.Uses == 1 1302 && is20Bit(int64(i)-4) 1303 && setPos(v, x.Pos) 1304 && clobber(x) 1305 => (STM2 [i-4] {s} p w0 w1 mem) 1306(MOVWstore [i] {s} p w2 x:(STM2 [i-8] {s} p w0 w1 mem)) 1307 && x.Uses == 1 1308 && is20Bit(int64(i)-8) 1309 && setPos(v, x.Pos) 1310 && clobber(x) 1311 => (STM3 [i-8] {s} p w0 w1 w2 mem) 1312(MOVWstore [i] {s} p w3 x:(STM3 [i-12] {s} p w0 w1 w2 mem)) 1313 && x.Uses == 1 1314 && is20Bit(int64(i)-12) 1315 && setPos(v, x.Pos) 1316 && clobber(x) 1317 => (STM4 [i-12] {s} p w0 w1 w2 w3 mem) 1318(STM2 [i] {s} p w2 w3 x:(STM2 [i-8] {s} p w0 w1 mem)) 1319 && x.Uses == 1 1320 && is20Bit(int64(i)-8) 1321 && setPos(v, x.Pos) 1322 && clobber(x) 1323 => (STM4 [i-8] {s} p w0 w1 w2 w3 mem) 1324// 64-bit 1325(MOVDstore [i] {s} p w1 x:(MOVDstore [i-8] {s} p w0 mem)) 1326 && p.Op != OpSB 1327 && x.Uses == 1 1328 && is20Bit(int64(i)-8) 1329 && setPos(v, x.Pos) 1330 && clobber(x) 1331 => (STMG2 [i-8] {s} p w0 w1 mem) 1332(MOVDstore [i] {s} p w2 x:(STMG2 [i-16] {s} p w0 w1 mem)) 1333 && x.Uses == 1 1334 && is20Bit(int64(i)-16) 1335 && setPos(v, x.Pos) 1336 && clobber(x) 1337 => (STMG3 [i-16] {s} p w0 w1 w2 mem) 1338(MOVDstore [i] {s} p w3 x:(STMG3 [i-24] {s} p w0 w1 w2 mem)) 1339 && x.Uses == 1 1340 && is20Bit(int64(i)-24) 1341 && setPos(v, x.Pos) 1342 && clobber(x) 1343 => (STMG4 [i-24] {s} p w0 w1 w2 w3 mem) 1344(STMG2 [i] {s} p w2 w3 x:(STMG2 [i-16] {s} p w0 w1 mem)) 1345 && x.Uses == 1 1346 && is20Bit(int64(i)-16) 1347 && setPos(v, x.Pos) 1348 && clobber(x) 1349 => (STMG4 [i-16] {s} p w0 w1 w2 w3 mem) 1350 1351// Convert 32-bit store multiples into 64-bit stores. 1352(STM2 [i] {s} p (SRDconst [32] x) x mem) => (MOVDstore [i] {s} p x mem) 1353 1354// Fold bit reversal into loads. 1355(MOVWBR x:(MOVWZload [off] {sym} ptr mem)) && x.Uses == 1 => @x.Block (MOVWZreg (MOVWBRload [off] {sym} ptr mem)) // need zero extension? 1356(MOVWBR x:(MOVWZloadidx [off] {sym} ptr idx mem)) && x.Uses == 1 => @x.Block (MOVWZreg (MOVWBRloadidx [off] {sym} ptr idx mem)) // need zero extension? 1357(MOVDBR x:(MOVDload [off] {sym} ptr mem)) && x.Uses == 1 => @x.Block (MOVDBRload [off] {sym} ptr mem) 1358(MOVDBR x:(MOVDloadidx [off] {sym} ptr idx mem)) && x.Uses == 1 => @x.Block (MOVDBRloadidx [off] {sym} ptr idx mem) 1359 1360// Fold bit reversal into stores. 1361(MOV(D|W)store [off] {sym} ptr r:(MOV(D|W)BR x) mem) && r.Uses == 1 => (MOV(D|W)BRstore [off] {sym} ptr x mem) 1362(MOV(D|W)storeidx [off] {sym} ptr idx r:(MOV(D|W)BR x) mem) && r.Uses == 1 => (MOV(D|W)BRstoreidx [off] {sym} ptr idx x mem) 1363 1364// Special bswap16 rules 1365(Bswap16 x:(MOVHZload [off] {sym} ptr mem)) => @x.Block (MOVHZreg (MOVHBRload [off] {sym} ptr mem)) 1366(Bswap16 x:(MOVHZloadidx [off] {sym} ptr idx mem)) => @x.Block (MOVHZreg (MOVHBRloadidx [off] {sym} ptr idx mem)) 1367(MOVHstore [off] {sym} ptr (Bswap16 val) mem) => (MOVHBRstore [off] {sym} ptr val mem) 1368(MOVHstoreidx [off] {sym} ptr idx (Bswap16 val) mem) => (MOVHBRstoreidx [off] {sym} ptr idx val mem) 1369