cmd/compile: implement non-constant rotates

Makes math/bits.Rotate{Left,Right} fast on amd64.

name              old time/op  new time/op  delta
RotateLeft-12     7.42ns ± 6%  5.45ns ± 6%  -26.54%   (p=0.000 n=9+10)
RotateLeft8-12    4.77ns ± 5%  3.42ns ± 7%  -28.25%   (p=0.000 n=8+10)
RotateLeft16-12   4.82ns ± 8%  3.40ns ± 7%  -29.36%  (p=0.000 n=10+10)
RotateLeft32-12   4.87ns ± 7%  3.48ns ± 7%  -28.51%    (p=0.000 n=8+9)
RotateLeft64-12   5.23ns ±10%  3.35ns ± 6%  -35.97%   (p=0.000 n=9+10)
RotateRight-12    7.59ns ± 8%  5.71ns ± 1%  -24.72%   (p=0.000 n=10+8)
RotateRight8-12   4.98ns ± 7%  3.36ns ± 9%  -32.55%  (p=0.000 n=10+10)
RotateRight16-12  5.12ns ± 2%  3.45ns ± 5%  -32.62%  (p=0.000 n=10+10)
RotateRight32-12  4.80ns ± 6%  3.42ns ±16%  -28.68%  (p=0.000 n=10+10)
RotateRight64-12  4.78ns ± 6%  3.42ns ± 6%  -28.50%  (p=0.000 n=10+10)

Update #18940

Change-Id: Ie79fb5581c489ed4d3b859314c5e669a134c119b
Reviewed-on: https://go-review.googlesource.com/39711
Run-TryBot: Keith Randall <[email protected]>
TryBot-Result: Gobot Gobot <[email protected]>
Reviewed-by: Josh Bleecher Snyder <[email protected]>

Author: randall77

src/cmd/compile/internal/amd64/ssa.go

@@ -189,6 +189,8 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) {
 		ssa.OpAMD64SHLQ, ssa.OpAMD64SHLL,
 		ssa.OpAMD64SHRQ, ssa.OpAMD64SHRL, ssa.OpAMD64SHRW, ssa.OpAMD64SHRB,
 		ssa.OpAMD64SARQ, ssa.OpAMD64SARL, ssa.OpAMD64SARW, ssa.OpAMD64SARB,
+		ssa.OpAMD64ROLQ, ssa.OpAMD64ROLL, ssa.OpAMD64ROLW, ssa.OpAMD64ROLB,
+		ssa.OpAMD64RORQ, ssa.OpAMD64RORL, ssa.OpAMD64RORW, ssa.OpAMD64RORB,
 		ssa.OpAMD64ADDSS, ssa.OpAMD64ADDSD, ssa.OpAMD64SUBSS, ssa.OpAMD64SUBSD,
 		ssa.OpAMD64MULSS, ssa.OpAMD64MULSD, ssa.OpAMD64DIVSS, ssa.OpAMD64DIVSD,
 		ssa.OpAMD64PXOR:

src/cmd/compile/internal/gc/asm_test.go

@@ -800,6 +800,63 @@ var linuxAMD64Tests = []*asmTest{
 		}`,
 		[]string{"\tCMPQ\t[A-Z]"},
 	},
+	// Non-constant rotate
+	{
+		`func rot64l(x uint64, y int) uint64 {
+			z := uint(y & 63)
+			return x << z | x >> (64-z)
+		}`,
+		[]string{"\tROLQ\t"},
+	},
+	{
+		`func rot64r(x uint64, y int) uint64 {
+			z := uint(y & 63)
+			return x >> z | x << (64-z)
+		}`,
+		[]string{"\tRORQ\t"},
+	},
+	{
+		`func rot32l(x uint32, y int) uint32 {
+			z := uint(y & 31)
+			return x << z | x >> (32-z)
+		}`,
+		[]string{"\tROLL\t"},
+	},
+	{
+		`func rot32r(x uint32, y int) uint32 {
+			z := uint(y & 31)
+			return x >> z | x << (32-z)
+		}`,
+		[]string{"\tRORL\t"},
+	},
+	{
+		`func rot16l(x uint16, y int) uint16 {
+			z := uint(y & 15)
+			return x << z | x >> (16-z)
+		}`,
+		[]string{"\tROLW\t"},
+	},
+	{
+		`func rot16r(x uint16, y int) uint16 {
+			z := uint(y & 15)
+			return x >> z | x << (16-z)
+		}`,
+		[]string{"\tRORW\t"},
+	},
+	{
+		`func rot8l(x uint8, y int) uint8 {
+			z := uint(y & 7)
+			return x << z | x >> (8-z)
+		}`,
+		[]string{"\tROLB\t"},
+	},
+	{
+		`func rot8r(x uint8, y int) uint8 {
+			z := uint(y & 7)
+			return x >> z | x << (8-z)
+		}`,
+		[]string{"\tRORB\t"},
+	},
 }
 
 var linux386Tests = []*asmTest{

src/cmd/compile/internal/ssa/gen/AMD64.rules

@@ -621,19 +621,60 @@
 (SARB x (MOVQconst [c])) -> (SARBconst [min(c&31,7)] x)
 (SARB x (MOVLconst [c])) -> (SARBconst [min(c&31,7)] x)
 
-(SARL x (ANDLconst [31] y)) -> (SARL x y)
-(SARQ x (ANDQconst [63] y)) -> (SARQ x y)
-
-(SHLL x (ANDLconst [31] y)) -> (SHLL x y)
-(SHLQ x (ANDQconst [63] y)) -> (SHLQ x y)
-(SHLQ x (ANDLconst [63] y)) -> (SHLQ x y)
-
-(SHRL x (ANDLconst [31] y)) -> (SHRL x y)
-(SHRQ x (ANDQconst [63] y)) -> (SHRQ x y)
-(SHRQ x (ANDLconst [63] y)) -> (SHRQ x y)
-
-// Rotate instructions
-
+// Operations which don't affect the low 6/5 bits of the shift amount are NOPs.
+(SHLQ x (ADDQconst [c] y)) && c & 63 == 0 -> (SHLQ x y)
+(SHRQ x (ADDQconst [c] y)) && c & 63 == 0 -> (SHRQ x y)
+(SARQ x (ADDQconst [c] y)) && c & 63 == 0 -> (SARQ x y)
+(SHLQ x (NEGQ <t> (ADDQconst [c] y))) && c & 63 == 0 -> (SHLQ x (NEGQ <t> y))
+(SHRQ x (NEGQ <t> (ADDQconst [c] y))) && c & 63 == 0 -> (SHRQ x (NEGQ <t> y))
+(SARQ x (NEGQ <t> (ADDQconst [c] y))) && c & 63 == 0 -> (SARQ x (NEGQ <t> y))
+(SHLQ x (ANDQconst [c] y)) && c & 63 == 63 -> (SHLQ x y)
+(SHRQ x (ANDQconst [c] y)) && c & 63 == 63 -> (SHRQ x y)
+(SARQ x (ANDQconst [c] y)) && c & 63 == 63 -> (SARQ x y)
+(SHLQ x (NEGQ <t> (ANDQconst [c] y))) && c & 63 == 63 -> (SHLQ x (NEGQ <t> y))
+(SHRQ x (NEGQ <t> (ANDQconst [c] y))) && c & 63 == 63 -> (SHRQ x (NEGQ <t> y))
+(SARQ x (NEGQ <t> (ANDQconst [c] y))) && c & 63 == 63 -> (SARQ x (NEGQ <t> y))
+
+(SHLL x (ADDQconst [c] y)) && c & 31 == 0 -> (SHLL x y)
+(SHRL x (ADDQconst [c] y)) && c & 31 == 0 -> (SHRL x y)
+(SARL x (ADDQconst [c] y)) && c & 31 == 0 -> (SARL x y)
+(SHLL x (NEGQ <t> (ADDQconst [c] y))) && c & 31 == 0 -> (SHLL x (NEGQ <t> y))
+(SHRL x (NEGQ <t> (ADDQconst [c] y))) && c & 31 == 0 -> (SHRL x (NEGQ <t> y))
+(SARL x (NEGQ <t> (ADDQconst [c] y))) && c & 31 == 0 -> (SARL x (NEGQ <t> y))
+(SHLL x (ANDQconst [c] y)) && c & 31 == 31 -> (SHLL x y)
+(SHRL x (ANDQconst [c] y)) && c & 31 == 31 -> (SHRL x y)
+(SARL x (ANDQconst [c] y)) && c & 31 == 31 -> (SARL x y)
+(SHLL x (NEGQ <t> (ANDQconst [c] y))) && c & 31 == 31 -> (SHLL x (NEGQ <t> y))
+(SHRL x (NEGQ <t> (ANDQconst [c] y))) && c & 31 == 31 -> (SHRL x (NEGQ <t> y))
+(SARL x (NEGQ <t> (ANDQconst [c] y))) && c & 31 == 31 -> (SARL x (NEGQ <t> y))
+
+(SHLQ x (ADDLconst [c] y)) && c & 63 == 0 -> (SHLQ x y)
+(SHRQ x (ADDLconst [c] y)) && c & 63 == 0 -> (SHRQ x y)
+(SARQ x (ADDLconst [c] y)) && c & 63 == 0 -> (SARQ x y)
+(SHLQ x (NEGL <t> (ADDLconst [c] y))) && c & 63 == 0 -> (SHLQ x (NEGL <t> y))
+(SHRQ x (NEGL <t> (ADDLconst [c] y))) && c & 63 == 0 -> (SHRQ x (NEGL <t> y))
+(SARQ x (NEGL <t> (ADDLconst [c] y))) && c & 63 == 0 -> (SARQ x (NEGL <t> y))
+(SHLQ x (ANDLconst [c] y)) && c & 63 == 63 -> (SHLQ x y)
+(SHRQ x (ANDLconst [c] y)) && c & 63 == 63 -> (SHRQ x y)
+(SARQ x (ANDLconst [c] y)) && c & 63 == 63 -> (SARQ x y)
+(SHLQ x (NEGL <t> (ANDLconst [c] y))) && c & 63 == 63 -> (SHLQ x (NEGL <t> y))
+(SHRQ x (NEGL <t> (ANDLconst [c] y))) && c & 63 == 63 -> (SHRQ x (NEGL <t> y))
+(SARQ x (NEGL <t> (ANDLconst [c] y))) && c & 63 == 63 -> (SARQ x (NEGL <t> y))
+
+(SHLL x (ADDLconst [c] y)) && c & 31 == 0 -> (SHLL x y)
+(SHRL x (ADDLconst [c] y)) && c & 31 == 0 -> (SHRL x y)
+(SARL x (ADDLconst [c] y)) && c & 31 == 0 -> (SARL x y)
+(SHLL x (NEGL <t> (ADDLconst [c] y))) && c & 31 == 0 -> (SHLL x (NEGL <t> y))
+(SHRL x (NEGL <t> (ADDLconst [c] y))) && c & 31 == 0 -> (SHRL x (NEGL <t> y))
+(SARL x (NEGL <t> (ADDLconst [c] y))) && c & 31 == 0 -> (SARL x (NEGL <t> y))
+(SHLL x (ANDLconst [c] y)) && c & 31 == 31 -> (SHLL x y)
+(SHRL x (ANDLconst [c] y)) && c & 31 == 31 -> (SHRL x y)
+(SARL x (ANDLconst [c] y)) && c & 31 == 31 -> (SARL x y)
+(SHLL x (NEGL <t> (ANDLconst [c] y))) && c & 31 == 31 -> (SHLL x (NEGL <t> y))
+(SHRL x (NEGL <t> (ANDLconst [c] y))) && c & 31 == 31 -> (SHRL x (NEGL <t> y))
+(SARL x (NEGL <t> (ANDLconst [c] y))) && c & 31 == 31 -> (SARL x (NEGL <t> y))
+
+// Constant rotate instructions
 (ADDQ (SHLQconst x [c]) (SHRQconst x [d])) && d==64-c -> (ROLQconst x [c])
 ( ORQ (SHLQconst x [c]) (SHRQconst x [d])) && d==64-c -> (ROLQconst x [c])
 (XORQ (SHLQconst x [c]) (SHRQconst x [d])) && d==64-c -> (ROLQconst x [c])
@@ -655,10 +696,109 @@
 (ROLWconst [c] (ROLWconst [d] x)) -> (ROLWconst [(c+d)&15] x)
 (ROLBconst [c] (ROLBconst [d] x)) -> (ROLBconst [(c+d)& 7] x)
 
-// TODO: non-constant rotates if shift amount is known to be bounded (shift & 63 or something).
+// Non-constant rotates.
+// We want to issue a rotate when the Go source contains code like
+//     y &= 63
+//     x << y | x >> (64-y)
+// The shift rules above convert << to SHLx and >> to SHRx.
+// SHRx converts its shift argument from 64-y to -y.
+// A tricky situation occurs when y==0. Then the original code would be:
+//     x << 0 | x >> 64
+// But x >> 64 is 0, not x. So there's an additional mask that is ANDed in
+// to force the second term to 0. We don't need that mask, but we must match
+// it in order to strip it out.
+(ORQ (SHLQ x y) (ANDQ (SHRQ x (NEGQ y)) (SBBQcarrymask (CMPQconst (NEGQ (ADDQconst (ANDQconst y [63]) [-64])) [64])))) -> (ROLQ x y)
+(ORQ (SHLQ x y) (ANDQ (SHRQ x (NEGL y)) (SBBQcarrymask (CMPLconst (NEGL (ADDLconst (ANDLconst y [63]) [-64])) [64])))) -> (ROLQ x y)
+(ORQ (SHRQ x y) (ANDQ (SHLQ x (NEGQ y)) (SBBQcarrymask (CMPQconst (NEGQ (ADDQconst (ANDQconst y [63]) [-64])) [64])))) -> (RORQ x y)
+(ORQ (SHRQ x y) (ANDQ (SHLQ x (NEGL y)) (SBBQcarrymask (CMPLconst (NEGL (ADDLconst (ANDLconst y [63]) [-64])) [64])))) -> (RORQ x y)
+
+(ORL (SHLL x y) (ANDL (SHRL x (NEGQ y)) (SBBLcarrymask (CMPQconst (NEGQ (ADDQconst (ANDQconst y [31]) [-32])) [32])))) -> (ROLL x y)
+(ORL (SHLL x y) (ANDL (SHRL x (NEGL y)) (SBBLcarrymask (CMPLconst (NEGL (ADDLconst (ANDLconst y [31]) [-32])) [32])))) -> (ROLL x y)
+(ORL (SHRL x y) (ANDL (SHLL x (NEGQ y)) (SBBLcarrymask (CMPQconst (NEGQ (ADDQconst (ANDQconst y [31]) [-32])) [32])))) -> (RORL x y)
+(ORL (SHRL x y) (ANDL (SHLL x (NEGL y)) (SBBLcarrymask (CMPLconst (NEGL (ADDLconst (ANDLconst y [31]) [-32])) [32])))) -> (RORL x y)
+
+// Help with rotate detection
+(CMPQconst (NEGQ (ADDQconst [-16] (ANDQconst [15] _))) [32]) -> (FlagLT_ULT)
+(CMPQconst (NEGQ (ADDQconst [ -8] (ANDQconst  [7] _))) [32]) -> (FlagLT_ULT)
+
+(ORL (SHLL x (ANDQconst y [15]))
+     (ANDL (SHRW x (NEGQ (ADDQconst (ANDQconst y [15]) [-16])))
+           (SBBLcarrymask (CMPQconst (NEGQ (ADDQconst (ANDQconst y [15]) [-16])) [16]))))
+  && v.Type.Size() == 2
+  -> (ROLW x y)
+(ORL (SHLL x (ANDLconst y [15]))
+     (ANDL (SHRW x (NEGL (ADDLconst (ANDLconst y [15]) [-16])))
+           (SBBLcarrymask (CMPLconst (NEGL (ADDLconst (ANDLconst y [15]) [-16])) [16]))))
+  && v.Type.Size() == 2
+  -> (ROLW x y)
+(ORL (SHRW x (ANDQconst y [15]))
+     (SHLL x (NEGQ (ADDQconst (ANDQconst y [15]) [-16]))))
+  && v.Type.Size() == 2
+  -> (RORW x y)
+(ORL (SHRW x (ANDLconst y [15]))
+     (SHLL x (NEGL (ADDLconst (ANDLconst y [15]) [-16]))))
+  && v.Type.Size() == 2
+  -> (RORW x y)
+
+(ORL (SHLL x (ANDQconst y [ 7]))
+     (ANDL (SHRB x (NEGQ (ADDQconst (ANDQconst y [ 7]) [ -8])))
+           (SBBLcarrymask (CMPQconst (NEGQ (ADDQconst (ANDQconst y [ 7]) [ -8])) [ 8]))))
+  && v.Type.Size() == 1
+  -> (ROLB x y)
+(ORL (SHLL x (ANDLconst y [ 7]))
+     (ANDL (SHRB x (NEGL (ADDLconst (ANDLconst y [ 7]) [ -8])))
+           (SBBLcarrymask (CMPLconst (NEGL (ADDLconst (ANDLconst y [ 7]) [ -8])) [ 8]))))
+  && v.Type.Size() == 1
+  -> (ROLB x y)
+(ORL (SHRB x (ANDQconst y [ 7]))
+     (SHLL x (NEGQ (ADDQconst (ANDQconst y [ 7]) [ -8]))))
+  && v.Type.Size() == 1
+  -> (RORB x y)
+(ORL (SHRB x (ANDLconst y [ 7]))
+     (SHLL x (NEGL (ADDLconst (ANDLconst y [ 7]) [ -8]))))
+  && v.Type.Size() == 1
+  -> (RORB x y)
+
+// rotate left negative = rotate right
+(ROLQ x (NEGQ y)) -> (RORQ x y)
+(ROLQ x (NEGL y)) -> (RORQ x y)
+(ROLL x (NEGQ y)) -> (RORL x y)
+(ROLL x (NEGL y)) -> (RORL x y)
+(ROLW x (NEGQ y)) -> (RORW x y)
+(ROLW x (NEGL y)) -> (RORW x y)
+(ROLB x (NEGQ y)) -> (RORB x y)
+(ROLB x (NEGL y)) -> (RORB x y)
+
+// rotate right negative = rotate left
+(RORQ x (NEGQ y)) -> (ROLQ x y)
+(RORQ x (NEGL y)) -> (ROLQ x y)
+(RORL x (NEGQ y)) -> (ROLL x y)
+(RORL x (NEGL y)) -> (ROLL x y)
+(RORW x (NEGQ y)) -> (ROLW x y)
+(RORW x (NEGL y)) -> (ROLW x y)
+(RORB x (NEGQ y)) -> (ROLB x y)
+(RORB x (NEGL y)) -> (ROLB x y)
+
+// rotate by constants
+(ROLQ x (MOVQconst [c])) -> (ROLQconst [c&63] x)
+(ROLQ x (MOVLconst [c])) -> (ROLQconst [c&63] x)
+(ROLL x (MOVQconst [c])) -> (ROLLconst [c&31] x)
+(ROLL x (MOVLconst [c])) -> (ROLLconst [c&31] x)
+(ROLW x (MOVQconst [c])) -> (ROLWconst [c&15] x)
+(ROLW x (MOVLconst [c])) -> (ROLWconst [c&15] x)
+(ROLB x (MOVQconst [c])) -> (ROLBconst [c&7 ] x)
+(ROLB x (MOVLconst [c])) -> (ROLBconst [c&7 ] x)
+
+(RORQ x (MOVQconst [c])) -> (ROLQconst [(-c)&63] x)
+(RORQ x (MOVLconst [c])) -> (ROLQconst [(-c)&63] x)
+(RORL x (MOVQconst [c])) -> (ROLLconst [(-c)&31] x)
+(RORL x (MOVLconst [c])) -> (ROLLconst [(-c)&31] x)
+(RORW x (MOVQconst [c])) -> (ROLWconst [(-c)&15] x)
+(RORW x (MOVLconst [c])) -> (ROLWconst [(-c)&15] x)
+(RORB x (MOVQconst [c])) -> (ROLBconst [(-c)&7 ] x)
+(RORB x (MOVLconst [c])) -> (ROLBconst [(-c)&7 ] x)
 
 // Constant shift simplifications
-
 (SHLQconst x [0]) -> x
 (SHRQconst x [0]) -> x
 (SARQconst x [0]) -> x
@@ -1171,6 +1311,7 @@
 (CMPLconst (ANDLconst _ [m]) [n]) && 0 <= int32(m) && int32(m) < int32(n) -> (FlagLT_ULT)
 (CMPWconst (ANDLconst _ [m]) [n]) && 0 <= int16(m) && int16(m) < int16(n) -> (FlagLT_ULT)
 (CMPBconst (ANDLconst _ [m]) [n]) && 0 <= int8(m) && int8(m) < int8(n) -> (FlagLT_ULT)
+
 // TODO: DIVxU also.
 
 // Absorb flag constants into SBB ops.

src/cmd/compile/internal/ssa/gen/AMD64Ops.go

@@ -284,6 +284,14 @@ func init() {
 		{name: "SARWconst", argLength: 1, reg: gp11, asm: "SARW", aux: "Int8", resultInArg0: true, clobberFlags: true}, // signed arg0 >> auxint, shift amount 0-15
 		{name: "SARBconst", argLength: 1, reg: gp11, asm: "SARB", aux: "Int8", resultInArg0: true, clobberFlags: true}, // signed arg0 >> auxint, shift amount 0-7
 
+		{name: "ROLQ", argLength: 2, reg: gp21shift, asm: "ROLQ", resultInArg0: true, clobberFlags: true},              // arg0 rotate left arg1 bits.
+		{name: "ROLL", argLength: 2, reg: gp21shift, asm: "ROLL", resultInArg0: true, clobberFlags: true},              // arg0 rotate left arg1 bits.
+		{name: "ROLW", argLength: 2, reg: gp21shift, asm: "ROLW", resultInArg0: true, clobberFlags: true},              // arg0 rotate left arg1 bits.
+		{name: "ROLB", argLength: 2, reg: gp21shift, asm: "ROLB", resultInArg0: true, clobberFlags: true},              // arg0 rotate left arg1 bits.
+		{name: "RORQ", argLength: 2, reg: gp21shift, asm: "RORQ", resultInArg0: true, clobberFlags: true},              // arg0 rotate right arg1 bits.
+		{name: "RORL", argLength: 2, reg: gp21shift, asm: "RORL", resultInArg0: true, clobberFlags: true},              // arg0 rotate right arg1 bits.
+		{name: "RORW", argLength: 2, reg: gp21shift, asm: "RORW", resultInArg0: true, clobberFlags: true},              // arg0 rotate right arg1 bits.
+		{name: "RORB", argLength: 2, reg: gp21shift, asm: "RORB", resultInArg0: true, clobberFlags: true},              // arg0 rotate right arg1 bits.
 		{name: "ROLQconst", argLength: 1, reg: gp11, asm: "ROLQ", aux: "Int8", resultInArg0: true, clobberFlags: true}, // arg0 rotate left auxint, rotate amount 0-63
 		{name: "ROLLconst", argLength: 1, reg: gp11, asm: "ROLL", aux: "Int8", resultInArg0: true, clobberFlags: true}, // arg0 rotate left auxint, rotate amount 0-31
 		{name: "ROLWconst", argLength: 1, reg: gp11, asm: "ROLW", aux: "Int8", resultInArg0: true, clobberFlags: true}, // arg0 rotate left auxint, rotate amount 0-15