36 import org.graalvm.compiler.core.common.LIRKind; |
36 import org.graalvm.compiler.core.common.LIRKind; |
37 import org.graalvm.compiler.core.common.calc.CanonicalCondition; |
37 import org.graalvm.compiler.core.common.calc.CanonicalCondition; |
38 import org.graalvm.compiler.core.gen.NodeMatchRules; |
38 import org.graalvm.compiler.core.gen.NodeMatchRules; |
39 import org.graalvm.compiler.core.match.ComplexMatchResult; |
39 import org.graalvm.compiler.core.match.ComplexMatchResult; |
40 import org.graalvm.compiler.core.match.MatchRule; |
40 import org.graalvm.compiler.core.match.MatchRule; |
41 import org.graalvm.compiler.graph.Node; |
|
42 import org.graalvm.compiler.lir.LIRFrameState; |
41 import org.graalvm.compiler.lir.LIRFrameState; |
43 import org.graalvm.compiler.lir.LabelRef; |
42 import org.graalvm.compiler.lir.LabelRef; |
44 import org.graalvm.compiler.lir.Variable; |
43 import org.graalvm.compiler.lir.Variable; |
45 import org.graalvm.compiler.lir.aarch64.AArch64ArithmeticOp; |
44 import org.graalvm.compiler.lir.aarch64.AArch64ArithmeticOp; |
46 import org.graalvm.compiler.lir.aarch64.AArch64BitFieldOp; |
45 import org.graalvm.compiler.lir.aarch64.AArch64BitFieldOp; |
56 import org.graalvm.compiler.nodes.calc.AndNode; |
55 import org.graalvm.compiler.nodes.calc.AndNode; |
57 import org.graalvm.compiler.nodes.calc.BinaryNode; |
56 import org.graalvm.compiler.nodes.calc.BinaryNode; |
58 import org.graalvm.compiler.nodes.calc.IntegerLessThanNode; |
57 import org.graalvm.compiler.nodes.calc.IntegerLessThanNode; |
59 import org.graalvm.compiler.nodes.calc.LeftShiftNode; |
58 import org.graalvm.compiler.nodes.calc.LeftShiftNode; |
60 import org.graalvm.compiler.nodes.calc.MulNode; |
59 import org.graalvm.compiler.nodes.calc.MulNode; |
|
60 import org.graalvm.compiler.nodes.calc.NarrowNode; |
|
61 import org.graalvm.compiler.nodes.calc.NegateNode; |
61 import org.graalvm.compiler.nodes.calc.NotNode; |
62 import org.graalvm.compiler.nodes.calc.NotNode; |
62 import org.graalvm.compiler.nodes.calc.OrNode; |
63 import org.graalvm.compiler.nodes.calc.OrNode; |
63 import org.graalvm.compiler.nodes.calc.RightShiftNode; |
64 import org.graalvm.compiler.nodes.calc.RightShiftNode; |
64 import org.graalvm.compiler.nodes.calc.SubNode; |
65 import org.graalvm.compiler.nodes.calc.SubNode; |
|
66 import org.graalvm.compiler.nodes.calc.UnaryNode; |
65 import org.graalvm.compiler.nodes.calc.UnsignedRightShiftNode; |
67 import org.graalvm.compiler.nodes.calc.UnsignedRightShiftNode; |
66 import org.graalvm.compiler.nodes.calc.XorNode; |
68 import org.graalvm.compiler.nodes.calc.XorNode; |
67 import org.graalvm.compiler.nodes.memory.Access; |
69 import org.graalvm.compiler.nodes.memory.Access; |
68 |
70 |
69 public class AArch64NodeMatchRules extends NodeMatchRules { |
71 public class AArch64NodeMatchRules extends NodeMatchRules { |
70 private static final EconomicMap<Class<? extends Node>, AArch64ArithmeticOp> nodeOpMap; |
72 private static final EconomicMap<Class<? extends BinaryNode>, AArch64ArithmeticOp> binaryOpMap; |
71 private static final EconomicMap<Class<? extends BinaryNode>, AArch64BitFieldOp.BitFieldOpCode> bitFieldOpMap; |
73 private static final EconomicMap<Class<? extends BinaryNode>, AArch64BitFieldOp.BitFieldOpCode> bitFieldOpMap; |
72 private static final EconomicMap<Class<? extends BinaryNode>, AArch64MacroAssembler.ShiftType> shiftTypeMap; |
74 private static final EconomicMap<Class<? extends BinaryNode>, AArch64MacroAssembler.ShiftType> shiftTypeMap; |
73 |
75 |
74 static { |
76 static { |
75 nodeOpMap = EconomicMap.create(Equivalence.IDENTITY, 5); |
77 binaryOpMap = EconomicMap.create(Equivalence.IDENTITY, 9); |
76 nodeOpMap.put(AddNode.class, AArch64ArithmeticOp.ADD); |
78 binaryOpMap.put(AddNode.class, AArch64ArithmeticOp.ADD); |
77 nodeOpMap.put(SubNode.class, AArch64ArithmeticOp.SUB); |
79 binaryOpMap.put(SubNode.class, AArch64ArithmeticOp.SUB); |
78 nodeOpMap.put(AndNode.class, AArch64ArithmeticOp.AND); |
80 binaryOpMap.put(MulNode.class, AArch64ArithmeticOp.MUL); |
79 nodeOpMap.put(OrNode.class, AArch64ArithmeticOp.OR); |
81 binaryOpMap.put(AndNode.class, AArch64ArithmeticOp.AND); |
80 nodeOpMap.put(XorNode.class, AArch64ArithmeticOp.XOR); |
82 binaryOpMap.put(OrNode.class, AArch64ArithmeticOp.OR); |
|
83 binaryOpMap.put(XorNode.class, AArch64ArithmeticOp.XOR); |
|
84 binaryOpMap.put(LeftShiftNode.class, AArch64ArithmeticOp.SHL); |
|
85 binaryOpMap.put(RightShiftNode.class, AArch64ArithmeticOp.ASHR); |
|
86 binaryOpMap.put(UnsignedRightShiftNode.class, AArch64ArithmeticOp.LSHR); |
81 |
87 |
82 bitFieldOpMap = EconomicMap.create(Equivalence.IDENTITY, 2); |
88 bitFieldOpMap = EconomicMap.create(Equivalence.IDENTITY, 2); |
83 bitFieldOpMap.put(UnsignedRightShiftNode.class, AArch64BitFieldOp.BitFieldOpCode.UBFX); |
89 bitFieldOpMap.put(UnsignedRightShiftNode.class, AArch64BitFieldOp.BitFieldOpCode.UBFX); |
84 bitFieldOpMap.put(LeftShiftNode.class, AArch64BitFieldOp.BitFieldOpCode.UBFIZ); |
90 bitFieldOpMap.put(LeftShiftNode.class, AArch64BitFieldOp.BitFieldOpCode.UBFIZ); |
85 |
91 |
151 trueProbability, nbits)); |
157 trueProbability, nbits)); |
152 return null; |
158 return null; |
153 }; |
159 }; |
154 } |
160 } |
155 |
161 |
|
162 private static boolean isNarrowingLongToInt(NarrowNode narrow) { |
|
163 return narrow.getInputBits() == 64 && narrow.getResultBits() == 32; |
|
164 } |
|
165 |
156 @MatchRule("(And (UnsignedRightShift=shift a Constant=b) Constant=c)") |
166 @MatchRule("(And (UnsignedRightShift=shift a Constant=b) Constant=c)") |
157 @MatchRule("(LeftShift=shift (And a Constant=c) Constant=b)") |
167 @MatchRule("(LeftShift=shift (And a Constant=c) Constant=b)") |
158 public ComplexMatchResult unsignedBitField(BinaryNode shift, ValueNode a, ConstantNode b, ConstantNode c) { |
168 public ComplexMatchResult unsignedBitField(BinaryNode shift, ValueNode a, ConstantNode b, ConstantNode c) { |
159 JavaKind srcKind = a.getStackKind(); |
169 JavaKind srcKind = a.getStackKind(); |
160 assert srcKind.isNumericInteger(); |
170 assert srcKind.isNumericInteger(); |
192 @MatchRule("(Add=binary a (UnsignedRightShift=shift b Constant))") |
202 @MatchRule("(Add=binary a (UnsignedRightShift=shift b Constant))") |
193 @MatchRule("(Sub=binary a (LeftShift=shift b Constant))") |
203 @MatchRule("(Sub=binary a (LeftShift=shift b Constant))") |
194 @MatchRule("(Sub=binary a (RightShift=shift b Constant))") |
204 @MatchRule("(Sub=binary a (RightShift=shift b Constant))") |
195 @MatchRule("(Sub=binary a (UnsignedRightShift=shift b Constant))") |
205 @MatchRule("(Sub=binary a (UnsignedRightShift=shift b Constant))") |
196 public ComplexMatchResult addSubShift(BinaryNode binary, ValueNode a, BinaryNode shift) { |
206 public ComplexMatchResult addSubShift(BinaryNode binary, ValueNode a, BinaryNode shift) { |
197 AArch64ArithmeticOp op = nodeOpMap.get(binary.getClass()); |
207 AArch64ArithmeticOp op = binaryOpMap.get(binary.getClass()); |
198 assert op != null; |
208 assert op != null; |
199 return emitBinaryShift(op, a, shift, false); |
209 return emitBinaryShift(op, a, shift, false); |
200 } |
210 } |
201 |
211 |
202 @MatchRule("(And=binary a (LeftShift=shift b Constant))") |
212 @MatchRule("(And=binary a (LeftShift=shift b Constant))") |
216 @MatchRule("(Or=binary a (Not (UnsignedRightShift=shift b Constant)))") |
226 @MatchRule("(Or=binary a (Not (UnsignedRightShift=shift b Constant)))") |
217 @MatchRule("(Xor=binary a (Not (LeftShift=shift b Constant)))") |
227 @MatchRule("(Xor=binary a (Not (LeftShift=shift b Constant)))") |
218 @MatchRule("(Xor=binary a (Not (RightShift=shift b Constant)))") |
228 @MatchRule("(Xor=binary a (Not (RightShift=shift b Constant)))") |
219 @MatchRule("(Xor=binary a (Not (UnsignedRightShift=shift b Constant)))") |
229 @MatchRule("(Xor=binary a (Not (UnsignedRightShift=shift b Constant)))") |
220 public ComplexMatchResult logicShift(BinaryNode binary, ValueNode a, BinaryNode shift) { |
230 public ComplexMatchResult logicShift(BinaryNode binary, ValueNode a, BinaryNode shift) { |
221 AArch64ArithmeticOp op = nodeOpMap.get(binary.getClass()); |
231 AArch64ArithmeticOp op = binaryOpMap.get(binary.getClass()); |
222 assert op != null; |
232 assert op != null; |
223 ValueNode operand = binary.getX() == a ? binary.getY() : binary.getX(); |
233 ValueNode operand = binary.getX() == a ? binary.getY() : binary.getX(); |
224 boolean isShiftNot = operand instanceof NotNode; |
234 boolean isShiftNot = operand instanceof NotNode; |
225 return emitBinaryShift(op, a, shift, isShiftNot); |
235 return emitBinaryShift(op, a, shift, isShiftNot); |
226 } |
236 } |
248 public ComplexMatchResult signedMultiplyLong(MulNode mul, ValueNode a, ValueNode b) { |
258 public ComplexMatchResult signedMultiplyLong(MulNode mul, ValueNode a, ValueNode b) { |
249 assert a.getStackKind() == JavaKind.Int && b.getStackKind() == JavaKind.Int; |
259 assert a.getStackKind() == JavaKind.Int && b.getStackKind() == JavaKind.Int; |
250 LIRKind resultKind = LIRKind.fromJavaKind(gen.target().arch, mul.getStackKind()); |
260 LIRKind resultKind = LIRKind.fromJavaKind(gen.target().arch, mul.getStackKind()); |
251 return builder -> getArithmeticLIRGenerator().emitBinary( |
261 return builder -> getArithmeticLIRGenerator().emitBinary( |
252 resultKind, AArch64ArithmeticOp.SMULL, true, operand(a), operand(b)); |
262 resultKind, AArch64ArithmeticOp.SMULL, true, operand(a), operand(b)); |
|
263 } |
|
264 |
|
265 @MatchRule("(Add=binary (Narrow=narrow a) (Narrow b))") |
|
266 @MatchRule("(Sub=binary (Narrow=narrow a) (Narrow b))") |
|
267 @MatchRule("(Mul=binary (Narrow=narrow a) (Narrow b))") |
|
268 @MatchRule("(And=binary (Narrow=narrow a) (Narrow b))") |
|
269 @MatchRule("(Or=binary (Narrow=narrow a) (Narrow b))") |
|
270 @MatchRule("(Xor=binary (Narrow=narrow a) (Narrow b))") |
|
271 @MatchRule("(LeftShift=binary (Narrow=narrow a) (Narrow b))") |
|
272 @MatchRule("(RightShift=binary (Narrow=narrow a) (Narrow b))") |
|
273 @MatchRule("(UnsignedRightShift=binary (Narrow=narrow a) (Narrow b))") |
|
274 @MatchRule("(Add=binary a (Narrow=narrow b))") |
|
275 @MatchRule("(Sub=binary a (Narrow=narrow b))") |
|
276 @MatchRule("(Mul=binary a (Narrow=narrow b))") |
|
277 @MatchRule("(And=binary a (Narrow=narrow b))") |
|
278 @MatchRule("(Or=binary a (Narrow=narrow b))") |
|
279 @MatchRule("(Xor=binary a (Narrow=narrow b))") |
|
280 @MatchRule("(LeftShift=binary a (Narrow=narrow b))") |
|
281 @MatchRule("(RightShift=binary a (Narrow=narrow b))") |
|
282 @MatchRule("(UnsignedRightShift=binary a (Narrow=narrow b))") |
|
283 @MatchRule("(Sub=binary (Narrow=narrow a) b)") |
|
284 @MatchRule("(LeftShift=binary (Narrow=narrow a) b)") |
|
285 @MatchRule("(RightShift=binary (Narrow=narrow a) b)") |
|
286 @MatchRule("(UnsignedRightShift=binary (Narrow=narrow a) b)") |
|
287 public ComplexMatchResult elideL2IForBinary(BinaryNode binary, NarrowNode narrow) { |
|
288 assert binary.getStackKind().isNumericInteger(); |
|
289 |
|
290 ValueNode a = narrow; |
|
291 ValueNode b = binary.getX() == narrow ? binary.getY() : binary.getX(); |
|
292 boolean isL2Ia = isNarrowingLongToInt((NarrowNode) a); |
|
293 boolean isL2Ib = (b instanceof NarrowNode) && isNarrowingLongToInt((NarrowNode) b); |
|
294 if (!isL2Ia && !isL2Ib) { |
|
295 return null; |
|
296 } |
|
297 // Get the value of L2I NarrowNode as the src value. |
|
298 ValueNode src1 = isL2Ia ? ((NarrowNode) a).getValue() : a; |
|
299 ValueNode src2 = isL2Ib ? ((NarrowNode) b).getValue() : b; |
|
300 |
|
301 AArch64ArithmeticOp op = binaryOpMap.get(binary.getClass()); |
|
302 assert op != null; |
|
303 boolean commutative = binary.getNodeClass().isCommutative(); |
|
304 LIRKind resultKind = LIRKind.fromJavaKind(gen.target().arch, binary.getStackKind()); |
|
305 |
|
306 // Must keep the right operator order for un-commutative binary operations. |
|
307 if (a == binary.getX()) { |
|
308 return builder -> getArithmeticLIRGenerator().emitBinary( |
|
309 resultKind, op, commutative, operand(src1), operand(src2)); |
|
310 } |
|
311 return builder -> getArithmeticLIRGenerator().emitBinary( |
|
312 resultKind, op, commutative, operand(src2), operand(src1)); |
|
313 } |
|
314 |
|
315 @MatchRule("(Negate=unary (Narrow=narrow value))") |
|
316 @MatchRule("(Not=unary (Narrow=narrow value))") |
|
317 public ComplexMatchResult elideL2IForUnary(UnaryNode unary, NarrowNode narrow) { |
|
318 assert unary.getStackKind().isNumericInteger(); |
|
319 if (!isNarrowingLongToInt(narrow)) { |
|
320 return null; |
|
321 } |
|
322 |
|
323 AArch64ArithmeticOp op = unary instanceof NegateNode ? AArch64ArithmeticOp.NEG |
|
324 : AArch64ArithmeticOp.NOT; |
|
325 return builder -> { |
|
326 AllocatableValue input = gen.asAllocatable(operand(narrow.getValue())); |
|
327 LIRKind resultKind = LIRKind.fromJavaKind(gen.target().arch, unary.getStackKind()); |
|
328 Variable result = gen.newVariable(resultKind); |
|
329 gen.append(new AArch64ArithmeticOp.UnaryOp(op, result, moveSp(input))); |
|
330 return result; |
|
331 }; |
253 } |
332 } |
254 |
333 |
255 @MatchRule("(Mul (Negate a) b)") |
334 @MatchRule("(Mul (Negate a) b)") |
256 @MatchRule("(Negate (Mul a b))") |
335 @MatchRule("(Negate (Mul a b))") |
257 public ComplexMatchResult multiplyNegate(ValueNode a, ValueNode b) { |
336 public ComplexMatchResult multiplyNegate(ValueNode a, ValueNode b) { |