Operand(InputOperandAt(instruction, 1)));
}
+void LocationsBuilderARM64::VisitArm64MultiplyAccumulate(HArm64MultiplyAccumulate* instr) {
+ LocationSummary* locations =
+ new (GetGraph()->GetArena()) LocationSummary(instr, LocationSummary::kNoCall);
+ locations->SetInAt(HArm64MultiplyAccumulate::kInputAccumulatorIndex,
+ Location::RequiresRegister());
+ locations->SetInAt(HArm64MultiplyAccumulate::kInputMulLeftIndex, Location::RequiresRegister());
+ locations->SetInAt(HArm64MultiplyAccumulate::kInputMulRightIndex, Location::RequiresRegister());
+ locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
+}
+
+void InstructionCodeGeneratorARM64::VisitArm64MultiplyAccumulate(HArm64MultiplyAccumulate* instr) {
+ Register res = OutputRegister(instr);
+ Register accumulator = InputRegisterAt(instr, HArm64MultiplyAccumulate::kInputAccumulatorIndex);
+ Register mul_left = InputRegisterAt(instr, HArm64MultiplyAccumulate::kInputMulLeftIndex);
+ Register mul_right = InputRegisterAt(instr, HArm64MultiplyAccumulate::kInputMulRightIndex);
+
+ // Avoid emitting code that could trigger Cortex A53's erratum 835769.
+ // This fixup should be carried out for all multiply-accumulate instructions:
+ // madd, msub, smaddl, smsubl, umaddl and umsubl.
+ if (instr->GetType() == Primitive::kPrimLong &&
+ codegen_->GetInstructionSetFeatures().NeedFixCortexA53_835769()) {
+ MacroAssembler* masm = down_cast<CodeGeneratorARM64*>(codegen_)->GetVIXLAssembler();
+ vixl::Instruction* prev = masm->GetCursorAddress<vixl::Instruction*>() - vixl::kInstructionSize;
+ if (prev->IsLoadOrStore()) {
+ // Make sure we emit only exactly one nop.
+ vixl::CodeBufferCheckScope scope(masm,
+ vixl::kInstructionSize,
+ vixl::CodeBufferCheckScope::kCheck,
+ vixl::CodeBufferCheckScope::kExactSize);
+ __ nop();
+ }
+ }
+
+ if (instr->GetOpKind() == HInstruction::kAdd) {
+ __ Madd(res, mul_left, mul_right, accumulator);
+ } else {
+ DCHECK(instr->GetOpKind() == HInstruction::kSub);
+ __ Msub(res, mul_left, mul_right, accumulator);
+ }
+}
+
void LocationsBuilderARM64::VisitArrayGet(HArrayGet* instruction) {
LocationSummary* locations =
new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
RecordSimplification();
}
+bool InstructionSimplifierArm64Visitor::TrySimpleMultiplyAccumulatePatterns(
+ HMul* mul, HBinaryOperation* input_binop, HInstruction* input_other) {
+ DCHECK(Primitive::IsIntOrLongType(mul->GetType()));
+ DCHECK(input_binop->IsAdd() || input_binop->IsSub());
+ DCHECK_NE(input_binop, input_other);
+ if (!input_binop->HasOnlyOneNonEnvironmentUse()) {
+ return false;
+ }
+
+ // Try to interpret patterns like
+ // a * (b <+/-> 1)
+ // as
+ // (a * b) <+/-> a
+ HInstruction* input_a = input_other;
+ HInstruction* input_b = nullptr; // Set to a non-null value if we found a pattern to optimize.
+ HInstruction::InstructionKind op_kind;
+
+ if (input_binop->IsAdd()) {
+ if ((input_binop->GetConstantRight() != nullptr) && input_binop->GetConstantRight()->IsOne()) {
+ // Interpret
+ // a * (b + 1)
+ // as
+ // (a * b) + a
+ input_b = input_binop->GetLeastConstantLeft();
+ op_kind = HInstruction::kAdd;
+ }
+ } else {
+ DCHECK(input_binop->IsSub());
+ if (input_binop->GetRight()->IsConstant() &&
+ input_binop->GetRight()->AsConstant()->IsMinusOne()) {
+ // Interpret
+ // a * (b - (-1))
+ // as
+ // a + (a * b)
+ input_b = input_binop->GetLeft();
+ op_kind = HInstruction::kAdd;
+ } else if (input_binop->GetLeft()->IsConstant() &&
+ input_binop->GetLeft()->AsConstant()->IsOne()) {
+ // Interpret
+ // a * (1 - b)
+ // as
+ // a - (a * b)
+ input_b = input_binop->GetRight();
+ op_kind = HInstruction::kSub;
+ }
+ }
+
+ if (input_b == nullptr) {
+ // We did not find a pattern we can optimize.
+ return false;
+ }
+
+ HArm64MultiplyAccumulate* mulacc = new(GetGraph()->GetArena()) HArm64MultiplyAccumulate(
+ mul->GetType(), op_kind, input_a, input_a, input_b, mul->GetDexPc());
+
+ mul->GetBlock()->ReplaceAndRemoveInstructionWith(mul, mulacc);
+ input_binop->GetBlock()->RemoveInstruction(input_binop);
+
+ return false;
+}
+
void InstructionSimplifierArm64Visitor::VisitArrayGet(HArrayGet* instruction) {
TryExtractArrayAccessAddress(instruction,
instruction->GetArray(),
Primitive::ComponentSize(instruction->GetComponentType()));
}
+void InstructionSimplifierArm64Visitor::VisitMul(HMul* instruction) {
+ Primitive::Type type = instruction->GetType();
+ if (!Primitive::IsIntOrLongType(type)) {
+ return;
+ }
+
+ HInstruction* use = instruction->HasNonEnvironmentUses()
+ ? instruction->GetUses().GetFirst()->GetUser()
+ : nullptr;
+
+ if (instruction->HasOnlyOneNonEnvironmentUse() && (use->IsAdd() || use->IsSub())) {
+ // Replace code looking like
+ // MUL tmp, x, y
+ // SUB dst, acc, tmp
+ // with
+ // MULSUB dst, acc, x, y
+ // Note that we do not want to (unconditionally) perform the merge when the
+ // multiplication has multiple uses and it can be merged in all of them.
+ // Multiple uses could happen on the same control-flow path, and we would
+ // then increase the amount of work. In the future we could try to evaluate
+ // whether all uses are on different control-flow paths (using dominance and
+ // reverse-dominance information) and only perform the merge when they are.
+ HInstruction* accumulator = nullptr;
+ HBinaryOperation* binop = use->AsBinaryOperation();
+ HInstruction* binop_left = binop->GetLeft();
+ HInstruction* binop_right = binop->GetRight();
+ // Be careful after GVN. This should not happen since the `HMul` has only
+ // one use.
+ DCHECK_NE(binop_left, binop_right);
+ if (binop_right == instruction) {
+ accumulator = binop_left;
+ } else if (use->IsAdd()) {
+ DCHECK_EQ(binop_left, instruction);
+ accumulator = binop_right;
+ }
+
+ if (accumulator != nullptr) {
+ HArm64MultiplyAccumulate* mulacc =
+ new (GetGraph()->GetArena()) HArm64MultiplyAccumulate(type,
+ binop->GetKind(),
+ accumulator,
+ instruction->GetLeft(),
+ instruction->GetRight());
+
+ binop->GetBlock()->ReplaceAndRemoveInstructionWith(binop, mulacc);
+ DCHECK(!instruction->HasUses());
+ instruction->GetBlock()->RemoveInstruction(instruction);
+ RecordSimplification();
+ return;
+ }
+ }
+
+ // Use multiply accumulate instruction for a few simple patterns.
+ // We prefer not applying the following transformations if the left and
+ // right inputs perform the same operation.
+ // We rely on GVN having squashed the inputs if appropriate. However the
+ // results are still correct even if that did not happen.
+ if (instruction->GetLeft() == instruction->GetRight()) {
+ return;
+ }
+
+ HInstruction* left = instruction->GetLeft();
+ HInstruction* right = instruction->GetRight();
+ if ((right->IsAdd() || right->IsSub()) &&
+ TrySimpleMultiplyAccumulatePatterns(instruction, right->AsBinaryOperation(), left)) {
+ return;
+ }
+ if ((left->IsAdd() || left->IsSub()) &&
+ TrySimpleMultiplyAccumulatePatterns(instruction, left->AsBinaryOperation(), right)) {
+ return;
+ }
+}
+
} // namespace arm64
} // namespace art
--- /dev/null
+/*
+* Copyright (C) 2015 The Android Open Source Project
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+public class Main {
+
+ // A dummy value to defeat inlining of these routines.
+ static boolean doThrow = false;
+
+ public static void assertIntEquals(int expected, int result) {
+ if (expected != result) {
+ throw new Error("Expected: " + expected + ", found: " + result);
+ }
+ }
+
+ public static void assertLongEquals(long expected, long result) {
+ if (expected != result) {
+ throw new Error("Expected: " + expected + ", found: " + result);
+ }
+ }
+
+ /**
+ * Test basic merging of `MUL+ADD` into `MULADD`.
+ */
+
+ /// CHECK-START-ARM64: int Main.$opt$noinline$mulAdd(int, int, int) instruction_simplifier_arm64 (before)
+ /// CHECK: <<Acc:i\d+>> ParameterValue
+ /// CHECK: <<Left:i\d+>> ParameterValue
+ /// CHECK: <<Right:i\d+>> ParameterValue
+ /// CHECK: <<Mul:i\d+>> Mul [<<Left>>,<<Right>>]
+ /// CHECK: <<Add:i\d+>> Add [<<Acc>>,<<Mul>>]
+ /// CHECK: Return [<<Add>>]
+
+ /// CHECK-START-ARM64: int Main.$opt$noinline$mulAdd(int, int, int) instruction_simplifier_arm64 (after)
+ /// CHECK: <<Acc:i\d+>> ParameterValue
+ /// CHECK: <<Left:i\d+>> ParameterValue
+ /// CHECK: <<Right:i\d+>> ParameterValue
+ /// CHECK: <<MulAdd:i\d+>> Arm64MultiplyAccumulate [<<Acc>>,<<Left>>,<<Right>>] kind:Add
+ /// CHECK: Return [<<MulAdd>>]
+
+ /// CHECK-START-ARM64: int Main.$opt$noinline$mulAdd(int, int, int) instruction_simplifier_arm64 (after)
+ /// CHECK-NOT: Mul
+ /// CHECK-NOT: Add
+
+ /// CHECK-START-ARM64: int Main.$opt$noinline$mulAdd(int, int, int) disassembly (after)
+ /// CHECK: madd w{{\d+}}, w{{\d+}}, w{{\d+}}, w{{\d+}}
+
+ public static int $opt$noinline$mulAdd(int acc, int left, int right) {
+ if (doThrow) throw new Error();
+ return acc + left * right;
+ }
+
+ /**
+ * Test basic merging of `MUL+SUB` into `MULSUB`.
+ */
+
+ /// CHECK-START-ARM64: long Main.$opt$noinline$mulSub(long, long, long) instruction_simplifier_arm64 (before)
+ /// CHECK: <<Acc:j\d+>> ParameterValue
+ /// CHECK: <<Left:j\d+>> ParameterValue
+ /// CHECK: <<Right:j\d+>> ParameterValue
+ /// CHECK: <<Mul:j\d+>> Mul [<<Left>>,<<Right>>]
+ /// CHECK: <<Sub:j\d+>> Sub [<<Acc>>,<<Mul>>]
+ /// CHECK: Return [<<Sub>>]
+
+ /// CHECK-START-ARM64: long Main.$opt$noinline$mulSub(long, long, long) instruction_simplifier_arm64 (after)
+ /// CHECK: <<Acc:j\d+>> ParameterValue
+ /// CHECK: <<Left:j\d+>> ParameterValue
+ /// CHECK: <<Right:j\d+>> ParameterValue
+ /// CHECK: <<MulSub:j\d+>> Arm64MultiplyAccumulate [<<Acc>>,<<Left>>,<<Right>>] kind:Sub
+ /// CHECK: Return [<<MulSub>>]
+
+ /// CHECK-START-ARM64: long Main.$opt$noinline$mulSub(long, long, long) instruction_simplifier_arm64 (after)
+ /// CHECK-NOT: Mul
+ /// CHECK-NOT: Sub
+
+ /// CHECK-START-ARM64: long Main.$opt$noinline$mulSub(long, long, long) disassembly (after)
+ /// CHECK: msub x{{\d+}}, x{{\d+}}, x{{\d+}}, x{{\d+}}
+
+ public static long $opt$noinline$mulSub(long acc, long left, long right) {
+ if (doThrow) throw new Error();
+ return acc - left * right;
+ }
+
+ /**
+ * Test that we do not create a multiply-accumulate instruction when there
+ * are other uses of the multiplication that cannot merge it.
+ */
+
+ /// CHECK-START-ARM64: int Main.$opt$noinline$multipleUses1(int, int, int) instruction_simplifier_arm64 (before)
+ /// CHECK: <<Acc:i\d+>> ParameterValue
+ /// CHECK: <<Left:i\d+>> ParameterValue
+ /// CHECK: <<Right:i\d+>> ParameterValue
+ /// CHECK: <<Mul:i\d+>> Mul [<<Left>>,<<Right>>]
+ /// CHECK: <<Add:i\d+>> Add [<<Acc>>,<<Mul>>]
+ /// CHECK: <<Or:i\d+>> Or [<<Mul>>,<<Add>>]
+ /// CHECK: Return [<<Or>>]
+
+ /// CHECK-START-ARM64: int Main.$opt$noinline$multipleUses1(int, int, int) instruction_simplifier_arm64 (after)
+ /// CHECK: <<Acc:i\d+>> ParameterValue
+ /// CHECK: <<Left:i\d+>> ParameterValue
+ /// CHECK: <<Right:i\d+>> ParameterValue
+ /// CHECK: <<Mul:i\d+>> Mul [<<Left>>,<<Right>>]
+ /// CHECK: <<Add:i\d+>> Add [<<Acc>>,<<Mul>>]
+ /// CHECK: <<Or:i\d+>> Or [<<Mul>>,<<Add>>]
+ /// CHECK: Return [<<Or>>]
+
+ /// CHECK-START-ARM64: int Main.$opt$noinline$multipleUses1(int, int, int) instruction_simplifier_arm64 (after)
+ /// CHECK-NOT: Arm64MultiplyAccumulate
+
+ public static int $opt$noinline$multipleUses1(int acc, int left, int right) {
+ if (doThrow) throw new Error();
+ int temp = left * right;
+ return temp | (acc + temp);
+ }
+
+ /**
+ * Test that we do not create a multiply-accumulate instruction even when all
+ * uses of the multiplication can merge it.
+ */
+
+ /// CHECK-START-ARM64: long Main.$opt$noinline$multipleUses2(long, long, long) instruction_simplifier_arm64 (before)
+ /// CHECK: <<Acc:j\d+>> ParameterValue
+ /// CHECK: <<Left:j\d+>> ParameterValue
+ /// CHECK: <<Right:j\d+>> ParameterValue
+ /// CHECK: <<Mul:j\d+>> Mul [<<Left>>,<<Right>>]
+ /// CHECK: <<Add:j\d+>> Add [<<Acc>>,<<Mul>>]
+ /// CHECK: <<Sub:j\d+>> Sub [<<Acc>>,<<Mul>>]
+ /// CHECK: <<Res:j\d+>> Add [<<Add>>,<<Sub>>]
+ /// CHECK: Return [<<Res>>]
+
+ /// CHECK-START-ARM64: long Main.$opt$noinline$multipleUses2(long, long, long) instruction_simplifier_arm64 (after)
+ /// CHECK: <<Acc:j\d+>> ParameterValue
+ /// CHECK: <<Left:j\d+>> ParameterValue
+ /// CHECK: <<Right:j\d+>> ParameterValue
+ /// CHECK: <<Mul:j\d+>> Mul [<<Left>>,<<Right>>]
+ /// CHECK: <<Add:j\d+>> Add [<<Acc>>,<<Mul>>]
+ /// CHECK: <<Sub:j\d+>> Sub [<<Acc>>,<<Mul>>]
+ /// CHECK: <<Res:j\d+>> Add [<<Add>>,<<Sub>>]
+ /// CHECK: Return [<<Res>>]
+
+ /// CHECK-START-ARM64: long Main.$opt$noinline$multipleUses2(long, long, long) instruction_simplifier_arm64 (after)
+ /// CHECK-NOT: Arm64MultiplyAccumulate
+
+
+ public static long $opt$noinline$multipleUses2(long acc, long left, long right) {
+ if (doThrow) throw new Error();
+ long temp = left * right;
+ return (acc + temp) + (acc - temp);
+ }
+
+
+ /**
+ * Test the interpretation of `a * (b + 1)` as `a + (a * b)`.
+ */
+
+ /// CHECK-START-ARM64: int Main.$opt$noinline$mulPlusOne(int, int) instruction_simplifier_arm64 (before)
+ /// CHECK: <<Acc:i\d+>> ParameterValue
+ /// CHECK: <<Var:i\d+>> ParameterValue
+ /// CHECK: <<Const1:i\d+>> IntConstant 1
+ /// CHECK: <<Add:i\d+>> Add [<<Var>>,<<Const1>>]
+ /// CHECK: <<Mul:i\d+>> Mul [<<Acc>>,<<Add>>]
+ /// CHECK: Return [<<Mul>>]
+
+ /// CHECK-START-ARM64: int Main.$opt$noinline$mulPlusOne(int, int) instruction_simplifier_arm64 (after)
+ /// CHECK: <<Acc:i\d+>> ParameterValue
+ /// CHECK: <<Var:i\d+>> ParameterValue
+ /// CHECK: <<MulAdd:i\d+>> Arm64MultiplyAccumulate [<<Acc>>,<<Acc>>,<<Var>>] kind:Add
+ /// CHECK: Return [<<MulAdd>>]
+
+ /// CHECK-START-ARM64: int Main.$opt$noinline$mulPlusOne(int, int) instruction_simplifier_arm64 (after)
+ /// CHECK-NOT: Mul
+ /// CHECK-NOT: Add
+
+ /// CHECK-START-ARM64: int Main.$opt$noinline$mulPlusOne(int, int) disassembly (after)
+ /// CHECK: madd w{{\d+}}, w{{\d+}}, w{{\d+}}, w{{\d+}}
+
+ public static int $opt$noinline$mulPlusOne(int acc, int var) {
+ if (doThrow) throw new Error();
+ return acc * (var + 1);
+ }
+
+
+ /**
+ * Test the interpretation of `a * (1 - b)` as `a - (a * b)`.
+ */
+
+ /// CHECK-START-ARM64: long Main.$opt$noinline$mulMinusOne(long, long) instruction_simplifier_arm64 (before)
+ /// CHECK: <<Acc:j\d+>> ParameterValue
+ /// CHECK: <<Var:j\d+>> ParameterValue
+ /// CHECK: <<Const1:j\d+>> LongConstant 1
+ /// CHECK: <<Sub:j\d+>> Sub [<<Const1>>,<<Var>>]
+ /// CHECK: <<Mul:j\d+>> Mul [<<Acc>>,<<Sub>>]
+ /// CHECK: Return [<<Mul>>]
+
+ /// CHECK-START-ARM64: long Main.$opt$noinline$mulMinusOne(long, long) instruction_simplifier_arm64 (after)
+ /// CHECK: <<Acc:j\d+>> ParameterValue
+ /// CHECK: <<Var:j\d+>> ParameterValue
+ /// CHECK: <<MulSub:j\d+>> Arm64MultiplyAccumulate [<<Acc>>,<<Acc>>,<<Var>>] kind:Sub
+ /// CHECK: Return [<<MulSub>>]
+
+ /// CHECK-START-ARM64: long Main.$opt$noinline$mulMinusOne(long, long) instruction_simplifier_arm64 (after)
+ /// CHECK-NOT: Mul
+ /// CHECK-NOT: Sub
+
+ /// CHECK-START-ARM64: long Main.$opt$noinline$mulMinusOne(long, long) disassembly (after)
+ /// CHECK: msub x{{\d+}}, x{{\d+}}, x{{\d+}}, x{{\d+}}
+
+ public static long $opt$noinline$mulMinusOne(long acc, long var) {
+ if (doThrow) throw new Error();
+ return acc * (1 - var);
+ }
+
+
+ public static void main(String[] args) {
+ assertIntEquals(7, $opt$noinline$mulAdd(1, 2, 3));
+ assertLongEquals(-26, $opt$noinline$mulSub(4, 5, 6));
+ assertIntEquals(79, $opt$noinline$multipleUses1(7, 8, 9));
+ assertLongEquals(20, $opt$noinline$multipleUses2(10, 11, 12));
+ assertIntEquals(195, $opt$noinline$mulPlusOne(13, 14));
+ assertLongEquals(-225, $opt$noinline$mulMinusOne(15, 16));
+ }
+}
OSDN Git Service
