namespace {
-/// \brief Check if the instr pair, FirstMI and SecondMI, should be fused
-/// together. Given SecondMI, when FirstMI is unspecified, then check if
-/// SecondMI may be part of a fused pair at all.
-static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
- const TargetSubtargetInfo &TSI,
- const MachineInstr *FirstMI,
- const MachineInstr &SecondMI) {
- const AArch64Subtarget &ST = static_cast<const AArch64Subtarget&>(TSI);
+// Fuse CMN, CMP, TST followed by Bcc.
+static bool isArithmeticBccPair(const MachineInstr *FirstMI,
+ const MachineInstr &SecondMI) {
+ if (SecondMI.getOpcode() == AArch64::Bcc) {
+ // Assume the 1st instr to be a wildcard if it is unspecified.
+ if (!FirstMI)
+ return true;
+
+ switch (FirstMI->getOpcode()) {
+ case AArch64::ADDSWri:
+ case AArch64::ADDSWrr:
+ case AArch64::ADDSXri:
+ case AArch64::ADDSXrr:
+ case AArch64::ANDSWri:
+ case AArch64::ANDSWrr:
+ case AArch64::ANDSXri:
+ case AArch64::ANDSXrr:
+ case AArch64::SUBSWri:
+ case AArch64::SUBSWrr:
+ case AArch64::SUBSXri:
+ case AArch64::SUBSXrr:
+ case AArch64::BICSWrr:
+ case AArch64::BICSXrr:
+ return true;
+ case AArch64::ADDSWrs:
+ case AArch64::ADDSXrs:
+ case AArch64::ANDSWrs:
+ case AArch64::ANDSXrs:
+ case AArch64::SUBSWrs:
+ case AArch64::SUBSXrs:
+ case AArch64::BICSWrs:
+ case AArch64::BICSXrs:
+ // Shift value can be 0 making these behave like the "rr" variant...
+ if (!AArch64InstrInfo::hasShiftedReg(*FirstMI))
+ return true;
+ }
+ }
+ return false;
+}
- // Assume wildcards for unspecified instrs.
+// Fuse ALU operations followed by CBZ/CBNZ.
+static bool isArithmeticCbzPair(const MachineInstr *FirstMI,
+ const MachineInstr &SecondMI) {
+ unsigned SecondOpcode = SecondMI.getOpcode();
+
+ if (SecondOpcode == AArch64::CBNZW || SecondOpcode == AArch64::CBNZX ||
+ SecondOpcode == AArch64::CBZW || SecondOpcode == AArch64::CBZX) {
+ // Assume the 1st instr to be a wildcard if it is unspecified.
+ if (!FirstMI)
+ return true;
+
+ switch (FirstMI->getOpcode()) {
+ case AArch64::ADDWri:
+ case AArch64::ADDWrr:
+ case AArch64::ADDXri:
+ case AArch64::ADDXrr:
+ case AArch64::ANDWri:
+ case AArch64::ANDWrr:
+ case AArch64::ANDXri:
+ case AArch64::ANDXrr:
+ case AArch64::EORWri:
+ case AArch64::EORWrr:
+ case AArch64::EORXri:
+ case AArch64::EORXrr:
+ case AArch64::ORRWri:
+ case AArch64::ORRWrr:
+ case AArch64::ORRXri:
+ case AArch64::ORRXrr:
+ case AArch64::SUBWri:
+ case AArch64::SUBWrr:
+ case AArch64::SUBXri:
+ case AArch64::SUBXrr:
+ return true;
+ case AArch64::ADDWrs:
+ case AArch64::ADDXrs:
+ case AArch64::ANDWrs:
+ case AArch64::ANDXrs:
+ case AArch64::SUBWrs:
+ case AArch64::SUBXrs:
+ case AArch64::BICWrs:
+ case AArch64::BICXrs:
+ // Shift value can be 0 making these behave like the "rr" variant...
+ if (!AArch64InstrInfo::hasShiftedReg(*FirstMI))
+ return true;
+ }
+ }
+ return false;
+}
+
+// Fuse AES crypto encoding or decoding.
+static bool isAESPair(const MachineInstr *FirstMI,
+ const MachineInstr &SecondMI) {
+ // Assume the 1st instr to be a wildcard if it is unspecified.
unsigned FirstOpcode =
FirstMI ? FirstMI->getOpcode()
: static_cast<unsigned>(AArch64::INSTRUCTION_LIST_END);
unsigned SecondOpcode = SecondMI.getOpcode();
- if (ST.hasArithmeticBccFusion())
- // Fuse CMN, CMP, TST followed by Bcc.
- if (SecondOpcode == AArch64::Bcc)
- switch (FirstOpcode) {
- default:
- return false;
- case AArch64::ADDSWri:
- case AArch64::ADDSWrr:
- case AArch64::ADDSXri:
- case AArch64::ADDSXrr:
- case AArch64::ANDSWri:
- case AArch64::ANDSWrr:
- case AArch64::ANDSXri:
- case AArch64::ANDSXrr:
- case AArch64::SUBSWri:
- case AArch64::SUBSWrr:
- case AArch64::SUBSXri:
- case AArch64::SUBSXrr:
- case AArch64::BICSWrr:
- case AArch64::BICSXrr:
- return true;
- case AArch64::ADDSWrs:
- case AArch64::ADDSXrs:
- case AArch64::ANDSWrs:
- case AArch64::ANDSXrs:
- case AArch64::SUBSWrs:
- case AArch64::SUBSXrs:
- case AArch64::BICSWrs:
- case AArch64::BICSXrs:
- // Shift value can be 0 making these behave like the "rr" variant...
- return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
- case AArch64::INSTRUCTION_LIST_END:
- return true;
- }
-
- if (ST.hasArithmeticCbzFusion())
- // Fuse ALU operations followed by CBZ/CBNZ.
- if (SecondOpcode == AArch64::CBNZW || SecondOpcode == AArch64::CBNZX ||
- SecondOpcode == AArch64::CBZW || SecondOpcode == AArch64::CBZX)
- switch (FirstOpcode) {
- default:
- return false;
- case AArch64::ADDWri:
- case AArch64::ADDWrr:
- case AArch64::ADDXri:
- case AArch64::ADDXrr:
- case AArch64::ANDWri:
- case AArch64::ANDWrr:
- case AArch64::ANDXri:
- case AArch64::ANDXrr:
- case AArch64::EORWri:
- case AArch64::EORWrr:
- case AArch64::EORXri:
- case AArch64::EORXrr:
- case AArch64::ORRWri:
- case AArch64::ORRWrr:
- case AArch64::ORRXri:
- case AArch64::ORRXrr:
- case AArch64::SUBWri:
- case AArch64::SUBWrr:
- case AArch64::SUBXri:
- case AArch64::SUBXrr:
- return true;
- case AArch64::ADDWrs:
- case AArch64::ADDXrs:
- case AArch64::ANDWrs:
- case AArch64::ANDXrs:
- case AArch64::SUBWrs:
- case AArch64::SUBXrs:
- case AArch64::BICWrs:
- case AArch64::BICXrs:
- // Shift value can be 0 making these behave like the "rr" variant...
- return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
- case AArch64::INSTRUCTION_LIST_END:
- return true;
- }
-
- if (ST.hasFuseAES())
- // Fuse AES crypto operations.
- switch(SecondOpcode) {
- // AES encode.
- case AArch64::AESMCrr:
- case AArch64::AESMCrrTied:
- return FirstOpcode == AArch64::AESErr ||
- FirstOpcode == AArch64::INSTRUCTION_LIST_END;
- // AES decode.
- case AArch64::AESIMCrr:
- case AArch64::AESIMCrrTied:
- return FirstOpcode == AArch64::AESDrr ||
- FirstOpcode == AArch64::INSTRUCTION_LIST_END;
- }
+ // AES encode.
+ if ((FirstOpcode == AArch64::INSTRUCTION_LIST_END ||
+ FirstOpcode == AArch64::AESErr) &&
+ (SecondOpcode == AArch64::AESMCrr ||
+ SecondOpcode == AArch64::AESMCrrTied))
+ return true;
+ // AES decode.
+ if ((FirstOpcode == AArch64::INSTRUCTION_LIST_END ||
+ FirstOpcode == AArch64::AESDrr) &&
+ (SecondOpcode == AArch64::AESIMCrr ||
+ SecondOpcode == AArch64::AESIMCrrTied))
+ return true;
- if (ST.hasFuseLiterals())
- // Fuse literal generation operations.
- switch (SecondOpcode) {
- // PC relative address.
- case AArch64::ADDXri:
- return FirstOpcode == AArch64::ADRP ||
- FirstOpcode == AArch64::INSTRUCTION_LIST_END;
- // 32 bit immediate.
- case AArch64::MOVKWi:
- return (FirstOpcode == AArch64::MOVZWi &&
- SecondMI.getOperand(3).getImm() == 16) ||
- FirstOpcode == AArch64::INSTRUCTION_LIST_END;
- // Lower and upper half of 64 bit immediate.
- case AArch64::MOVKXi:
- return FirstOpcode == AArch64::INSTRUCTION_LIST_END ||
- (FirstOpcode == AArch64::MOVZXi &&
- SecondMI.getOperand(3).getImm() == 16) ||
- (FirstOpcode == AArch64::MOVKXi &&
- FirstMI->getOperand(3).getImm() == 32 &&
- SecondMI.getOperand(3).getImm() == 48);
- }
+ return false;
+}
+
+// Fuse literal generation.
+static bool isLiteralsPair(const MachineInstr *FirstMI,
+ const MachineInstr &SecondMI) {
+ // Assume the 1st instr to be a wildcard if it is unspecified.
+ unsigned FirstOpcode =
+ FirstMI ? FirstMI->getOpcode()
+ : static_cast<unsigned>(AArch64::INSTRUCTION_LIST_END);
+ unsigned SecondOpcode = SecondMI.getOpcode();
+
+ // PC relative address.
+ if ((FirstOpcode == AArch64::INSTRUCTION_LIST_END ||
+ FirstOpcode == AArch64::ADRP) &&
+ SecondOpcode == AArch64::ADDXri)
+ return true;
+ // 32 bit immediate.
+ if ((FirstOpcode == AArch64::INSTRUCTION_LIST_END ||
+ FirstOpcode == AArch64::MOVZWi) &&
+ SecondOpcode == AArch64::MOVKWi && SecondMI.getOperand(3).getImm() == 16)
+ return true;
+ // Lower half of 64 bit immediate.
+ if((FirstOpcode == AArch64::INSTRUCTION_LIST_END ||
+ FirstOpcode == AArch64::MOVZXi) &&
+ SecondOpcode == AArch64::MOVKXi && SecondMI.getOperand(3).getImm() == 16)
+ return true;
+ // Upper half of 64 bit immediate.
+ if ((FirstOpcode == AArch64::INSTRUCTION_LIST_END ||
+ (FirstOpcode == AArch64::MOVKXi &&
+ FirstMI->getOperand(3).getImm() == 32)) &&
+ SecondOpcode == AArch64::MOVKXi && SecondMI.getOperand(3).getImm() == 48)
+ return true;
+
+ return false;
+}
+
+// Fuse address generation and loads or stores.
+static bool isAddressLdStPair(const MachineInstr *FirstMI,
+ const MachineInstr &SecondMI) {
+ unsigned SecondOpcode = SecondMI.getOpcode();
- if (ST.hasFuseAddress()) {
- // Fuse address generation and loads and stores.
- if ((FirstOpcode == AArch64::INSTRUCTION_LIST_END ||
- FirstOpcode == AArch64::ADR ||
- FirstOpcode == AArch64::ADRP) &&
- ((SecondOpcode == AArch64::STRBBui ||
- SecondOpcode == AArch64::STRBui ||
- SecondOpcode == AArch64::STRDui ||
- SecondOpcode == AArch64::STRHHui ||
- SecondOpcode == AArch64::STRHui ||
- SecondOpcode == AArch64::STRQui ||
- SecondOpcode == AArch64::STRSui ||
- SecondOpcode == AArch64::STRWui ||
- SecondOpcode == AArch64::STRXui ||
- SecondOpcode == AArch64::LDRBBui ||
- SecondOpcode == AArch64::LDRBui ||
- SecondOpcode == AArch64::LDRDui ||
- SecondOpcode == AArch64::LDRHHui ||
- SecondOpcode == AArch64::LDRHui ||
- SecondOpcode == AArch64::LDRQui ||
- SecondOpcode == AArch64::LDRSBWui ||
- SecondOpcode == AArch64::LDRSBXui ||
- SecondOpcode == AArch64::LDRSHWui ||
- SecondOpcode == AArch64::LDRSHXui ||
- SecondOpcode == AArch64::LDRSWui ||
- SecondOpcode == AArch64::LDRSui ||
- SecondOpcode == AArch64::LDRWui ||
- SecondOpcode == AArch64::LDRXui) &&
- (FirstOpcode != AArch64::ADR ||
- SecondMI.getOperand(2).getImm() == 0)))
+ switch (SecondOpcode) {
+ case AArch64::STRBBui:
+ case AArch64::STRBui:
+ case AArch64::STRDui:
+ case AArch64::STRHHui:
+ case AArch64::STRHui:
+ case AArch64::STRQui:
+ case AArch64::STRSui:
+ case AArch64::STRWui:
+ case AArch64::STRXui:
+ case AArch64::LDRBBui:
+ case AArch64::LDRBui:
+ case AArch64::LDRDui:
+ case AArch64::LDRHHui:
+ case AArch64::LDRHui:
+ case AArch64::LDRQui:
+ case AArch64::LDRSui:
+ case AArch64::LDRWui:
+ case AArch64::LDRXui:
+ case AArch64::LDRSBWui:
+ case AArch64::LDRSBXui:
+ case AArch64::LDRSHWui:
+ case AArch64::LDRSHXui:
+ case AArch64::LDRSWui:
+ // Assume the 1st instr to be a wildcard if it is unspecified.
+ if (!FirstMI)
return true;
+
+ switch (FirstMI->getOpcode()) {
+ case AArch64::ADR:
+ if (SecondMI.getOperand(2).getImm() == 0)
+ return true;
+ return false;
+ case AArch64::ADRP:
+ return true;
+ }
}
+ return false;
+}
+
+/// \brief Check if the instr pair, FirstMI and SecondMI, should be fused
+/// together. Given SecondMI, when FirstMI is unspecified, then check if
+/// SecondMI may be part of a fused pair at all.
+static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
+ const TargetSubtargetInfo &TSI,
+ const MachineInstr *FirstMI,
+ const MachineInstr &SecondMI) {
+ const AArch64Subtarget &ST = static_cast<const AArch64Subtarget&>(TSI);
+
+ if (ST.hasArithmeticBccFusion() && isArithmeticBccPair(FirstMI, SecondMI))
+ return true;
+ if (ST.hasArithmeticCbzFusion() && isArithmeticCbzPair(FirstMI, SecondMI))
+ return true;
+ if (ST.hasFuseAES() && isAESPair(FirstMI, SecondMI))
+ return true;
+ if (ST.hasFuseLiterals() && isLiteralsPair(FirstMI, SecondMI))
+ return true;
+ if (ST.hasFuseAddress() && isAddressLdStPair(FirstMI, SecondMI))
+ return true;
return false;
}
projects / platform / upstream / llvm.git / commitdiff