RVV intrinsic function has several policy variants.
Include TU, TA, TAMU, TAMA, TUMU, TUMA, MU, MA, TUM, TAM
Currently, the clang side enumerates these policies, but it's hard to add a new policy.
This patch use structure to replace the origin policy enumeration, and enhance some policy transform logic.
This is a clean-up job that will not affect the RVV intrinsic functionality and make sure riscv_vector_builtin_cg.inc is the same as the original one.
Reviewed By: kito-cheng
Differential Revision: https://reviews.llvm.org/D139995
LLVM_MARK_AS_BITMASK_ENUM(LMUL1),
};
-// The Lowerest two bit equal to policy value.
-enum Policy : uint8_t {
- TU = 0, // For unmasked TU, last two bit is TUMU
- TA = 1, // // For unmasked TA, last two bit is TAMU
- TUMA = 2,
- TAMA = 3,
- TUMU = 4,
- TAMU = 5,
- MU = 6, // For masked MU, last two bit is TAMU
- MA = 7, // For masked MA, last two bit is TAMA
- TUM = 10, // For masked MA, last two bit is TUMA
- TAM = 11, // For masked MA, last two bit is TAMA
- PolicyNone,
+struct Policy {
+ bool PolicyNone = false;
+ enum PolicyType {
+ Undisturbed,
+ Agnostic,
+ Omit, // No policy required.
+ };
+ PolicyType TailPolicy = Omit;
+ PolicyType MaskPolicy = Omit;
+ bool IntrinsicWithoutMU = false;
+ Policy() : PolicyNone(true) {}
+ Policy(PolicyType _TailPolicy, PolicyType _MaskPolicy,
+ bool _IntrinsicWithoutMU = false)
+ : TailPolicy(_TailPolicy), MaskPolicy(_MaskPolicy),
+ IntrinsicWithoutMU(_IntrinsicWithoutMU) {}
+
+ bool isTAMAPolicy() const {
+ return TailPolicy == Agnostic && MaskPolicy == Agnostic;
+ }
+
+ bool isTAMUPolicy() const {
+ return TailPolicy == Agnostic && MaskPolicy == Undisturbed;
+ }
+
+ bool isTUMAPolicy() const {
+ return TailPolicy == Undisturbed && MaskPolicy == Agnostic;
+ }
+
+ bool isTUMUPolicy() const {
+ return TailPolicy == Undisturbed && MaskPolicy == Undisturbed;
+ }
+
+ bool isTUMPolicy() const {
+ return TailPolicy == Undisturbed && MaskPolicy == Agnostic &&
+ IntrinsicWithoutMU;
+ }
+
+ bool isTAMPolicy() const {
+ return TailPolicy == Agnostic && MaskPolicy == Agnostic &&
+ IntrinsicWithoutMU;
+ }
+
+ bool isTAPolicy() const {
+ return TailPolicy == Agnostic && MaskPolicy == Omit;
+ }
+
+ bool isTUPolicy() const {
+ return TailPolicy == Undisturbed && MaskPolicy == Omit;
+ }
+
+ bool isMAPolicy() const {
+ return MaskPolicy == Agnostic && TailPolicy == Omit;
+ }
+
+ bool isMUPolicy() const {
+ return MaskPolicy == Undisturbed && TailPolicy == Omit;
+ }
+
+ bool isPolicyNonePolicy() const { return PolicyNone; }
+
+ bool operator==(const Policy &Other) const {
+ return PolicyNone == Other.PolicyNone && TailPolicy == Other.TailPolicy &&
+ MaskPolicy == Other.MaskPolicy &&
+ IntrinsicWithoutMU == Other.IntrinsicWithoutMU;
+ }
+
+ bool operator!=(const Policy &Other) const { return !(*this == Other); }
+
+ bool operator<(const Policy &Other) const {
+ // Just for maintain the old order for quick test.
+ if (MaskPolicy != Other.MaskPolicy)
+ return Other.MaskPolicy < MaskPolicy;
+ return TailPolicy < Other.TailPolicy;
+ }
};
// PrototypeDescriptor is used to compute type info of arguments or return
// InputTypes. -1 means the return type.
std::vector<int64_t> IntrinsicTypes;
unsigned NF = 1;
- Policy DefaultPolicy = Policy::PolicyNone;
+ Policy DefaultPolicy;
public:
RVVIntrinsic(llvm::StringRef Name, llvm::StringRef Suffix,
return IntrinsicTypes;
}
Policy getDefaultPolicy() const {
- assert(DefaultPolicy != Policy::PolicyNone);
+ assert(DefaultPolicy.PolicyNone == false);
return DefaultPolicy;
}
unsigned getDefaultPolicyBits() const {
- assert(DefaultPolicy != Policy::PolicyNone);
- return static_cast<unsigned>(DefaultPolicy) & 3;
+ // Return following value.
+ // constexpr unsigned TAIL_UNDISTURBED = 0;
+ // constexpr unsigned TAIL_AGNOSTIC = 1;
+ // constexpr unsigned TAIL_AGNOSTIC_MASK_AGNOSTIC = 3;
+ // FIXME: how about value 2
+ // int DefaultPolicy = TAIL_UNDISTURBED;
+ assert(DefaultPolicy.PolicyNone == false);
+
+ if (DefaultPolicy.isTUMAPolicy())
+ return 2;
+ if (DefaultPolicy.isTAMAPolicy())
+ return 3;
+ if (DefaultPolicy.isTUMUPolicy())
+ return 0;
+ if (DefaultPolicy.isTAMUPolicy())
+ return 1;
+ if (DefaultPolicy.isTUPolicy())
+ return 0;
+ if (DefaultPolicy.isTAPolicy())
+ return 1;
+
+ llvm_unreachable("unsupport policy");
+ return 0;
}
// Return the type string for a BUILTIN() macro in Builtins.def.
bool IsMasked, bool HasMaskedOffOperand, bool HasVL,
unsigned NF, bool IsPrototypeDefaultTU,
PolicyScheme DefaultScheme,
- Policy DefaultPolicy = Policy::PolicyNone);
+ Policy DefaultPolicy);
static llvm::SmallVector<Policy>
getSupportedMaskedPolicies(bool HasTailPolicy, bool HasMaskPolicy);
RVVIntrinsic::computeBuiltinTypes(
BasicProtoSeq, /*IsMasked=*/false,
/*HasMaskedOffOperand=*/false, Record.HasVL, Record.NF,
- Record.IsPrototypeDefaultTU, UnMaskedPolicyScheme);
+ Record.IsPrototypeDefaultTU, UnMaskedPolicyScheme, Policy());
llvm::SmallVector<PrototypeDescriptor> ProtoMaskSeq =
RVVIntrinsic::computeBuiltinTypes(
BasicProtoSeq, /*IsMasked=*/true, Record.HasMaskedOffOperand,
Record.HasVL, Record.NF, Record.IsPrototypeDefaultTU,
- MaskedPolicyScheme);
+ MaskedPolicyScheme, Policy());
bool UnMaskedHasPolicy = UnMaskedPolicyScheme != PolicyScheme::SchemeNone;
bool MaskedHasPolicy = MaskedPolicyScheme != PolicyScheme::SchemeNone;
// If unmasked builtin supports policy, they should be TU or TA.
- llvm::SmallVector<Policy> SupportedUnMaskedPolicies = {Policy::TU,
- Policy::TA};
+ llvm::SmallVector<Policy> SupportedUnMaskedPolicies;
+ SupportedUnMaskedPolicies.emplace_back(Policy(
+ Policy::PolicyType::Undisturbed, Policy::PolicyType::Omit)); // TU
+ SupportedUnMaskedPolicies.emplace_back(
+ Policy(Policy::PolicyType::Agnostic, Policy::PolicyType::Omit)); // TA
llvm::SmallVector<Policy> SupportedMaskedPolicies =
RVVIntrinsic::getSupportedMaskedPolicies(Record.HasTailPolicy,
Record.HasMaskPolicy);
// Create non-masked intrinsic.
InitRVVIntrinsic(Record, SuffixStr, OverloadedSuffixStr, false, *Types,
- UnMaskedHasPolicy, Policy::PolicyNone,
+ UnMaskedHasPolicy, Policy(),
Record.IsPrototypeDefaultTU);
// Create non-masked policy intrinsic.
Optional<RVVTypes> MaskTypes =
TypeCache.computeTypes(BaseType, Log2LMUL, Record.NF, ProtoMaskSeq);
InitRVVIntrinsic(Record, SuffixStr, OverloadedSuffixStr, true,
- *MaskTypes, MaskedHasPolicy, Policy::PolicyNone,
+ *MaskTypes, MaskedHasPolicy, Policy(),
Record.IsPrototypeDefaultTU);
if (Record.MaskedPolicyScheme == PolicyScheme::SchemeNone)
continue;
SmallVector<PrototypeDescriptor> NewPrototype(Prototype.begin(),
Prototype.end());
// Update DefaultPolicy if need (TA or TAMA) for compute builtin types.
- switch (DefaultPolicy) {
- case Policy::MA:
- DefaultPolicy = Policy::TAMA;
- break;
- case Policy::TAM:
- DefaultPolicy = Policy::TAMA;
- break;
- case Policy::PolicyNone:
- // Masked with no policy would not be TAMA.
+ if (DefaultPolicy.isMAPolicy())
+ DefaultPolicy.TailPolicy = Policy::PolicyType::Agnostic; // TAMA
+ if (DefaultPolicy.isPolicyNonePolicy()) {
if (!IsMasked) {
+ DefaultPolicy.PolicyNone = false;
if (IsPrototypeDefaultTU)
- DefaultPolicy = Policy::TU;
+ DefaultPolicy.TailPolicy = Policy::PolicyType::Undisturbed; // TU
else
- DefaultPolicy = Policy::TA;
+ DefaultPolicy.TailPolicy = Policy::PolicyType::Agnostic; // TA
}
- break;
- default:
- break;
}
bool HasPassthruOp = DefaultScheme == PolicyScheme::HasPassthruOperand;
if (IsMasked) {
// If HasMaskedOffOperand, insert result type as first input operand if
// need.
- if (HasMaskedOffOperand && DefaultPolicy != Policy::TAMA) {
+ if (HasMaskedOffOperand && !DefaultPolicy.isTAMAPolicy()) {
if (NF == 1) {
NewPrototype.insert(NewPrototype.begin() + 1, NewPrototype[0]);
} else if (NF > 1) {
}
}
// Erase passthru operand for TAM
- if (NF == 1 && IsPrototypeDefaultTU && DefaultPolicy == Policy::TAMA &&
+ if (NF == 1 && IsPrototypeDefaultTU && DefaultPolicy.isTAMAPolicy() &&
HasPassthruOp && !HasMaskedOffOperand)
NewPrototype.erase(NewPrototype.begin() + 1);
if (HasMaskedOffOperand && NF > 1) {
}
} else {
if (NF == 1) {
- if (DefaultPolicy == Policy::TU && HasPassthruOp && !IsPrototypeDefaultTU)
+ if (DefaultPolicy.isTUPolicy() && HasPassthruOp && !IsPrototypeDefaultTU)
NewPrototype.insert(NewPrototype.begin(), NewPrototype[0]);
- else if (DefaultPolicy == Policy::TA && HasPassthruOp &&
+ else if (DefaultPolicy.isTAPolicy() && HasPassthruOp &&
IsPrototypeDefaultTU)
NewPrototype.erase(NewPrototype.begin() + 1);
if (DefaultScheme == PolicyScheme::HasPassthruOperandAtIdx1) {
- if (DefaultPolicy == Policy::TU && !IsPrototypeDefaultTU) {
+ if (DefaultPolicy.isTUPolicy() && !IsPrototypeDefaultTU) {
// Insert undisturbed output to index 1
NewPrototype.insert(NewPrototype.begin() + 2, NewPrototype[0]);
- } else if (DefaultPolicy == Policy::TA && IsPrototypeDefaultTU) {
+ } else if (DefaultPolicy.isTAPolicy() && IsPrototypeDefaultTU) {
// Erase passthru for TA policy
NewPrototype.erase(NewPrototype.begin() + 2);
}
}
- } else if (DefaultPolicy == Policy::TU && HasPassthruOp) {
+ } else if (DefaultPolicy.isTUPolicy() && HasPassthruOp) {
// NF > 1 cases for segment load operations.
// Convert
// (void, op0 address, op1 address, ...)
RVVIntrinsic::getSupportedMaskedPolicies(bool HasTailPolicy,
bool HasMaskPolicy) {
if (HasTailPolicy && HasMaskPolicy)
- return {Policy::TUMA, Policy::TAMA, Policy::TUMU, Policy::TAMU};
- else if (HasTailPolicy)
- return {Policy::TUM, Policy::TAM};
- return {Policy::MA, Policy::MU};
+ return {Policy(Policy::PolicyType::Undisturbed,
+ Policy::PolicyType::Agnostic), // TUMA
+ Policy(Policy::PolicyType::Agnostic,
+ Policy::PolicyType::Agnostic), // TAMA
+ Policy(Policy::PolicyType::Undisturbed,
+ Policy::PolicyType::Undisturbed), // TUMU
+ Policy(Policy::PolicyType::Agnostic,
+ Policy::PolicyType::Undisturbed)}; // TAMU
+
+ if (HasTailPolicy)
+ return {Policy(Policy::PolicyType::Undisturbed,
+ Policy::PolicyType::Agnostic, true), // TUM
+ Policy(Policy::PolicyType::Agnostic, Policy::PolicyType::Agnostic,
+ true)}; // TAM
+
+ return {
+ Policy(Policy::PolicyType::Omit, Policy::PolicyType::Agnostic), // MA
+ Policy(Policy::PolicyType::Omit, Policy::PolicyType::Undisturbed)}; // MU
}
void RVVIntrinsic::updateNamesAndPolicy(bool IsMasked, bool HasPolicy,
OverloadedName += suffix;
};
- switch (DefaultPolicy) {
- case Policy::TU:
- appendPolicySuffix("_tu");
- break;
- case Policy::TA:
- appendPolicySuffix("_ta");
- break;
- case Policy::MU:
- appendPolicySuffix("_mu");
- DefaultPolicy = Policy::TAMU;
- break;
- case Policy::MA:
- appendPolicySuffix("_ma");
- DefaultPolicy = Policy::TAMA;
- break;
- case Policy::TUM:
- appendPolicySuffix("_tum");
- DefaultPolicy = Policy::TUMA;
- break;
- case Policy::TAM:
- appendPolicySuffix("_tam");
- DefaultPolicy = Policy::TAMA;
- break;
- case Policy::TUMU:
- appendPolicySuffix("_tumu");
- break;
- case Policy::TAMU:
- appendPolicySuffix("_tamu");
- break;
- case Policy::TUMA:
- appendPolicySuffix("_tuma");
- break;
- case Policy::TAMA:
- appendPolicySuffix("_tama");
- break;
- default:
+ if (DefaultPolicy.isPolicyNonePolicy()) {
+ DefaultPolicy.PolicyNone = false;
if (IsMasked) {
Name += "_m";
// FIXME: Currently _m default policy implementation is different with
// RVV intrinsic spec (TUMA)
- DefaultPolicy = Policy::TUMU;
+ DefaultPolicy.TailPolicy = Policy::PolicyType::Undisturbed;
+ DefaultPolicy.MaskPolicy = Policy::PolicyType::Undisturbed;
if (HasPolicy)
BuiltinName += "_tumu";
else
BuiltinName += "_m";
} else if (IsPrototypeDefaultTU) {
- DefaultPolicy = Policy::TU;
+ DefaultPolicy.TailPolicy = Policy::PolicyType::Undisturbed;
if (HasPolicy)
BuiltinName += "_tu";
} else {
- DefaultPolicy = Policy::TA;
+ DefaultPolicy.TailPolicy = Policy::PolicyType::Agnostic;
if (HasPolicy)
BuiltinName += "_ta";
}
+ } else {
+ if (DefaultPolicy.isTUMPolicy())
+ appendPolicySuffix("_tum");
+ else if (DefaultPolicy.isTAMPolicy())
+ appendPolicySuffix("_tam");
+ else if (DefaultPolicy.isTUMUPolicy())
+ appendPolicySuffix("_tumu");
+ else if (DefaultPolicy.isTAMUPolicy())
+ appendPolicySuffix("_tamu");
+ else if (DefaultPolicy.isTUMAPolicy())
+ appendPolicySuffix("_tuma");
+ else if (DefaultPolicy.isTAMAPolicy())
+ appendPolicySuffix("_tama");
+ else if (DefaultPolicy.isTUPolicy())
+ appendPolicySuffix("_tu");
+ else if (DefaultPolicy.isTAPolicy())
+ appendPolicySuffix("_ta");
+ else if (DefaultPolicy.isMUPolicy()) {
+ appendPolicySuffix("_mu");
+ DefaultPolicy.TailPolicy = Policy::PolicyType::Agnostic;
+ } else if (DefaultPolicy.isMAPolicy()) {
+ appendPolicySuffix("_ma");
+ DefaultPolicy.TailPolicy = Policy::PolicyType::Agnostic;
+ }
}
}
if (RVVI->getNF() >= 2)
OS << " NF = " + utostr(RVVI->getNF()) + ";\n";
// We had initialized DefaultPolicy as TU/TUMU in CodeGen function.
- if (RVVI->getDefaultPolicy() != Policy::TU &&
- RVVI->getDefaultPolicy() != Policy::TUMU && !RVVI->hasPassthruOperand() &&
+ if (!RVVI->getDefaultPolicy().isTUPolicy() &&
+ !RVVI->getDefaultPolicy().isTUMUPolicy() && !RVVI->hasPassthruOperand() &&
!RVVI->hasManualCodegen() && RVVI->hasVL())
OS << " DefaultPolicy = " << RVVI->getDefaultPolicyBits() << ";\n";
OS << " Ops.push_back(ConstantInt::get(Ops.back()->getType(),"
" DefaultPolicy));\n";
if (RVVI->hasMaskedOffOperand() &&
- RVVI->getDefaultPolicy() == Policy::TAMA)
+ RVVI->getDefaultPolicy().isTAMAPolicy())
OS << " Ops.insert(Ops.begin(), llvm::UndefValue::get(ResultType));\n";
// Masked reduction cases.
if (!RVVI->hasMaskedOffOperand() && RVVI->hasPassthruOperand() &&
- RVVI->getDefaultPolicy() == Policy::TAMA)
+ RVVI->getDefaultPolicy().isTAMAPolicy())
OS << " Ops.insert(Ops.begin(), llvm::UndefValue::get(ResultType));\n";
} else {
OS << " std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());\n";
OS << " Ops.push_back(ConstantInt::get(Ops.back()->getType(), "
"DefaultPolicy));\n";
else if (RVVI->hasPassthruOperand() &&
- RVVI->getDefaultPolicy() == Policy::TA)
+ RVVI->getDefaultPolicy().isTAPolicy())
OS << " Ops.insert(Ops.begin(), llvm::UndefValue::get(ResultType));\n";
}
unsigned NF = R->getValueAsInt("NF");
// If unmasked builtin supports policy, they should be TU or TA.
- SmallVector<Policy> SupportedUnMaskedPolicies = {Policy::TU, Policy::TA};
+ llvm::SmallVector<Policy> SupportedUnMaskedPolicies;
+ SupportedUnMaskedPolicies.emplace_back(Policy(
+ Policy::PolicyType::Undisturbed, Policy::PolicyType::Omit)); // TU
+ SupportedUnMaskedPolicies.emplace_back(
+ Policy(Policy::PolicyType::Agnostic, Policy::PolicyType::Omit)); // TA
SmallVector<Policy> SupportedMaskedPolicies =
RVVIntrinsic::getSupportedMaskedPolicies(HasTailPolicy, HasMaskPolicy);
auto Prototype = RVVIntrinsic::computeBuiltinTypes(
BasicPrototype, /*IsMasked=*/false,
/*HasMaskedOffOperand=*/false, HasVL, NF, IsPrototypeDefaultTU,
- UnMaskedPolicyScheme);
+ UnMaskedPolicyScheme, Policy());
auto MaskedPrototype = RVVIntrinsic::computeBuiltinTypes(
BasicPrototype, /*IsMasked=*/true, HasMaskedOffOperand, HasVL, NF,
- IsPrototypeDefaultTU, MaskedPolicyScheme);
+ IsPrototypeDefaultTU, MaskedPolicyScheme, Policy());
// Create Intrinsics for each type and LMUL.
for (char I : TypeRange) {
/*IsMasked=*/false, /*HasMaskedOffOperand=*/false, HasVL,
UnMaskedPolicyScheme, SupportOverloading, HasBuiltinAlias,
ManualCodegen, *Types, IntrinsicTypes, RequiredFeatures, NF,
- Policy::PolicyNone, IsPrototypeDefaultTU));
+ Policy(), IsPrototypeDefaultTU));
if (UnMaskedPolicyScheme != PolicyScheme::SchemeNone)
for (auto P : SupportedUnMaskedPolicies) {
SmallVector<PrototypeDescriptor> PolicyPrototype =
/*IsMasked=*/true, HasMaskedOffOperand, HasVL, MaskedPolicyScheme,
SupportOverloading, HasBuiltinAlias, MaskedManualCodegen,
*MaskTypes, IntrinsicTypes, RequiredFeatures, NF,
- Policy::PolicyNone, IsPrototypeDefaultTU));
+ Policy(), IsPrototypeDefaultTU));
if (MaskedPolicyScheme == PolicyScheme::SchemeNone)
continue;
for (auto P : SupportedMaskedPolicies) {