From 33bb3efbb34b9ca41a5ef3d2fe3687d9d7390a2a Mon Sep 17 00:00:00 2001 From: Krzysztof Parzyszek Date: Thu, 8 Oct 2020 17:12:32 -0500 Subject: [PATCH] [Hexagon] Generalize handling of SDNodes created during ISel The selection of HVX shuffles can produce more nodes in the DAG, which need special handling, or otherwise they would be left unselected by the main selection code. Make the handling of such nodes more general. --- llvm/lib/Target/Hexagon/HexagonISelDAGToDAGHVX.cpp | 171 ++++++++++----------- llvm/lib/Target/Hexagon/HexagonISelLowering.cpp | 1 + llvm/lib/Target/Hexagon/HexagonISelLowering.h | 3 + 3 files changed, 88 insertions(+), 87 deletions(-) diff --git a/llvm/lib/Target/Hexagon/HexagonISelDAGToDAGHVX.cpp b/llvm/lib/Target/Hexagon/HexagonISelDAGToDAGHVX.cpp index f422057..508eb42 100644 --- a/llvm/lib/Target/Hexagon/HexagonISelDAGToDAGHVX.cpp +++ b/llvm/lib/Target/Hexagon/HexagonISelDAGToDAGHVX.cpp @@ -828,6 +828,7 @@ namespace llvm { void selectVAlign(SDNode *N); private: + void select(SDNode *ISelN); void materialize(const ResultStack &Results); SDValue getVectorConstant(ArrayRef Data, const SDLoc &dl); @@ -931,46 +932,19 @@ bool HvxSelector::selectVectorConstants(SDNode *N) { SmallVector Nodes; SetVector WorkQ; - // The one-use test for VSPLATW's operand may fail due to dead nodes - // left over in the DAG. - DAG.RemoveDeadNodes(); - // The DAG can change (due to CSE) during selection, so cache all the // unselected nodes first to avoid traversing a mutating DAG. - - auto IsNodeToSelect = [] (SDNode *N) { - if (N->isMachineOpcode()) - return false; - switch (N->getOpcode()) { - case HexagonISD::VZERO: - case HexagonISD::VSPLATW: - return true; - case ISD::LOAD: { - SDValue Addr = cast(N)->getBasePtr(); - unsigned AddrOpc = Addr.getOpcode(); - if (AddrOpc == HexagonISD::AT_PCREL || AddrOpc == HexagonISD::CP) - if (Addr.getOperand(0).getOpcode() == ISD::TargetConstantPool) - return true; - } - break; - } - // Make sure to select the operand of VSPLATW. - bool IsSplatOp = N->hasOneUse() && - N->use_begin()->getOpcode() == HexagonISD::VSPLATW; - return IsSplatOp; - }; - WorkQ.insert(N); for (unsigned i = 0; i != WorkQ.size(); ++i) { SDNode *W = WorkQ[i]; - if (IsNodeToSelect(W)) + if (!W->isMachineOpcode() && W->getOpcode() == HexagonISD::ISEL) Nodes.push_back(W); for (unsigned j = 0, f = W->getNumOperands(); j != f; ++j) WorkQ.insert(W->getOperand(j).getNode()); } for (SDNode *L : Nodes) - ISel.Select(L); + select(L); return !Nodes.empty(); } @@ -1358,6 +1332,82 @@ namespace { }; } +void HvxSelector::select(SDNode *ISelN) { + // What's important here is to select the right set of nodes. The main + // selection algorithm loops over nodes in a topological order, i.e. users + // are visited before their operands. + // + // It is an error to have an unselected node with a selected operand, and + // there is an assertion in the main selector code to enforce that. + // + // Such a situation could occur if we selected a node, which is both a + // subnode of ISelN, and a subnode of an unrelated (and yet unselected) + // node in the DAG. + assert(ISelN->getOpcode() == HexagonISD::ISEL); + SDNode *N0 = ISelN->getOperand(0).getNode(); + if (N0->isMachineOpcode()) { + ISel.ReplaceNode(ISelN, N0); + return; + } + + // There could have been nodes created (i.e. inserted into the DAG) + // that are now dead. Remove them, in case they use any of the nodes + // to select (and make them look shared). + DAG.RemoveDeadNodes(); + + SetVector SubNodes, TmpQ; + std::map NumOps; + + // Don't want to select N0 if it's shared with another node, except if + // it's shared with other ISELs. + auto IsISelN = [](SDNode *T) { return T->getOpcode() == HexagonISD::ISEL; }; + if (llvm::all_of(N0->uses(), IsISelN)) + SubNodes.insert(N0); + + auto InSubNodes = [&SubNodes](SDNode *T) { return SubNodes.count(T); }; + for (unsigned I = 0; I != SubNodes.size(); ++I) { + SDNode *S = SubNodes[I]; + unsigned OpN = 0; + // Only add subnodes that are only reachable from N0. + for (SDValue Op : S->ops()) { + SDNode *O = Op.getNode(); + if (llvm::all_of(O->uses(), InSubNodes)) { + SubNodes.insert(O); + ++OpN; + } + } + NumOps.insert({S, OpN}); + if (OpN == 0) + TmpQ.insert(S); + } + + for (unsigned I = 0; I != TmpQ.size(); ++I) { + SDNode *S = TmpQ[I]; + for (SDNode *U : S->uses()) { + if (U == ISelN) + continue; + auto F = NumOps.find(U); + assert(F != NumOps.end()); + if (F->second > 0 && !--F->second) + TmpQ.insert(F->first); + } + } + + // Remove the marker. + ISel.ReplaceNode(ISelN, N0); + + assert(SubNodes.size() == TmpQ.size()); + NullifyingVector Queue(TmpQ.takeVector()); + + Deleter DUQ(DAG, Queue); + for (SDNode *S : reverse(Queue)) { + if (S == nullptr) + continue; + DEBUG_WITH_TYPE("isel", {dbgs() << "HVX selecting: "; S->dump(&DAG);}); + ISel.Select(S); + } +} + bool HvxSelector::scalarizeShuffle(ArrayRef Mask, const SDLoc &dl, MVT ResTy, SDValue Va, SDValue Vb, SDNode *N) { @@ -1379,12 +1429,7 @@ bool HvxSelector::scalarizeShuffle(ArrayRef Mask, const SDLoc &dl, // nodes, these nodes would not be selected (since the "local" selection // only visits nodes that are not in AllNodes). // To avoid this issue, remove all dead nodes from the DAG now. - DAG.RemoveDeadNodes(); - DenseSet AllNodes; - for (SDNode &S : DAG.allnodes()) - AllNodes.insert(&S); - - Deleter DUA(DAG, AllNodes); +// DAG.RemoveDeadNodes(); SmallVector Ops; LLVMContext &Ctx = *DAG.getContext(); @@ -1434,57 +1479,9 @@ bool HvxSelector::scalarizeShuffle(ArrayRef Mask, const SDLoc &dl, } assert(!N->use_empty()); - ISel.ReplaceNode(N, LV.getNode()); - - if (AllNodes.count(LV.getNode())) { - DAG.RemoveDeadNodes(); - return true; - } - - // The lowered build-vector node will now need to be selected. It needs - // to be done here because this node and its submodes are not included - // in the main selection loop. - // Implement essentially the same topological ordering algorithm as is - // used in SelectionDAGISel. - - SetVector SubNodes, TmpQ; - std::map NumOps; - - SubNodes.insert(LV.getNode()); - for (unsigned I = 0; I != SubNodes.size(); ++I) { - unsigned OpN = 0; - SDNode *S = SubNodes[I]; - for (SDValue Op : S->ops()) { - if (AllNodes.count(Op.getNode())) - continue; - SubNodes.insert(Op.getNode()); - ++OpN; - } - NumOps.insert({S, OpN}); - if (OpN == 0) - TmpQ.insert(S); - } - - for (unsigned I = 0; I != TmpQ.size(); ++I) { - SDNode *S = TmpQ[I]; - for (SDNode *U : S->uses()) { - if (!SubNodes.count(U)) - continue; - auto F = NumOps.find(U); - assert(F != NumOps.end()); - assert(F->second > 0); - if (!--F->second) - TmpQ.insert(F->first); - } - } - assert(SubNodes.size() == TmpQ.size()); - NullifyingVector Queue(TmpQ.takeVector()); - - Deleter DUQ(DAG, Queue); - for (SDNode *S : reverse(Queue)) - if (S != nullptr) - ISel.Select(S); - + SDValue IS = DAG.getNode(HexagonISD::ISEL, dl, ResTy, LV); + ISel.ReplaceNode(N, IS.getNode()); + select(IS.getNode()); DAG.RemoveDeadNodes(); return true; } @@ -2014,7 +2011,7 @@ SDValue HvxSelector::getVectorConstant(ArrayRef Data, SDValue BV = DAG.getBuildVector(VecTy, dl, Elems); SDValue LV = Lower.LowerOperation(BV, DAG); DAG.RemoveDeadNode(BV.getNode()); - return LV; + return DAG.getNode(HexagonISD::ISEL, dl, VecTy, LV); } void HvxSelector::selectShuffle(SDNode *N) { diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp index 20e5e5a..0e443c6 100644 --- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp +++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp @@ -1865,6 +1865,7 @@ const char* HexagonTargetLowering::getTargetNodeName(unsigned Opcode) const { case HexagonISD::VPACKL: return "HexagonISD::VPACKL"; case HexagonISD::VUNPACK: return "HexagonISD::VUNPACK"; case HexagonISD::VUNPACKU: return "HexagonISD::VUNPACKU"; + case HexagonISD::ISEL: return "HexagonISD::ISEL"; case HexagonISD::OP_END: break; } return nullptr; diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.h b/llvm/lib/Target/Hexagon/HexagonISelLowering.h index cc34a4c..f41efd5 100644 --- a/llvm/lib/Target/Hexagon/HexagonISelLowering.h +++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.h @@ -96,6 +96,9 @@ enum NodeType : unsigned { // unspecified. VUNPACK, // Unpacking into low elements with sign extension. VUNPACKU, // Unpacking into low elements with zero extension. + ISEL, // Marker for nodes that were created during ISel, and + // which need explicit selection (would have been left + // unselected otherwise). OP_END }; -- 2.7.4