}
SDValue VectorLegalizer::ExpandStore(SDValue Op) {
- SDLoc dl(Op);
StoreSDNode *ST = cast<StoreSDNode>(Op.getNode());
- SDValue Chain = ST->getChain();
- SDValue BasePTR = ST->getBasePtr();
- SDValue Value = ST->getValue();
- EVT StVT = ST->getMemoryVT();
-
- unsigned Alignment = ST->getAlignment();
- bool isVolatile = ST->isVolatile();
- bool isNonTemporal = ST->isNonTemporal();
- AAMDNodes AAInfo = ST->getAAInfo();
- unsigned NumElem = StVT.getVectorNumElements();
- // The type of the data we want to save
- EVT RegVT = Value.getValueType();
- EVT RegSclVT = RegVT.getScalarType();
- // The type of data as saved in memory.
+ EVT StVT = ST->getMemoryVT();
EVT MemSclVT = StVT.getScalarType();
-
- // Cast floats into integers
unsigned ScalarSize = MemSclVT.getSizeInBits();
// Round odd types to the next pow of two.
- if (!isPowerOf2_32(ScalarSize))
- ScalarSize = NextPowerOf2(ScalarSize);
-
- // Store Stride in bytes
- unsigned Stride = ScalarSize/8;
- // Extract each of the elements from the original vector
- // and save them into memory individually.
- SmallVector<SDValue, 8> Stores;
- for (unsigned Idx = 0; Idx < NumElem; Idx++) {
- SDValue Ex = DAG.getNode(
- ISD::EXTRACT_VECTOR_ELT, dl, RegSclVT, Value,
- DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
-
- // This scalar TruncStore may be illegal, but we legalize it later.
- SDValue Store = DAG.getTruncStore(Chain, dl, Ex, BasePTR,
- ST->getPointerInfo().getWithOffset(Idx*Stride), MemSclVT,
- isVolatile, isNonTemporal, MinAlign(Alignment, Idx*Stride),
- AAInfo);
-
- BasePTR = DAG.getNode(ISD::ADD, dl, BasePTR.getValueType(), BasePTR,
- DAG.getConstant(Stride, dl, BasePTR.getValueType()));
-
- Stores.push_back(Store);
+ if (!isPowerOf2_32(ScalarSize)) {
+ // FIXME: This is completely broken and inconsistent with ExpandLoad
+ // handling.
+
+ // For sub-byte element sizes, this ends up with 0 stride between elements,
+ // so the same element just gets re-written to the same location. There seem
+ // to be tests explicitly testing for this broken behavior though. tests
+ // for this broken behavior.
+
+ LLVMContext &Ctx = *DAG.getContext();
+
+ EVT NewMemVT
+ = EVT::getVectorVT(Ctx,
+ MemSclVT.getIntegerVT(Ctx, NextPowerOf2(ScalarSize)),
+ StVT.getVectorNumElements());
+
+ SDValue NewVectorStore
+ = DAG.getTruncStore(ST->getChain(), SDLoc(Op), ST->getValue(),
+ ST->getBasePtr(),
+ ST->getPointerInfo(), NewMemVT,
+ ST->isVolatile(), ST->isNonTemporal(),
+ ST->getAlignment(),
+ ST->getAAInfo());
+ ST = cast<StoreSDNode>(NewVectorStore.getNode());
}
- SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
+
+ SDValue TF = TLI.scalarizeVectorStore(ST, DAG);
AddLegalizedOperand(Op, TF);
return TF;
}
return DAG.getMergeValues({ Value, NewChain }, SL);
}
+// FIXME: This relies on each element having a byte size, otherwise the stride
+// is 0 and just overwrites the same location. ExpandStore currently expects
+// this broken behavior.
+SDValue TargetLowering::scalarizeVectorStore(StoreSDNode *ST,
+ SelectionDAG &DAG) const {
+ SDLoc SL(ST);
+
+ SDValue Chain = ST->getChain();
+ SDValue BasePtr = ST->getBasePtr();
+ SDValue Value = ST->getValue();
+ EVT StVT = ST->getMemoryVT();
+
+ unsigned Alignment = ST->getAlignment();
+ bool isVolatile = ST->isVolatile();
+ bool isNonTemporal = ST->isNonTemporal();
+ AAMDNodes AAInfo = ST->getAAInfo();
+
+ // The type of the data we want to save
+ EVT RegVT = Value.getValueType();
+ EVT RegSclVT = RegVT.getScalarType();
+
+ // The type of data as saved in memory.
+ EVT MemSclVT = StVT.getScalarType();
+
+ EVT PtrVT = BasePtr.getValueType();
+
+ // Store Stride in bytes
+ unsigned Stride = MemSclVT.getSizeInBits() / 8;
+ EVT IdxVT = getVectorIdxTy(DAG.getDataLayout());
+ unsigned NumElem = StVT.getVectorNumElements();
+
+ // Extract each of the elements from the original vector and save them into
+ // memory individually.
+ SmallVector<SDValue, 8> Stores;
+ for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
+ SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value,
+ DAG.getConstant(Idx, SL, IdxVT));
+
+ SDValue Ptr = DAG.getNode(ISD::ADD, SL, PtrVT, BasePtr,
+ DAG.getConstant(Idx * Stride, SL, PtrVT));
+
+ // This scalar TruncStore may be illegal, but we legalize it later.
+ SDValue Store = DAG.getTruncStore(
+ Chain, SL, Elt, Ptr,
+ ST->getPointerInfo().getWithOffset(Idx * Stride), MemSclVT,
+ isVolatile, isNonTemporal, MinAlign(Alignment, Idx * Stride),
+ AAInfo);
+
+ Stores.push_back(Store);
+ }
+
+ return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, Stores);
+}
+
//===----------------------------------------------------------------------===//
// Implementation of Emulated TLS Model
//===----------------------------------------------------------------------===//