}
static StringRef getMapAttrName() { return "map"; }
+
+ // TODO: Remove once prefixing is flipped.
+ operand_range getIndices() { return indices(); }
}];
}
}
static StringRef getMapAttrName() { return "map"; }
+
+ // TODO: Remove once prefixing is flipped.
+ operand_range getIndices() { return indices(); }
}];
}
LogicalResult reifyResultShapes(OpBuilder &b,
ReifiedRankedShapedTypeDims &reifiedReturnShapes);
+ // TODO: Remove once prefixing is flipped.
+ ArrayAttr getIteratorTypes() { return iterator_types(); }
+
//========================================================================//
// Helper functions to mutate the `operand_segment_sizes` attribute.
// These are useful when cloning and changing operand types.
StructuredGenerator(OpBuilder &builder, StructuredOpInterface op)
: builder(builder), ctx(op.getContext()), loc(op.getLoc()),
- iterators(op.iterator_types()), maps(op.getIndexingMaps()), op(op) {}
+ iterators(op.getIteratorTypes()), maps(op.getIndexingMaps()), op(op) {}
bool iters(ArrayRef<IteratorType> its) {
if (its.size() != iterators.size())
];
let extraClassDeclaration = [{
VectorType getLhsType() {
- return lhs().getType().cast<VectorType>();
+ return getLhs().getType().cast<VectorType>();
}
VectorType getRhsType() {
- return rhs().getType().cast<VectorType>();
+ return getRhs().getType().cast<VectorType>();
}
- Type getAccType() { return acc().getType(); }
+ Type getAccType() { return getAcc().getType(); }
VectorType getLHSVectorMaskType() {
- if (llvm::size(masks()) != 2) return VectorType();
+ if (llvm::size(getMasks()) != 2) return VectorType();
return getOperand(3).getType().cast<VectorType>();
}
VectorType getRHSVectorMaskType() {
- if (llvm::size(masks()) != 2) return VectorType();
+ if (llvm::size(getMasks()) != 2) return VectorType();
return getOperand(4).getType().cast<VectorType>();
}
Type getResultType() { return getResult().getType(); }
}];
let extraClassDeclaration = [{
VectorType getVectorType() {
- return vector().getType().cast<VectorType>();
+ return getVector().getType().cast<VectorType>();
}
}];
let builders = [
static StringRef getReductionDimsAttrStrName() { return "reduction_dims"; }
VectorType getSourceVectorType() {
- return source().getType().cast<VectorType>();
+ return getSource().getType().cast<VectorType>();
}
Type getDestType() {
- return dest().getType();
+ return getDest().getType();
}
bool isReducedDim(int64_t d) {
SmallVector<bool> getReductionMask() {
SmallVector<bool> res(getSourceVectorType().getRank(), false);
- for (auto ia : reduction_dims().getAsRange<IntegerAttr>())
+ for (auto ia : getReductionDims().getAsRange<IntegerAttr>())
res[ia.getInt()] = true;
return res;
}
```
}];
let extraClassDeclaration = [{
- Type getSourceType() { return source().getType(); }
+ Type getSourceType() { return getSource().getType(); }
VectorType getVectorType() {
- return vector().getType().cast<VectorType>();
+ return getVector().getType().cast<VectorType>();
}
}];
let assemblyFormat = "$source attr-dict `:` type($source) `to` type($vector)";
let extraClassDeclaration = [{
static StringRef getMaskAttrStrName() { return "mask"; }
VectorType getV1VectorType() {
- return v1().getType().cast<VectorType>();
+ return getV1().getType().cast<VectorType>();
}
VectorType getV2VectorType() {
- return v2().getType().cast<VectorType>();
+ return getV2().getType().cast<VectorType>();
}
VectorType getVectorType() {
- return vector().getType().cast<VectorType>();
+ return getVector().getType().cast<VectorType>();
}
}];
let assemblyFormat = "operands $mask attr-dict `:` type(operands)";
];
let extraClassDeclaration = [{
VectorType getVectorType() {
- return vector().getType().cast<VectorType>();
+ return getVector().getType().cast<VectorType>();
}
}];
let hasVerifier = 1;
let extraClassDeclaration = [{
static StringRef getPositionAttrStrName() { return "position"; }
VectorType getVectorType() {
- return vector().getType().cast<VectorType>();
+ return getVector().getType().cast<VectorType>();
}
static bool isCompatibleReturnTypes(TypeRange l, TypeRange r);
}];
"AffineMap":$map)>];
let extraClassDeclaration = [{
VectorType getSourceVectorType() {
- return vector().getType().cast<VectorType>();
+ return getVector().getType().cast<VectorType>();
}
VectorType getResultType() {
return getResult().getType().cast<VectorType>();
}];
let assemblyFormat = "$lhs `,` $rhs `,` $acc attr-dict `:` type($lhs)";
let extraClassDeclaration = [{
- VectorType getVectorType() { return lhs().getType().cast<VectorType>(); }
+ VectorType getVectorType() { return getLhs().getType().cast<VectorType>(); }
}];
}
OpBuilder<(ins "Value":$source, "Value":$dest)>,
];
let extraClassDeclaration = [{
- Type getSourceType() { return source().getType(); }
+ Type getSourceType() { return getSource().getType(); }
VectorType getDestVectorType() {
- return dest().getType().cast<VectorType>();
+ return getDest().getType().cast<VectorType>();
}
}];
let hasVerifier = 1;
];
let extraClassDeclaration = [{
static StringRef getPositionAttrStrName() { return "position"; }
- Type getSourceType() { return source().getType(); }
+ Type getSourceType() { return getSource().getType(); }
VectorType getDestVectorType() {
- return dest().getType().cast<VectorType>();
+ return getDest().getType().cast<VectorType>();
}
}];
}];
let extraClassDeclaration = [{
VectorType getSourceVectorType() {
- return vector().getType().cast<VectorType>();
+ return getVector().getType().cast<VectorType>();
}
VectorType getResultType() {
return getResult().getType().cast<VectorType>();
static StringRef getOffsetsAttrStrName() { return "offsets"; }
static StringRef getStridesAttrStrName() { return "strides"; }
VectorType getSourceVectorType() {
- return source().getType().cast<VectorType>();
+ return getSource().getType().cast<VectorType>();
}
VectorType getDestVectorType() {
- return dest().getType().cast<VectorType>();
+ return getDest().getType().cast<VectorType>();
}
bool hasNonUnitStrides() {
- return llvm::any_of(strides(), [](Attribute attr) {
+ return llvm::any_of(getStrides(), [](Attribute attr) {
return attr.cast<IntegerAttr>().getInt() != 1;
});
}
];
let extraClassDeclaration = [{
VectorType getOperandVectorTypeLHS() {
- return lhs().getType().cast<VectorType>();
+ return getLhs().getType().cast<VectorType>();
}
Type getOperandTypeRHS() {
- return rhs().getType();
+ return getRhs().getType();
}
VectorType getOperandVectorTypeACC() {
- return (llvm::size(acc()) == 0)
+ return (llvm::size(getAcc()) == 0)
? VectorType()
- : (*acc().begin()).getType().cast<VectorType>();
+ : (*getAcc().begin()).getType().cast<VectorType>();
}
VectorType getVectorType() {
return getResult().getType().cast<VectorType>();
let extraClassDeclaration = [{
VectorType getInputVectorType() {
- return vector().getType().cast<VectorType>();
+ return getVector().getType().cast<VectorType>();
}
VectorType getOutputVectorType() {
return getResult().getType().cast<VectorType>();
}
/// Returns as integer value the number of input shape operands.
- int64_t getNumInputShapeSizes() { return input_shape().size(); }
+ int64_t getNumInputShapeSizes() { return getInputShape().size(); }
/// Returns as integer value the number of output shape operands.
- int64_t getNumOutputShapeSizes() { return output_shape().size(); }
+ int64_t getNumOutputShapeSizes() { return getOutputShape().size(); }
void getFixedVectorSizes(SmallVectorImpl<int64_t> &results);
static StringRef getOffsetsAttrStrName() { return "offsets"; }
static StringRef getSizesAttrStrName() { return "sizes"; }
static StringRef getStridesAttrStrName() { return "strides"; }
- VectorType getVectorType(){ return vector().getType().cast<VectorType>(); }
+ VectorType getVectorType(){ return getVector().getType().cast<VectorType>(); }
void getOffsets(SmallVectorImpl<int64_t> &results);
bool hasNonUnitStrides() {
- return llvm::any_of(strides(), [](Attribute attr) {
+ return llvm::any_of(getStrides(), [](Attribute attr) {
return attr.cast<IntegerAttr>().getInt() != 1;
});
}
let extraClassDeclaration = [{
MemRefType getMemRefType() {
- return base().getType().cast<MemRefType>();
+ return getBase().getType().cast<MemRefType>();
}
VectorType getVectorType() {
- return result().getType().cast<VectorType>();
+ return getResult().getType().cast<VectorType>();
}
}];
let extraClassDeclaration = [{
MemRefType getMemRefType() {
- return base().getType().cast<MemRefType>();
+ return getBase().getType().cast<MemRefType>();
}
VectorType getVectorType() {
- return valueToStore().getType().cast<VectorType>();
+ return getValueToStore().getType().cast<VectorType>();
}
}];
}];
let extraClassDeclaration = [{
MemRefType getMemRefType() {
- return base().getType().cast<MemRefType>();
+ return getBase().getType().cast<MemRefType>();
}
VectorType getMaskVectorType() {
- return mask().getType().cast<VectorType>();
+ return getMask().getType().cast<VectorType>();
}
VectorType getPassThruVectorType() {
- return pass_thru().getType().cast<VectorType>();
+ return getPassThru().getType().cast<VectorType>();
}
VectorType getVectorType() {
- return result().getType().cast<VectorType>();
+ return getResult().getType().cast<VectorType>();
}
}];
let assemblyFormat = "$base `[` $indices `]` `,` $mask `,` $pass_thru attr-dict `:` "
}];
let extraClassDeclaration = [{
MemRefType getMemRefType() {
- return base().getType().cast<MemRefType>();
+ return getBase().getType().cast<MemRefType>();
}
VectorType getMaskVectorType() {
- return mask().getType().cast<VectorType>();
+ return getMask().getType().cast<VectorType>();
}
VectorType getVectorType() {
- return valueToStore().getType().cast<VectorType>();
+ return getValueToStore().getType().cast<VectorType>();
}
}];
let assemblyFormat =
}];
let extraClassDeclaration = [{
MemRefType getMemRefType() {
- return base().getType().cast<MemRefType>();
+ return getBase().getType().cast<MemRefType>();
}
VectorType getIndexVectorType() {
- return index_vec().getType().cast<VectorType>();
+ return getIndexVec().getType().cast<VectorType>();
}
VectorType getMaskVectorType() {
- return mask().getType().cast<VectorType>();
+ return getMask().getType().cast<VectorType>();
}
VectorType getPassThruVectorType() {
- return pass_thru().getType().cast<VectorType>();
+ return getPassThru().getType().cast<VectorType>();
}
VectorType getVectorType() {
- return result().getType().cast<VectorType>();
+ return getResult().getType().cast<VectorType>();
}
}];
let assemblyFormat =
}];
let extraClassDeclaration = [{
MemRefType getMemRefType() {
- return base().getType().cast<MemRefType>();
+ return getBase().getType().cast<MemRefType>();
}
VectorType getIndexVectorType() {
- return index_vec().getType().cast<VectorType>();
+ return getIndexVec().getType().cast<VectorType>();
}
VectorType getMaskVectorType() {
- return mask().getType().cast<VectorType>();
+ return getMask().getType().cast<VectorType>();
}
VectorType getVectorType() {
- return valueToStore().getType().cast<VectorType>();
+ return getValueToStore().getType().cast<VectorType>();
}
}];
let assemblyFormat =
}];
let extraClassDeclaration = [{
MemRefType getMemRefType() {
- return base().getType().cast<MemRefType>();
+ return getBase().getType().cast<MemRefType>();
}
VectorType getMaskVectorType() {
- return mask().getType().cast<VectorType>();
+ return getMask().getType().cast<VectorType>();
}
VectorType getPassThruVectorType() {
- return pass_thru().getType().cast<VectorType>();
+ return getPassThru().getType().cast<VectorType>();
}
VectorType getVectorType() {
- return result().getType().cast<VectorType>();
+ return getResult().getType().cast<VectorType>();
}
}];
let assemblyFormat = "$base `[` $indices `]` `,` $mask `,` $pass_thru attr-dict `:` "
}];
let extraClassDeclaration = [{
MemRefType getMemRefType() {
- return base().getType().cast<MemRefType>();
+ return getBase().getType().cast<MemRefType>();
}
VectorType getMaskVectorType() {
- return mask().getType().cast<VectorType>();
+ return getMask().getType().cast<VectorType>();
}
VectorType getVectorType() {
- return valueToStore().getType().cast<VectorType>();
+ return getValueToStore().getType().cast<VectorType>();
}
}];
let assemblyFormat =
}];
let extraClassDeclaration = [{
VectorType getSourceVectorType() {
- return source().getType().cast<VectorType>();
+ return getSource().getType().cast<VectorType>();
}
VectorType getResultVectorType() {
return getResult().getType().cast<VectorType>();
}];
let extraClassDeclaration = [{
VectorType getSourceVectorType() {
- return source().getType().cast<VectorType>();
+ return getSource().getType().cast<VectorType>();
}
VectorType getResultVectorType() {
return getResult().getType().cast<VectorType>();
let extraClassDeclaration = [{
MemRefType getMemRefType() {
- return memref().getType().cast<MemRefType>();
+ return getMemref().getType().cast<MemRefType>();
}
MemRefType getResultMemRefType() {
return getResult().getType().cast<MemRefType>();
}
// Implement ViewLikeOpInterface.
- Value getViewSource() { return memref(); }
+ Value getViewSource() { return getMemref(); }
}];
let assemblyFormat = [{
];
let extraClassDeclaration = [{
VectorType getVectorType() {
- return vector().getType().cast<VectorType>();
+ return getVector().getType().cast<VectorType>();
}
VectorType getResultType() {
- return result().getType().cast<VectorType>();
+ return getResult().getType().cast<VectorType>();
}
void getTransp(SmallVectorImpl<int64_t> &results);
static StringRef getTranspAttrStrName() { return "transp"; }
}];
let extraClassDeclaration = [{
Type getPrintType() {
- return source().getType();
+ return getSource().getType();
}
}];
let assemblyFormat = "$source attr-dict `:` type($source)";
static StringRef getKindAttrStrName() { return "kind"; }
static StringRef getReductionDimAttrStrName() { return "reduction_dim"; }
VectorType getSourceType() {
- return source().getType().cast<VectorType>();
+ return getSource().getType().cast<VectorType>();
}
VectorType getDestType() {
- return dest().getType().cast<VectorType>();
+ return getDest().getType().cast<VectorType>();
}
VectorType getAccumulatorType() {
- return accumulated_value().getType().cast<VectorType>();
+ return getAccumulatedValue().getType().cast<VectorType>();
}
VectorType getInitialValueType() {
- return initial_value().getType().cast<VectorType>();
+ return getInitialValue().getType().cast<VectorType>();
}
}];
let assemblyFormat =
/*methodBody=*/"",
/*defaultImplementation=*/[{
return $_op.isBroadcastDim(dim)
- || ($_op.in_bounds()
- && $_op.in_bounds()->template cast<::mlir::ArrayAttr>()[dim]
+ || ($_op.getInBounds()
+ && $_op.getInBounds()->template cast<::mlir::ArrayAttr>()[dim]
.template cast<::mlir::BoolAttr>().getValue());
}]
>,
/*retTy=*/"::mlir::Value",
/*methodName=*/"source",
/*args=*/(ins),
- /*methodBody=*/"return $_op.source();"
+ /*methodBody=*/"return $_op.getSource();"
/*defaultImplementation=*/
>,
InterfaceMethod<
/*retTy=*/"::mlir::Value",
/*methodName=*/"vector",
/*args=*/(ins),
- /*methodBody=*/"return $_op.vector();"
+ /*methodBody=*/"return $_op.getVector();"
/*defaultImplementation=*/
>,
InterfaceMethod<
/*retTy=*/"::mlir::ValueRange",
/*methodName=*/"indices",
/*args=*/(ins),
- /*methodBody=*/"return $_op.indices();"
+ /*methodBody=*/"return $_op.getIndices();"
/*defaultImplementation=*/
>,
InterfaceMethod<
/*retTy=*/"::mlir::AffineMap",
/*methodName=*/"permutation_map",
/*args=*/(ins),
- /*methodBody=*/"return $_op.permutation_map();"
+ /*methodBody=*/"return $_op.getPermutationMap();"
/*defaultImplementation=*/
>,
InterfaceMethod<
/*args=*/(ins "unsigned":$idx),
/*methodBody=*/"",
/*defaultImplementation=*/[{
- auto expr = $_op.permutation_map().getResult(idx);
+ auto expr = $_op.getPermutationMap().getResult(idx);
return expr.template isa<::mlir::AffineConstantExpr>() &&
expr.template dyn_cast<::mlir::AffineConstantExpr>().getValue() == 0;
}]
/*retTy=*/"::mlir::Optional<::mlir::ArrayAttr>",
/*methodName=*/"in_bounds",
/*args=*/(ins),
- /*methodBody=*/"return $_op.in_bounds();"
+ /*methodBody=*/"return $_op.getInBounds();"
/*defaultImplementation=*/
>,
InterfaceMethod<
/*args=*/(ins),
/*methodBody=*/"",
/*defaultImplementation=*/
- "return $_op.source().getType().template cast<::mlir::ShapedType>();"
+ "return $_op.getSource().getType().template cast<::mlir::ShapedType>();"
>,
InterfaceMethod<
/*desc=*/"Return the VectorType.",
/*args=*/(ins),
/*methodBody=*/"",
/*defaultImplementation=*/[{
- return $_op.vector().getType().template dyn_cast<::mlir::VectorType>();
+ return $_op.getVector().getType().template dyn_cast<::mlir::VectorType>();
}]
>,
InterfaceMethod<
/*args=*/(ins),
/*methodBody=*/"",
/*defaultImplementation=*/[{
- return $_op.mask()
+ return $_op.getMask()
? ::mlir::vector::detail::transferMaskType(
- $_op.getVectorType(), $_op.permutation_map())
+ $_op.getVectorType(), $_op.getPermutationMap())
: ::mlir::VectorType();
}]
>,
/*args=*/(ins),
/*methodBody=*/"",
/*defaultImplementation=*/
- "return $_op.permutation_map().getNumResults();"
+ "return $_op.getPermutationMap().getNumResults();"
>,
InterfaceMethod<
/*desc=*/[{ Return the number of leading shaped dimensions that do not
// Return true if the contract op can be convert to MMA matmul.
static bool contractSupportsMMAMatrixType(vector::ContractionOp contract) {
- if (llvm::size(contract.masks()) != 0)
+ if (llvm::size(contract.getMasks()) != 0)
return false;
using MapList = ArrayRef<ArrayRef<AffineExpr>>;
auto infer = [](MapList m) { return AffineMap::inferFromExprList(m); };
AffineExpr m, n, k;
bindDims(contract.getContext(), m, n, k);
- auto iteratorTypes = contract.iterator_types().getValue();
+ auto iteratorTypes = contract.getIteratorTypes().getValue();
if (!(isParallelIterator(iteratorTypes[0]) &&
isParallelIterator(iteratorTypes[1]) &&
isReductionIterator(iteratorTypes[2])))
// Return true if the transfer op can be converted to a MMA matrix load.
static bool transferReadSupportsMMAMatrixType(vector::TransferReadOp readOp) {
- if (readOp.mask() || readOp.hasOutOfBoundsDim() ||
+ if (readOp.getMask() || readOp.hasOutOfBoundsDim() ||
readOp.getVectorType().getRank() != 2)
return false;
if (!getMemrefConstantHorizontalStride(readOp.getShapedType()))
return false;
- AffineMap map = readOp.permutation_map();
+ AffineMap map = readOp.getPermutationMap();
OpBuilder b(readOp.getContext());
AffineExpr innerDim = b.getAffineDimExpr(map.getNumDims() - 1);
AffineExpr zero = b.getAffineConstantExpr(0);
if (writeOp.getTransferRank() == 0)
return false;
- if (writeOp.mask() || writeOp.hasOutOfBoundsDim() ||
+ if (writeOp.getMask() || writeOp.hasOutOfBoundsDim() ||
writeOp.getVectorType().getRank() != 2)
return false;
if (!getMemrefConstantHorizontalStride(writeOp.getShapedType()))
return false;
// TODO: Support transpose once it is added to GPU dialect ops.
- if (!writeOp.permutation_map().isMinorIdentity())
+ if (!writeOp.getPermutationMap().isMinorIdentity())
return false;
return true;
}
/// Return true if this is a broadcast from scalar to a 2D vector.
static bool broadcastSupportsMMAMatrixType(vector::BroadcastOp broadcastOp) {
return broadcastOp.getVectorType().getRank() == 2 &&
- broadcastOp.source().getType().isa<FloatType>();
+ broadcastOp.getSource().getType().isa<FloatType>();
}
/// Return the MMA elementwise enum associated with `op` if it is supported.
LogicalResult matchAndRewrite(vector::ContractionOp op,
PatternRewriter &rewriter) const override {
Location loc = op.getLoc();
- Value lhs = op.lhs(), rhs = op.rhs(), res = op.acc();
+ Value lhs = op.getLhs(), rhs = op.getRhs(), res = op.getAcc();
// Set up the parallel/reduction structure in right form.
using MapList = ArrayRef<ArrayRef<AffineExpr>>;
AffineExpr m, n, k;
bindDims(rewriter.getContext(), m, n, k);
static constexpr std::array<int64_t, 2> perm = {1, 0};
- auto iteratorTypes = op.iterator_types().getValue();
+ auto iteratorTypes = op.getIteratorTypes().getValue();
SmallVector<AffineMap, 4> maps = op.getIndexingMaps();
if (!(isParallelIterator(iteratorTypes[0]) &&
isParallelIterator(iteratorTypes[1]) &&
rewriter.replaceOpWithNewOp<vector::ContractionOp>(
op, lhs, rhs, res,
rewriter.getAffineMapArrayAttr(infer({{m, k}, {k, n}, {m, n}})),
- op.iterator_types());
+ op.getIteratorTypes());
return success();
}
};
LogicalResult matchAndRewrite(vector::TransposeOp op,
PatternRewriter &rewriter) const override {
- auto transferReadOp = op.vector().getDefiningOp<vector::TransferReadOp>();
+ auto transferReadOp =
+ op.getVector().getDefiningOp<vector::TransferReadOp>();
if (!transferReadOp)
return failure();
if (transferReadOp.getTransferRank() == 0)
return failure();
- if (transferReadOp.mask() || transferReadOp.hasOutOfBoundsDim())
+ if (transferReadOp.getMask() || transferReadOp.hasOutOfBoundsDim())
return failure();
SmallVector<int64_t, 2> perm;
op.getTransp(perm);
permU.push_back(unsigned(o));
AffineMap permutationMap =
AffineMap::getPermutationMap(permU, op.getContext());
- AffineMap newMap = permutationMap.compose(transferReadOp.permutation_map());
+ AffineMap newMap =
+ permutationMap.compose(transferReadOp.getPermutationMap());
rewriter.replaceOpWithNewOp<vector::TransferReadOp>(
- op, op.getType(), transferReadOp.source(), transferReadOp.indices(),
- AffineMapAttr::get(newMap), transferReadOp.padding(),
- transferReadOp.mask(), transferReadOp.in_boundsAttr());
+ op, op.getType(), transferReadOp.getSource(),
+ transferReadOp.getIndices(), AffineMapAttr::get(newMap),
+ transferReadOp.getPadding(), transferReadOp.getMask(),
+ transferReadOp.getInBoundsAttr());
return success();
}
};
auto contract = dyn_cast<vector::ContractionOp>(users);
if (!contract)
continue;
- if (contract.lhs() == op.getResult())
+ if (contract.getLhs() == op.getResult())
return "AOp";
- if (contract.rhs() == op.getResult())
+ if (contract.getRhs() == op.getResult())
return "BOp";
}
return "COp";
assert(transferReadSupportsMMAMatrixType(op));
Optional<int64_t> stride =
getMemrefConstantHorizontalStride(op.getShapedType());
- AffineMap map = op.permutation_map();
+ AffineMap map = op.getPermutationMap();
// Handle broadcast by setting the stride to 0.
if (map.getResult(0).isa<AffineConstantExpr>()) {
assert(map.getResult(0).cast<AffineConstantExpr>().getValue() == 0);
op.getVectorType().getElementType(), fragType);
OpBuilder b(op);
Value load = b.create<gpu::SubgroupMmaLoadMatrixOp>(
- op.getLoc(), type, op.source(), op.indices(), b.getIndexAttr(*stride));
+ op.getLoc(), type, op.getSource(), op.getIndices(),
+ b.getIndexAttr(*stride));
valueMapping[op.getResult()] = load;
}
getMemrefConstantHorizontalStride(op.getShapedType());
assert(stride);
OpBuilder b(op);
- Value matrix = valueMapping.find(op.vector())->second;
- b.create<gpu::SubgroupMmaStoreMatrixOp>(
- op.getLoc(), matrix, op.source(), op.indices(), b.getIndexAttr(*stride));
+ Value matrix = valueMapping.find(op.getVector())->second;
+ b.create<gpu::SubgroupMmaStoreMatrixOp>(op.getLoc(), matrix, op.getSource(),
+ op.getIndices(),
+ b.getIndexAttr(*stride));
op.erase();
}
static void convertContractOp(vector::ContractionOp op,
llvm::DenseMap<Value, Value> &valueMapping) {
OpBuilder b(op);
- Value opA = valueMapping.find(op.lhs())->second;
- Value opB = valueMapping.find(op.rhs())->second;
- Value opC = valueMapping.find(op.acc())->second;
+ Value opA = valueMapping.find(op.getLhs())->second;
+ Value opB = valueMapping.find(op.getRhs())->second;
+ Value opC = valueMapping.find(op.getAcc())->second;
Value matmul = b.create<gpu::SubgroupMmaComputeOp>(op.getLoc(), opC.getType(),
opA, opB, opC);
valueMapping[op.getResult()] = matmul;
gpu::MMAMatrixType type = gpu::MMAMatrixType::get(
vecType.getShape(), vecType.getElementType(), llvm::StringRef(fragType));
auto matrix = b.create<gpu::SubgroupMmaConstantMatrixOp>(op.getLoc(), type,
- op.source());
+ op.getSource());
valueMapping[op.getResult()] = matrix;
}
matchAndRewrite(vector::MatmulOp matmulOp, OpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
rewriter.replaceOpWithNewOp<LLVM::MatrixMultiplyOp>(
- matmulOp, typeConverter->convertType(matmulOp.res().getType()),
- adaptor.lhs(), adaptor.rhs(), matmulOp.lhs_rows(),
- matmulOp.lhs_columns(), matmulOp.rhs_columns());
+ matmulOp, typeConverter->convertType(matmulOp.getRes().getType()),
+ adaptor.getLhs(), adaptor.getRhs(), matmulOp.getLhsRows(),
+ matmulOp.getLhsColumns(), matmulOp.getRhsColumns());
return success();
}
};
matchAndRewrite(vector::FlatTransposeOp transOp, OpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
rewriter.replaceOpWithNewOp<LLVM::MatrixTransposeOp>(
- transOp, typeConverter->convertType(transOp.res().getType()),
- adaptor.matrix(), transOp.rows(), transOp.columns());
+ transOp, typeConverter->convertType(transOp.getRes().getType()),
+ adaptor.getMatrix(), transOp.getRows(), transOp.getColumns());
return success();
}
};
VectorType vectorTy, Value ptr, unsigned align,
ConversionPatternRewriter &rewriter) {
rewriter.replaceOpWithNewOp<LLVM::MaskedLoadOp>(
- loadOp, vectorTy, ptr, adaptor.mask(), adaptor.pass_thru(), align);
+ loadOp, vectorTy, ptr, adaptor.getMask(), adaptor.getPassThru(), align);
}
static void replaceLoadOrStoreOp(vector::StoreOp storeOp,
vector::StoreOpAdaptor adaptor,
VectorType vectorTy, Value ptr, unsigned align,
ConversionPatternRewriter &rewriter) {
- rewriter.replaceOpWithNewOp<LLVM::StoreOp>(storeOp, adaptor.valueToStore(),
+ rewriter.replaceOpWithNewOp<LLVM::StoreOp>(storeOp, adaptor.getValueToStore(),
ptr, align);
}
VectorType vectorTy, Value ptr, unsigned align,
ConversionPatternRewriter &rewriter) {
rewriter.replaceOpWithNewOp<LLVM::MaskedStoreOp>(
- storeOp, adaptor.valueToStore(), ptr, adaptor.mask(), align);
+ storeOp, adaptor.getValueToStore(), ptr, adaptor.getMask(), align);
}
/// Conversion pattern for a vector.load, vector.store, vector.maskedload, and
// Resolve address.
auto vtype = this->typeConverter->convertType(loadOrStoreOp.getVectorType())
.template cast<VectorType>();
- Value dataPtr = this->getStridedElementPtr(loc, memRefTy, adaptor.base(),
- adaptor.indices(), rewriter);
+ Value dataPtr = this->getStridedElementPtr(loc, memRefTy, adaptor.getBase(),
+ adaptor.getIndices(), rewriter);
Value ptr = castDataPtr(rewriter, loc, dataPtr, memRefTy, vtype);
replaceLoadOrStoreOp(loadOrStoreOp, adaptor, vtype, ptr, align, rewriter);
// Resolve address.
Value ptrs;
VectorType vType = gather.getVectorType();
- Value ptr = getStridedElementPtr(loc, memRefType, adaptor.base(),
- adaptor.indices(), rewriter);
- if (failed(getIndexedPtrs(rewriter, loc, adaptor.base(), ptr,
- adaptor.index_vec(), memRefType, vType, ptrs)))
+ Value ptr = getStridedElementPtr(loc, memRefType, adaptor.getBase(),
+ adaptor.getIndices(), rewriter);
+ if (failed(getIndexedPtrs(rewriter, loc, adaptor.getBase(), ptr,
+ adaptor.getIndexVec(), memRefType, vType, ptrs)))
return failure();
// Replace with the gather intrinsic.
rewriter.replaceOpWithNewOp<LLVM::masked_gather>(
- gather, typeConverter->convertType(vType), ptrs, adaptor.mask(),
- adaptor.pass_thru(), rewriter.getI32IntegerAttr(align));
+ gather, typeConverter->convertType(vType), ptrs, adaptor.getMask(),
+ adaptor.getPassThru(), rewriter.getI32IntegerAttr(align));
return success();
}
};
// Resolve address.
Value ptrs;
VectorType vType = scatter.getVectorType();
- Value ptr = getStridedElementPtr(loc, memRefType, adaptor.base(),
- adaptor.indices(), rewriter);
- if (failed(getIndexedPtrs(rewriter, loc, adaptor.base(), ptr,
- adaptor.index_vec(), memRefType, vType, ptrs)))
+ Value ptr = getStridedElementPtr(loc, memRefType, adaptor.getBase(),
+ adaptor.getIndices(), rewriter);
+ if (failed(getIndexedPtrs(rewriter, loc, adaptor.getBase(), ptr,
+ adaptor.getIndexVec(), memRefType, vType, ptrs)))
return failure();
// Replace with the scatter intrinsic.
rewriter.replaceOpWithNewOp<LLVM::masked_scatter>(
- scatter, adaptor.valueToStore(), ptrs, adaptor.mask(),
+ scatter, adaptor.getValueToStore(), ptrs, adaptor.getMask(),
rewriter.getI32IntegerAttr(align));
return success();
}
// Resolve address.
auto vtype = typeConverter->convertType(expand.getVectorType());
- Value ptr = getStridedElementPtr(loc, memRefType, adaptor.base(),
- adaptor.indices(), rewriter);
+ Value ptr = getStridedElementPtr(loc, memRefType, adaptor.getBase(),
+ adaptor.getIndices(), rewriter);
rewriter.replaceOpWithNewOp<LLVM::masked_expandload>(
- expand, vtype, ptr, adaptor.mask(), adaptor.pass_thru());
+ expand, vtype, ptr, adaptor.getMask(), adaptor.getPassThru());
return success();
}
};
MemRefType memRefType = compress.getMemRefType();
// Resolve address.
- Value ptr = getStridedElementPtr(loc, memRefType, adaptor.base(),
- adaptor.indices(), rewriter);
+ Value ptr = getStridedElementPtr(loc, memRefType, adaptor.getBase(),
+ adaptor.getIndices(), rewriter);
rewriter.replaceOpWithNewOp<LLVM::masked_compressstore>(
- compress, adaptor.valueToStore(), ptr, adaptor.mask());
+ compress, adaptor.getValueToStore(), ptr, adaptor.getMask());
return success();
}
};
LogicalResult
matchAndRewrite(vector::ReductionOp reductionOp, OpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
- auto kind = reductionOp.kind();
- Type eltType = reductionOp.dest().getType();
+ auto kind = reductionOp.getKind();
+ Type eltType = reductionOp.getDest().getType();
Type llvmType = typeConverter->convertType(eltType);
Value operand = adaptor.getOperands()[0];
if (eltType.isIntOrIndex()) {
auto v2Type = shuffleOp.getV2VectorType();
auto vectorType = shuffleOp.getVectorType();
Type llvmType = typeConverter->convertType(vectorType);
- auto maskArrayAttr = shuffleOp.mask();
+ auto maskArrayAttr = shuffleOp.getMask();
// Bail if result type cannot be lowered.
if (!llvmType)
// there is direct shuffle support in LLVM. Use it!
if (rank == 1 && v1Type == v2Type) {
Value llvmShuffleOp = rewriter.create<LLVM::ShuffleVectorOp>(
- loc, adaptor.v1(), adaptor.v2(), maskArrayAttr);
+ loc, adaptor.getV1(), adaptor.getV2(), maskArrayAttr);
rewriter.replaceOp(shuffleOp, llvmShuffleOp);
return success();
}
int64_t insPos = 0;
for (const auto &en : llvm::enumerate(maskArrayAttr)) {
int64_t extPos = en.value().cast<IntegerAttr>().getInt();
- Value value = adaptor.v1();
+ Value value = adaptor.getV1();
if (extPos >= v1Dim) {
extPos -= v1Dim;
- value = adaptor.v2();
+ value = adaptor.getV2();
}
Value extract = extractOne(rewriter, *getTypeConverter(), loc, value,
eltType, rank, extPos);
loc, typeConverter->convertType(idxType),
rewriter.getIntegerAttr(idxType, 0));
rewriter.replaceOpWithNewOp<LLVM::ExtractElementOp>(
- extractEltOp, llvmType, adaptor.vector(), zero);
+ extractEltOp, llvmType, adaptor.getVector(), zero);
return success();
}
rewriter.replaceOpWithNewOp<LLVM::ExtractElementOp>(
- extractEltOp, llvmType, adaptor.vector(), adaptor.position());
+ extractEltOp, llvmType, adaptor.getVector(), adaptor.getPosition());
return success();
}
};
auto vectorType = extractOp.getVectorType();
auto resultType = extractOp.getResult().getType();
auto llvmResultType = typeConverter->convertType(resultType);
- auto positionArrayAttr = extractOp.position();
+ auto positionArrayAttr = extractOp.getPosition();
// Bail if result type cannot be lowered.
if (!llvmResultType)
// Extract entire vector. Should be handled by folder, but just to be safe.
if (positionArrayAttr.empty()) {
- rewriter.replaceOp(extractOp, adaptor.vector());
+ rewriter.replaceOp(extractOp, adaptor.getVector());
return success();
}
// One-shot extraction of vector from array (only requires extractvalue).
if (resultType.isa<VectorType>()) {
Value extracted = rewriter.create<LLVM::ExtractValueOp>(
- loc, llvmResultType, adaptor.vector(), positionArrayAttr);
+ loc, llvmResultType, adaptor.getVector(), positionArrayAttr);
rewriter.replaceOp(extractOp, extracted);
return success();
}
// Potential extraction of 1-D vector from array.
auto *context = extractOp->getContext();
- Value extracted = adaptor.vector();
+ Value extracted = adaptor.getVector();
auto positionAttrs = positionArrayAttr.getValue();
if (positionAttrs.size() > 1) {
auto oneDVectorType = reducedVectorTypeBack(vectorType);
VectorType vType = fmaOp.getVectorType();
if (vType.getRank() != 1)
return failure();
- rewriter.replaceOpWithNewOp<LLVM::FMulAddOp>(fmaOp, adaptor.lhs(),
- adaptor.rhs(), adaptor.acc());
+ rewriter.replaceOpWithNewOp<LLVM::FMulAddOp>(
+ fmaOp, adaptor.getLhs(), adaptor.getRhs(), adaptor.getAcc());
return success();
}
};
loc, typeConverter->convertType(idxType),
rewriter.getIntegerAttr(idxType, 0));
rewriter.replaceOpWithNewOp<LLVM::InsertElementOp>(
- insertEltOp, llvmType, adaptor.dest(), adaptor.source(), zero);
+ insertEltOp, llvmType, adaptor.getDest(), adaptor.getSource(), zero);
return success();
}
rewriter.replaceOpWithNewOp<LLVM::InsertElementOp>(
- insertEltOp, llvmType, adaptor.dest(), adaptor.source(),
- adaptor.position());
+ insertEltOp, llvmType, adaptor.getDest(), adaptor.getSource(),
+ adaptor.getPosition());
return success();
}
};
auto sourceType = insertOp.getSourceType();
auto destVectorType = insertOp.getDestVectorType();
auto llvmResultType = typeConverter->convertType(destVectorType);
- auto positionArrayAttr = insertOp.position();
+ auto positionArrayAttr = insertOp.getPosition();
// Bail if result type cannot be lowered.
if (!llvmResultType)
// Overwrite entire vector with value. Should be handled by folder, but
// just to be safe.
if (positionArrayAttr.empty()) {
- rewriter.replaceOp(insertOp, adaptor.source());
+ rewriter.replaceOp(insertOp, adaptor.getSource());
return success();
}
// One-shot insertion of a vector into an array (only requires insertvalue).
if (sourceType.isa<VectorType>()) {
Value inserted = rewriter.create<LLVM::InsertValueOp>(
- loc, llvmResultType, adaptor.dest(), adaptor.source(),
+ loc, llvmResultType, adaptor.getDest(), adaptor.getSource(),
positionArrayAttr);
rewriter.replaceOp(insertOp, inserted);
return success();
// Potential extraction of 1-D vector from array.
auto *context = insertOp->getContext();
- Value extracted = adaptor.dest();
+ Value extracted = adaptor.getDest();
auto positionAttrs = positionArrayAttr.getValue();
auto position = positionAttrs.back().cast<IntegerAttr>();
auto oneDVectorType = destVectorType;
auto constant = rewriter.create<LLVM::ConstantOp>(loc, i64Type, position);
Value inserted = rewriter.create<LLVM::InsertElementOp>(
loc, typeConverter->convertType(oneDVectorType), extracted,
- adaptor.source(), constant);
+ adaptor.getSource(), constant);
// Potential insertion of resulting 1-D vector into array.
if (positionAttrs.size() > 1) {
auto nMinusOnePositionAttrs =
ArrayAttr::get(context, positionAttrs.drop_back());
- inserted = rewriter.create<LLVM::InsertValueOp>(loc, llvmResultType,
- adaptor.dest(), inserted,
- nMinusOnePositionAttrs);
+ inserted = rewriter.create<LLVM::InsertValueOp>(
+ loc, llvmResultType, adaptor.getDest(), inserted,
+ nMinusOnePositionAttrs);
}
rewriter.replaceOp(insertOp, inserted);
loc, elemType, rewriter.getZeroAttr(elemType));
Value desc = rewriter.create<vector::SplatOp>(loc, vType, zero);
for (int64_t i = 0, e = vType.getShape().front(); i != e; ++i) {
- Value extrLHS = rewriter.create<ExtractOp>(loc, op.lhs(), i);
- Value extrRHS = rewriter.create<ExtractOp>(loc, op.rhs(), i);
- Value extrACC = rewriter.create<ExtractOp>(loc, op.acc(), i);
+ Value extrLHS = rewriter.create<ExtractOp>(loc, op.getLhs(), i);
+ Value extrRHS = rewriter.create<ExtractOp>(loc, op.getRhs(), i);
+ Value extrACC = rewriter.create<ExtractOp>(loc, op.getAcc(), i);
Value fma = rewriter.create<FMAOp>(loc, extrLHS, extrRHS, extrACC);
desc = rewriter.create<InsertOp>(loc, fma, desc, i);
}
// Unroll vector into elementary print calls.
int64_t rank = vectorType ? vectorType.getRank() : 0;
Type type = vectorType ? vectorType : eltType;
- emitRanks(rewriter, printOp, adaptor.source(), type, printer, rank,
+ emitRanks(rewriter, printOp, adaptor.getSource(), type, printer, rank,
conversion);
emitCall(rewriter, printOp->getLoc(),
LLVM::lookupOrCreatePrintNewlineFn(
// For 0-d vector, we simply do `insertelement`.
if (resultType.getRank() == 0) {
rewriter.replaceOpWithNewOp<LLVM::InsertElementOp>(
- splatOp, vectorType, undef, adaptor.input(), zero);
+ splatOp, vectorType, undef, adaptor.getInput(), zero);
return success();
}
// For 1-d vector, we additionally do a `vectorshuffle`.
auto v = rewriter.create<LLVM::InsertElementOp>(
- splatOp.getLoc(), vectorType, undef, adaptor.input(), zero);
+ splatOp.getLoc(), vectorType, undef, adaptor.getInput(), zero);
int64_t width = splatOp.getType().cast<VectorType>().getDimSize(0);
SmallVector<int32_t, 4> zeroValues(width, 0);
loc, typeConverter->convertType(rewriter.getIntegerType(32)),
rewriter.getZeroAttr(rewriter.getIntegerType(32)));
Value v = rewriter.create<LLVM::InsertElementOp>(loc, llvm1DVectorTy, vdesc,
- adaptor.input(), zero);
+ adaptor.getInput(), zero);
// Shuffle the value across the desired number of elements.
int64_t width = resultType.getDimSize(resultType.getRank() - 1);
Type &vecTy, Value &dwordConfig, Value &vindex, Value &offsetSizeInBytes,
Value &glc, Value &slc) {
auto adaptor = TransferWriteOpAdaptor(operands, xferOp->getAttrDictionary());
- rewriter.replaceOpWithNewOp<ROCDL::MubufStoreOp>(xferOp, adaptor.vector(),
+ rewriter.replaceOpWithNewOp<ROCDL::MubufStoreOp>(xferOp, adaptor.getVector(),
dwordConfig, vindex,
offsetSizeInBytes, glc, slc);
return success();
return failure();
if (xferOp.getVectorType().getRank() > 1 ||
- llvm::size(xferOp.indices()) == 0)
+ llvm::size(xferOp.getIndices()) == 0)
return failure();
- if (!xferOp.permutation_map().isMinorIdentity())
+ if (!xferOp.getPermutationMap().isMinorIdentity())
return failure();
// Have it handled in vector->llvm conversion pass.
// indices, so no need to calculate offset size in bytes again in
// the MUBUF instruction.
Value dataPtr = this->getStridedElementPtr(
- loc, memRefType, adaptor.source(), adaptor.indices(), rewriter);
+ loc, memRefType, adaptor.getSource(), adaptor.getIndices(), rewriter);
// 1. Create and fill a <4 x i32> dwordConfig with:
// 1st two elements holding the address of dataPtr.
static Optional<int64_t> unpackedDim(OpTy xferOp) {
// TODO: support 0-d corner case.
assert(xferOp.getTransferRank() > 0 && "unexpected 0-d transfer");
- auto map = xferOp.permutation_map();
+ auto map = xferOp.getPermutationMap();
if (auto expr = map.getResult(0).template dyn_cast<AffineDimExpr>()) {
return expr.getPosition();
}
static AffineMap unpackedPermutationMap(OpBuilder &b, OpTy xferOp) {
// TODO: support 0-d corner case.
assert(xferOp.getTransferRank() > 0 && "unexpected 0-d transfer");
- auto map = xferOp.permutation_map();
+ auto map = xferOp.getPermutationMap();
return AffineMap::get(map.getNumDims(), 0, map.getResults().drop_front(),
b.getContext());
}
typename OpTy::Adaptor adaptor(xferOp);
// Corresponding memref dim of the vector dim that is unpacked.
auto dim = unpackedDim(xferOp);
- auto prevIndices = adaptor.indices();
+ auto prevIndices = adaptor.getIndices();
indices.append(prevIndices.begin(), prevIndices.end());
Location loc = xferOp.getLoc();
if (!isBroadcast) {
AffineExpr d0, d1;
bindDims(xferOp.getContext(), d0, d1);
- Value offset = adaptor.indices()[dim.getValue()];
+ Value offset = adaptor.getIndices()[dim.getValue()];
indices[dim.getValue()] =
makeComposedAffineApply(b, loc, d0 + d1, {offset, iv});
}
/// * The to-be-unpacked dim of xferOp is a broadcast.
template <typename OpTy>
static Value generateMaskCheck(OpBuilder &b, OpTy xferOp, Value iv) {
- if (!xferOp.mask())
+ if (!xferOp.getMask())
return Value();
if (xferOp.getMaskType().getRank() != 1)
return Value();
return Value();
Location loc = xferOp.getLoc();
- return b.create<vector::ExtractElementOp>(loc, xferOp.mask(), iv);
+ return b.create<vector::ExtractElementOp>(loc, xferOp.getMask(), iv);
}
/// Helper function TransferOpConversion and TransferOp1dConversion.
Location loc = xferOp.getLoc();
ImplicitLocOpBuilder lb(xferOp.getLoc(), b);
if (!xferOp.isDimInBounds(0) && !isBroadcast) {
- Value memrefDim = vector::createOrFoldDimOp(b, loc, xferOp.source(), *dim);
+ Value memrefDim =
+ vector::createOrFoldDimOp(b, loc, xferOp.getSource(), *dim);
AffineExpr d0, d1;
bindDims(xferOp.getContext(), d0, d1);
- Value base = xferOp.indices()[dim.getValue()];
+ Value base = xferOp.getIndices()[dim.getValue()];
Value memrefIdx = makeComposedAffineApply(b, loc, d0 + d1, {base, iv});
cond = lb.create<arith::CmpIOp>(arith::CmpIPredicate::sgt, memrefDim,
memrefIdx);
auto bufferType = MemRefType::get({}, xferOp.getVectorType());
result.dataBuffer = b.create<memref::AllocaOp>(loc, bufferType);
- if (xferOp.mask()) {
- auto maskType = MemRefType::get({}, xferOp.mask().getType());
+ if (xferOp.getMask()) {
+ auto maskType = MemRefType::get({}, xferOp.getMask().getType());
auto maskBuffer = b.create<memref::AllocaOp>(loc, maskType);
b.setInsertionPoint(xferOp);
- b.create<memref::StoreOp>(loc, xferOp.mask(), maskBuffer);
+ b.create<memref::StoreOp>(loc, xferOp.getMask(), maskBuffer);
result.maskBuffer = b.create<memref::LoadOp>(loc, maskBuffer);
}
/// is similar to Strategy<TransferWriteOp>::getBuffer.
template <typename OpTy>
static Value getMaskBuffer(OpTy xferOp) {
- assert(xferOp.mask() && "Expected that transfer op has mask");
- auto loadOp = xferOp.mask().template getDefiningOp<memref::LoadOp>();
+ assert(xferOp.getMask() && "Expected that transfer op has mask");
+ auto loadOp = xferOp.getMask().template getDefiningOp<memref::LoadOp>();
assert(loadOp && "Expected transfer op mask produced by LoadOp");
return loadOp.getMemRef();
}
Location loc = xferOp.getLoc();
auto bufferType = buffer.getType().dyn_cast<ShapedType>();
auto vecType = bufferType.getElementType().dyn_cast<VectorType>();
- auto inBoundsAttr = dropFirstElem(b, xferOp.in_boundsAttr());
+ auto inBoundsAttr = dropFirstElem(b, xferOp.getInBoundsAttr());
auto newXferOp = b.create<vector::TransferReadOp>(
- loc, vecType, xferOp.source(), xferIndices,
- AffineMapAttr::get(unpackedPermutationMap(b, xferOp)), xferOp.padding(),
- Value(), inBoundsAttr);
+ loc, vecType, xferOp.getSource(), xferIndices,
+ AffineMapAttr::get(unpackedPermutationMap(b, xferOp)),
+ xferOp.getPadding(), Value(), inBoundsAttr);
maybeApplyPassLabel(b, newXferOp, options.targetRank);
- b.create<memref::StoreOp>(loc, newXferOp.vector(), buffer, storeIndices);
+ b.create<memref::StoreOp>(loc, newXferOp.getVector(), buffer, storeIndices);
return newXferOp;
}
Location loc = xferOp.getLoc();
auto bufferType = buffer.getType().dyn_cast<ShapedType>();
auto vecType = bufferType.getElementType().dyn_cast<VectorType>();
- auto vec = b.create<vector::SplatOp>(loc, vecType, xferOp.padding());
+ auto vec = b.create<vector::SplatOp>(loc, vecType, xferOp.getPadding());
b.create<memref::StoreOp>(loc, vec, buffer, storeIndices);
return Value();
/// vector.transfer_write %vec ... { __vector_to_scf_lowering__ } ...
/// ```
static Value getBuffer(TransferWriteOp xferOp) {
- auto loadOp = xferOp.vector().getDefiningOp<memref::LoadOp>();
+ auto loadOp = xferOp.getVector().getDefiningOp<memref::LoadOp>();
assert(loadOp && "Expected transfer op vector produced by LoadOp");
return loadOp.getMemRef();
}
/// Retrieve the indices of the current LoadOp that loads from the buffer.
static void getBufferIndices(TransferWriteOp xferOp,
SmallVector<Value, 8> &indices) {
- auto loadOp = xferOp.vector().getDefiningOp<memref::LoadOp>();
+ auto loadOp = xferOp.getVector().getDefiningOp<memref::LoadOp>();
auto prevIndices = memref::LoadOpAdaptor(loadOp).indices();
indices.append(prevIndices.begin(), prevIndices.end());
}
Location loc = xferOp.getLoc();
auto vec = b.create<memref::LoadOp>(loc, buffer, loadIndices);
- auto inBoundsAttr = dropFirstElem(b, xferOp.in_boundsAttr());
- auto source = loopState.empty() ? xferOp.source() : loopState[0];
+ auto inBoundsAttr = dropFirstElem(b, xferOp.getInBoundsAttr());
+ auto source = loopState.empty() ? xferOp.getSource() : loopState[0];
Type type = isTensorOp(xferOp) ? xferOp.getShapedType() : Type();
auto newXferOp = b.create<vector::TransferWriteOp>(
loc, type, vec, source, xferIndices,
/// Return the initial loop state for the generated scf.for loop.
static Value initialLoopState(TransferWriteOp xferOp) {
- return isTensorOp(xferOp) ? xferOp.source() : Value();
+ return isTensorOp(xferOp) ? xferOp.getSource() : Value();
}
};
auto buffers = allocBuffers(rewriter, xferOp);
auto *newXfer = rewriter.clone(*xferOp.getOperation());
newXfer->setAttr(kPassLabel, rewriter.getUnitAttr());
- if (xferOp.mask()) {
- dyn_cast<TransferReadOp>(newXfer).maskMutable().assign(
+ if (xferOp.getMask()) {
+ dyn_cast<TransferReadOp>(newXfer).getMaskMutable().assign(
buffers.maskBuffer);
}
Location loc = xferOp.getLoc();
auto buffers = allocBuffers(rewriter, xferOp);
- rewriter.create<memref::StoreOp>(loc, xferOp.vector(), buffers.dataBuffer);
+ rewriter.create<memref::StoreOp>(loc, xferOp.getVector(),
+ buffers.dataBuffer);
auto loadedVec = rewriter.create<memref::LoadOp>(loc, buffers.dataBuffer);
rewriter.updateRootInPlace(xferOp, [&]() {
- xferOp.vectorMutable().assign(loadedVec);
+ xferOp.getVectorMutable().assign(loadedVec);
xferOp->setAttr(kPassLabel, rewriter.getUnitAttr());
});
- if (xferOp.mask()) {
- rewriter.updateRootInPlace(
- xferOp, [&]() { xferOp.maskMutable().assign(buffers.maskBuffer); });
+ if (xferOp.getMask()) {
+ rewriter.updateRootInPlace(xferOp, [&]() {
+ xferOp.getMaskMutable().assign(buffers.maskBuffer);
+ });
}
return success();
// If the xferOp has a mask: Find and cast mask buffer.
Value castedMaskBuffer;
- if (xferOp.mask()) {
+ if (xferOp.getMask()) {
auto maskBuffer = getMaskBuffer(xferOp);
auto maskBufferType =
maskBuffer.getType().template dyn_cast<MemRefType>();
// the
// unpacked dim is not a broadcast, no mask is
// needed on the new transfer op.
- if (xferOp.mask() && (xferOp.isBroadcastDim(0) ||
- xferOp.getMaskType().getRank() > 1)) {
+ if (xferOp.getMask() && (xferOp.isBroadcastDim(0) ||
+ xferOp.getMaskType().getRank() > 1)) {
OpBuilder::InsertionGuard guard(b);
b.setInsertionPoint(newXfer); // Insert load before newXfer.
auto mask = b.create<memref::LoadOp>(loc, castedMaskBuffer,
loadIndices);
- rewriter.updateRootInPlace(
- newXfer, [&]() { newXfer.maskMutable().assign(mask); });
+ rewriter.updateRootInPlace(newXfer, [&]() {
+ newXfer.getMaskMutable().assign(mask);
+ });
}
return loopState.empty() ? Value() : newXfer->getResult(0);
template <typename OpTy>
static void maybeAssignMask(OpBuilder &b, OpTy xferOp, OpTy newXferOp,
int64_t i) {
- if (!xferOp.mask())
+ if (!xferOp.getMask())
return;
if (xferOp.isBroadcastDim(0)) {
// To-be-unpacked dimension is a broadcast, which does not have a
// corresponding mask dimension. Mask attribute remains unchanged.
- newXferOp.maskMutable().assign(xferOp.mask());
+ newXferOp.getMaskMutable().assign(xferOp.getMask());
return;
}
llvm::SmallVector<int64_t, 1> indices({i});
Location loc = xferOp.getLoc();
- auto newMask = b.create<vector::ExtractOp>(loc, xferOp.mask(), indices);
- newXferOp.maskMutable().assign(newMask);
+ auto newMask = b.create<vector::ExtractOp>(loc, xferOp.getMask(), indices);
+ newXferOp.getMaskMutable().assign(newMask);
}
// If we end up here: The mask of the old transfer op is 1D and the unpacked
Value getResultVector(TransferReadOp xferOp,
PatternRewriter &rewriter) const {
if (auto insertOp = getInsertOp(xferOp))
- return insertOp.dest();
+ return insertOp.getDest();
Location loc = xferOp.getLoc();
return rewriter.create<vector::SplatOp>(loc, xferOp.getVectorType(),
- xferOp.padding());
+ xferOp.getPadding());
}
/// If the result of the TransferReadOp has exactly one user, which is a
void getInsertionIndices(TransferReadOp xferOp,
SmallVector<int64_t, 8> &indices) const {
if (auto insertOp = getInsertOp(xferOp)) {
- llvm::for_each(insertOp.position(), [&](Attribute attr) {
+ llvm::for_each(insertOp.getPosition(), [&](Attribute attr) {
indices.push_back(attr.dyn_cast<IntegerAttr>().getInt());
});
}
getInsertionIndices(xferOp, insertionIndices);
insertionIndices.push_back(i);
- auto inBoundsAttr = dropFirstElem(b, xferOp.in_boundsAttr());
+ auto inBoundsAttr = dropFirstElem(b, xferOp.getInBoundsAttr());
auto newXferOp = b.create<vector::TransferReadOp>(
- loc, newXferVecType, xferOp.source(), xferIndices,
+ loc, newXferVecType, xferOp.getSource(), xferIndices,
AffineMapAttr::get(unpackedPermutationMap(b, xferOp)),
- xferOp.padding(), Value(), inBoundsAttr);
+ xferOp.getPadding(), Value(), inBoundsAttr);
maybeAssignMask(b, xferOp, newXferOp, i);
return b.create<vector::InsertOp>(loc, newXferOp, vec,
insertionIndices);
/// Return the vector from which newly generated ExtracOps will extract.
Value getDataVector(TransferWriteOp xferOp) const {
if (auto extractOp = getExtractOp(xferOp))
- return extractOp.vector();
- return xferOp.vector();
+ return extractOp.getVector();
+ return xferOp.getVector();
}
/// If the input of the given TransferWriteOp is an ExtractOp, return it.
vector::ExtractOp getExtractOp(TransferWriteOp xferOp) const {
- if (auto *op = xferOp.vector().getDefiningOp())
+ if (auto *op = xferOp.getVector().getDefiningOp())
return dyn_cast<vector::ExtractOp>(op);
return vector::ExtractOp();
}
void getExtractionIndices(TransferWriteOp xferOp,
SmallVector<int64_t, 8> &indices) const {
if (auto extractOp = getExtractOp(xferOp)) {
- llvm::for_each(extractOp.position(), [&](Attribute attr) {
+ llvm::for_each(extractOp.getPosition(), [&](Attribute attr) {
indices.push_back(attr.dyn_cast<IntegerAttr>().getInt());
});
}
auto vec = getDataVector(xferOp);
auto xferVecType = xferOp.getVectorType();
int64_t dimSize = xferVecType.getShape()[0];
- auto source = xferOp.source(); // memref or tensor to be written to.
+ auto source = xferOp.getSource(); // memref or tensor to be written to.
auto sourceType = isTensorOp(xferOp) ? xferOp.getShapedType() : Type();
// Generate fully unrolled loop of transfer ops.
auto extracted =
b.create<vector::ExtractOp>(loc, vec, extractionIndices);
- auto inBoundsAttr = dropFirstElem(b, xferOp.in_boundsAttr());
+ auto inBoundsAttr = dropFirstElem(b, xferOp.getInBoundsAttr());
auto newXferOp = b.create<vector::TransferWriteOp>(
loc, sourceType, extracted, source, xferIndices,
AffineMapAttr::get(unpackedPermutationMap(b, xferOp)), Value(),
static Optional<int64_t>
get1dMemrefIndices(OpBuilder &b, OpTy xferOp, Value iv,
SmallVector<Value, 8> &memrefIndices) {
- auto indices = xferOp.indices();
- auto map = xferOp.permutation_map();
+ auto indices = xferOp.getIndices();
+ auto map = xferOp.getPermutationMap();
assert(xferOp.getTransferRank() > 0 && "unexpected 0-d transfer");
memrefIndices.append(indices.begin(), indices.end());
b, xferOp, iv, dim, TypeRange(xferOp.getVectorType()),
/*inBoundsCase=*/
[&](OpBuilder &b, Location loc) {
- Value val = b.create<memref::LoadOp>(loc, xferOp.source(), indices);
+ Value val =
+ b.create<memref::LoadOp>(loc, xferOp.getSource(), indices);
return b.create<vector::InsertElementOp>(loc, val, vec, iv);
},
/*outOfBoundsCase=*/
// Inititalize vector with padding value.
Location loc = xferOp.getLoc();
return b.create<vector::SplatOp>(loc, xferOp.getVectorType(),
- xferOp.padding());
+ xferOp.getPadding());
}
};
b, xferOp, iv, dim,
/*inBoundsCase=*/[&](OpBuilder &b, Location loc) {
auto val =
- b.create<vector::ExtractElementOp>(loc, xferOp.vector(), iv);
- b.create<memref::StoreOp>(loc, val, xferOp.source(), indices);
+ b.create<vector::ExtractElementOp>(loc, xferOp.getVector(), iv);
+ b.create<memref::StoreOp>(loc, val, xferOp.getSource(), indices);
});
b.create<scf::YieldOp>(loc);
}
// TODO: support 0-d corner case.
if (xferOp.getTransferRank() == 0)
return failure();
- auto map = xferOp.permutation_map();
+ auto map = xferOp.getPermutationMap();
auto memRefType = xferOp.getShapedType().template dyn_cast<MemRefType>();
if (!memRefType)
if (!dstType)
return failure();
- if (dstType == adaptor.source().getType())
- rewriter.replaceOp(bitcastOp, adaptor.source());
+ if (dstType == adaptor.getSource().getType())
+ rewriter.replaceOp(bitcastOp, adaptor.getSource());
else
rewriter.replaceOpWithNewOp<spirv::BitcastOp>(bitcastOp, dstType,
- adaptor.source());
+ adaptor.getSource());
return success();
}
LogicalResult
matchAndRewrite(vector::BroadcastOp broadcastOp, OpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
- if (broadcastOp.source().getType().isa<VectorType>() ||
+ if (broadcastOp.getSource().getType().isa<VectorType>() ||
!spirv::CompositeType::isValid(broadcastOp.getVectorType()))
return failure();
SmallVector<Value, 4> source(broadcastOp.getVectorType().getNumElements(),
- adaptor.source());
+ adaptor.getSource());
rewriter.replaceOpWithNewOp<spirv::CompositeConstructOp>(
broadcastOp, broadcastOp.getVectorType(), source);
return success();
if (!dstType)
return failure();
- if (adaptor.vector().getType().isa<spirv::ScalarType>()) {
- rewriter.replaceOp(extractOp, adaptor.vector());
+ if (adaptor.getVector().getType().isa<spirv::ScalarType>()) {
+ rewriter.replaceOp(extractOp, adaptor.getVector());
return success();
}
- int32_t id = getFirstIntValue(extractOp.position());
+ int32_t id = getFirstIntValue(extractOp.getPosition());
rewriter.replaceOpWithNewOp<spirv::CompositeExtractOp>(
- extractOp, adaptor.vector(), id);
+ extractOp, adaptor.getVector(), id);
return success();
}
};
if (!dstType)
return failure();
-
- uint64_t offset = getFirstIntValue(extractOp.offsets());
- uint64_t size = getFirstIntValue(extractOp.sizes());
- uint64_t stride = getFirstIntValue(extractOp.strides());
+ uint64_t offset = getFirstIntValue(extractOp.getOffsets());
+ uint64_t size = getFirstIntValue(extractOp.getSizes());
+ uint64_t stride = getFirstIntValue(extractOp.getStrides());
if (stride != 1)
return failure();
if (!spirv::CompositeType::isValid(fmaOp.getVectorType()))
return failure();
rewriter.replaceOpWithNewOp<spirv::GLSLFmaOp>(
- fmaOp, fmaOp.getType(), adaptor.lhs(), adaptor.rhs(), adaptor.acc());
+ fmaOp, fmaOp.getType(), adaptor.getLhs(), adaptor.getRhs(),
+ adaptor.getAcc());
return success();
}
};
// Special case for inserting scalar values into size-1 vectors.
if (insertOp.getSourceType().isIntOrFloat() &&
insertOp.getDestVectorType().getNumElements() == 1) {
- rewriter.replaceOp(insertOp, adaptor.source());
+ rewriter.replaceOp(insertOp, adaptor.getSource());
return success();
}
if (insertOp.getSourceType().isa<VectorType>() ||
!spirv::CompositeType::isValid(insertOp.getDestVectorType()))
return failure();
- int32_t id = getFirstIntValue(insertOp.position());
+ int32_t id = getFirstIntValue(insertOp.getPosition());
rewriter.replaceOpWithNewOp<spirv::CompositeInsertOp>(
- insertOp, adaptor.source(), adaptor.dest(), id);
+ insertOp, adaptor.getSource(), adaptor.getDest(), id);
return success();
}
};
if (!spirv::CompositeType::isValid(extractElementOp.getVectorType()))
return failure();
rewriter.replaceOpWithNewOp<spirv::VectorExtractDynamicOp>(
- extractElementOp, extractElementOp.getType(), adaptor.vector(),
- extractElementOp.position());
+ extractElementOp, extractElementOp.getType(), adaptor.getVector(),
+ extractElementOp.getPosition());
return success();
}
};
if (!spirv::CompositeType::isValid(insertElementOp.getDestVectorType()))
return failure();
rewriter.replaceOpWithNewOp<spirv::VectorInsertDynamicOp>(
- insertElementOp, insertElementOp.getType(), insertElementOp.dest(),
- adaptor.source(), insertElementOp.position());
+ insertElementOp, insertElementOp.getType(), insertElementOp.getDest(),
+ adaptor.getSource(), insertElementOp.getPosition());
return success();
}
};
Value srcVector = adaptor.getOperands().front();
Value dstVector = adaptor.getOperands().back();
- uint64_t stride = getFirstIntValue(insertOp.strides());
+ uint64_t stride = getFirstIntValue(insertOp.getStrides());
if (stride != 1)
return failure();
- uint64_t offset = getFirstIntValue(insertOp.offsets());
+ uint64_t offset = getFirstIntValue(insertOp.getOffsets());
if (srcVector.getType().isa<spirv::ScalarType>()) {
assert(!dstVector.getType().isa<spirv::ScalarType>());
VectorType dstVecType = op.getType();
if (!spirv::CompositeType::isValid(dstVecType))
return failure();
- SmallVector<Value, 4> source(dstVecType.getNumElements(), adaptor.input());
+ SmallVector<Value, 4> source(dstVecType.getNumElements(),
+ adaptor.getInput());
rewriter.replaceOpWithNewOp<spirv::CompositeConstructOp>(op, dstVecType,
source);
return success();
auto oldSourceType = shuffleOp.getV1VectorType();
if (oldSourceType.getNumElements() > 1) {
SmallVector<int32_t, 4> components = llvm::to_vector<4>(
- llvm::map_range(shuffleOp.mask(), [](Attribute attr) -> int32_t {
+ llvm::map_range(shuffleOp.getMask(), [](Attribute attr) -> int32_t {
return attr.cast<IntegerAttr>().getValue().getZExtValue();
}));
rewriter.replaceOpWithNewOp<spirv::VectorShuffleOp>(
- shuffleOp, newResultType, adaptor.v1(), adaptor.v2(),
+ shuffleOp, newResultType, adaptor.getV1(), adaptor.getV2(),
rewriter.getI32ArrayAttr(components));
return success();
}
- SmallVector<Value, 2> oldOperands = {adaptor.v1(), adaptor.v2()};
+ SmallVector<Value, 2> oldOperands = {adaptor.getV1(), adaptor.getV2()};
SmallVector<Value, 4> newOperands;
newOperands.reserve(oldResultType.getNumElements());
- for (const APInt &i : shuffleOp.mask().getAsValueRange<IntegerAttr>()) {
+ for (const APInt &i : shuffleOp.getMask().getAsValueRange<IntegerAttr>()) {
newOperands.push_back(oldOperands[i.getZExtValue()]);
}
rewriter.replaceOpWithNewOp<spirv::CompositeConstructOp>(
LLVM_DEBUG(DBGS() << "maybeTransferReadUser: " << *maybeTransferReadUser
<< "\n");
auto read = dyn_cast<vector::TransferReadOp>(maybeTransferReadUser);
- if (read && read.indices() == write.transferWriteOp.indices() &&
+ if (read && read.getIndices() == write.transferWriteOp.getIndices() &&
read.getVectorType() == write.transferWriteOp.getVectorType())
return HoistableRead{read, sliceOp};
}
Value v = yieldOperand.get();
if (auto write = v.getDefiningOp<vector::TransferWriteOp>()) {
// Indexing must not depend on `forOp`.
- for (Value operand : write.indices())
+ for (Value operand : write.getIndices())
if (!forOp.isDefinedOutsideOfLoop(operand))
return HoistableWrite();
read.extractSliceOp.sourceMutable().assign(
forOp.getInitArgs()[initArgNumber]);
else
- read.transferReadOp.sourceMutable().assign(
+ read.transferReadOp.getSourceMutable().assign(
forOp.getInitArgs()[initArgNumber]);
// Hoist write after.
if (write.insertSliceOp)
yieldOp->setOperand(initArgNumber, write.insertSliceOp.dest());
else
- yieldOp->setOperand(initArgNumber, write.transferWriteOp.source());
+ yieldOp->setOperand(initArgNumber, write.transferWriteOp.getSource());
// Rewrite `loop` with additional new yields.
OpBuilder b(read.transferReadOp);
- auto newForOp = cloneWithNewYields(b, forOp, read.transferReadOp.vector(),
- write.transferWriteOp.vector());
+ auto newForOp = cloneWithNewYields(b, forOp, read.transferReadOp.getVector(),
+ write.transferWriteOp.getVector());
// Transfer write has been hoisted, need to update the vector and tensor
// source. Replace the result of the loop to use the new tensor created
// outside the loop.
if (write.insertSliceOp) {
newForOp.getResult(initArgNumber)
.replaceAllUsesWith(write.insertSliceOp.getResult());
- write.transferWriteOp.sourceMutable().assign(read.extractSliceOp.result());
+ write.transferWriteOp.getSourceMutable().assign(
+ read.extractSliceOp.result());
write.insertSliceOp.destMutable().assign(read.extractSliceOp.source());
} else {
newForOp.getResult(initArgNumber)
.replaceAllUsesWith(write.transferWriteOp.getResult());
- write.transferWriteOp.sourceMutable().assign(
+ write.transferWriteOp.getSourceMutable().assign(
newForOp.getResult(initArgNumber));
}
// Always update with the newly yield tensor and vector.
- write.transferWriteOp.vectorMutable().assign(newForOp.getResults().back());
+ write.transferWriteOp.getVectorMutable().assign(newForOp.getResults().back());
}
// To hoist transfer op on tensor the logic can be significantly simplified
if (write.insertSliceOp)
LLVM_DEBUG(DBGS() << "Candidate insert_slice for hoisting: "
<< *write.insertSliceOp.getOperation() << "\n");
- if (llvm::any_of(write.transferWriteOp.indices(),
+ if (llvm::any_of(write.transferWriteOp.getIndices(),
[&forOp](Value index) {
return !forOp.isDefinedOutsideOfLoop(index);
}))
vector::TransferWriteOp transferWrite;
for (auto *sliceOp : llvm::reverse(forwardSlice)) {
auto candidateWrite = dyn_cast<vector::TransferWriteOp>(sliceOp);
- if (!candidateWrite || candidateWrite.source() != transferRead.source())
+ if (!candidateWrite ||
+ candidateWrite.getSource() != transferRead.getSource())
continue;
transferWrite = candidateWrite;
}
// 2. no other operations in the loop access the same memref except
// for transfer_read/transfer_write accessing statically disjoint
// slices.
- if (transferRead.indices() != transferWrite.indices() &&
+ if (transferRead.getIndices() != transferWrite.getIndices() &&
transferRead.getVectorType() == transferWrite.getVectorType())
return WalkResult::advance();
DominanceInfo dom(loop);
if (!dom.properlyDominates(transferRead.getOperation(), transferWrite))
return WalkResult::advance();
- for (auto &use : transferRead.source().getUses()) {
+ for (auto &use : transferRead.getSource().getUses()) {
if (!loop->isAncestor(use.getOwner()))
continue;
if (use.getOwner() == transferRead.getOperation() ||
// Rewrite `loop` with new yields by cloning and erase the original loop.
OpBuilder b(transferRead);
- auto newForOp = cloneWithNewYields(b, loop, transferRead.vector(),
- transferWrite.vector());
+ auto newForOp = cloneWithNewYields(b, loop, transferRead.getVector(),
+ transferWrite.getVector());
// Transfer write has been hoisted, need to update the written value to
// the value yielded by the newForOp.
- transferWrite.vector().replaceAllUsesWith(
+ transferWrite.getVector().replaceAllUsesWith(
newForOp.getResults().take_back()[0]);
changed = true;
if (!padValue)
return failure();
// Padding value of existing `xferOp` is unused.
- if (xferOp.hasOutOfBoundsDim() || xferOp.mask())
+ if (xferOp.hasOutOfBoundsDim() || xferOp.getMask())
return failure();
rewriter.updateRootInPlace(xferOp, [&]() {
SmallVector<bool> inBounds(xferOp.getVectorType().getRank(), false);
xferOp->setAttr(xferOp.getInBoundsAttrName(),
rewriter.getBoolArrayAttr(inBounds));
- xferOp.sourceMutable().assign(padOp.source());
- xferOp.paddingMutable().assign(padValue);
+ xferOp.getSourceMutable().assign(padOp.source());
+ xferOp.getPaddingMutable().assign(padValue);
});
return success();
SmallVector<bool> inBounds(xferOp.getVectorType().getRank(), false);
auto newXferOp = rewriter.replaceOpWithNewOp<vector::TransferWriteOp>(
- xferOp, padOp.source().getType(), xferOp.vector(), padOp.source(),
- xferOp.indices(), xferOp.permutation_mapAttr(), xferOp.mask(),
+ xferOp, padOp.source().getType(), xferOp.getVector(), padOp.source(),
+ xferOp.getIndices(), xferOp.getPermutationMapAttr(), xferOp.getMask(),
rewriter.getBoolArrayAttr(inBounds));
rewriter.replaceOp(trimPadding, newXferOp->getResult(0));
vector::TransferReadOp xferOp, PatternRewriter &rewriter) const {
// TODO: support mask.
- if (xferOp.mask())
+ if (xferOp.getMask())
return failure();
// Transfer into `view`.
- Value viewOrAlloc = xferOp.source();
+ Value viewOrAlloc = xferOp.getSource();
if (!viewOrAlloc.getDefiningOp<memref::ViewOp>() &&
!viewOrAlloc.getDefiningOp<memref::AllocOp>())
return failure();
}
}
// Ensure padding matches.
- if (maybeFillOp && xferOp.padding() != maybeFillOp.value())
+ if (maybeFillOp && xferOp.getPadding() != maybeFillOp.value())
return failure();
if (maybeFillOp)
LDBG("with maybeFillOp " << *maybeFillOp);
// When forwarding to vector.transfer_read, the attribute must be reset
// conservatively.
Value res = rewriter.create<vector::TransferReadOp>(
- xferOp.getLoc(), xferOp.getVectorType(), in, xferOp.indices(),
- xferOp.permutation_mapAttr(), xferOp.padding(), xferOp.mask(),
+ xferOp.getLoc(), xferOp.getVectorType(), in, xferOp.getIndices(),
+ xferOp.getPermutationMapAttr(), xferOp.getPadding(), xferOp.getMask(),
// in_bounds is explicitly reset
/*inBoundsAttr=*/ArrayAttr());
LogicalResult LinalgCopyVTWForwardingPattern::matchAndRewrite(
vector::TransferWriteOp xferOp, PatternRewriter &rewriter) const {
// TODO: support mask.
- if (xferOp.mask())
+ if (xferOp.getMask())
return failure();
// Transfer into `viewOrAlloc`.
- Value viewOrAlloc = xferOp.source();
+ Value viewOrAlloc = xferOp.getSource();
if (!viewOrAlloc.getDefiningOp<memref::ViewOp>() &&
!viewOrAlloc.getDefiningOp<memref::AllocOp>())
return failure();
// When forwarding to vector.transfer_write, the attribute must be reset
// conservatively.
rewriter.create<vector::TransferWriteOp>(
- xferOp.getLoc(), xferOp.vector(), out, xferOp.indices(),
- xferOp.permutation_mapAttr(), xferOp.mask(),
+ xferOp.getLoc(), xferOp.getVector(), out, xferOp.getIndices(),
+ xferOp.getPermutationMapAttr(), xferOp.getMask(),
// in_bounds is explicitly reset
/*inBoundsAttr=*/ArrayAttr());
return op.memref();
}
-static Value getMemRefOperand(vector::TransferReadOp op) { return op.source(); }
+static Value getMemRefOperand(vector::TransferReadOp op) {
+ return op.getSource();
+}
static Value getMemRefOperand(vector::TransferWriteOp op) {
- return op.source();
+ return op.getSource();
}
/// Given the permutation map of the original
transferReadOp, transferReadOp.getVectorType(), subViewOp.source(),
sourceIndices,
getPermutationMapAttr(rewriter.getContext(), subViewOp,
- transferReadOp.permutation_map()),
- transferReadOp.padding(),
- /*mask=*/Value(), transferReadOp.in_boundsAttr());
+ transferReadOp.getPermutationMap()),
+ transferReadOp.getPadding(),
+ /*mask=*/Value(), transferReadOp.getInBoundsAttr());
}
template <typename StoreOpTy>
if (transferWriteOp.getTransferRank() == 0)
return;
rewriter.replaceOpWithNewOp<vector::TransferWriteOp>(
- transferWriteOp, transferWriteOp.vector(), subViewOp.source(),
+ transferWriteOp, transferWriteOp.getVector(), subViewOp.source(),
sourceIndices,
getPermutationMapAttr(rewriter.getContext(), subViewOp,
- transferWriteOp.permutation_map()),
- transferWriteOp.in_boundsAttr());
+ transferWriteOp.getPermutationMap()),
+ transferWriteOp.getInBoundsAttr());
}
} // namespace
SmallVector<Value, 4> sourceIndices;
if (failed(resolveSourceIndices(loadOp.getLoc(), rewriter, subViewOp,
- loadOp.indices(), sourceIndices)))
+ loadOp.getIndices(), sourceIndices)))
return failure();
replaceOp(loadOp, subViewOp, sourceIndices, rewriter);
SmallVector<Value, 4> sourceIndices;
if (failed(resolveSourceIndices(storeOp.getLoc(), rewriter, subViewOp,
- storeOp.indices(), sourceIndices)))
+ storeOp.getIndices(), sourceIndices)))
return failure();
replaceOp(storeOp, subViewOp, sourceIndices, rewriter);
// Inspect constant mask index. If the index exceeds the
// dimension size, all bits are set. If the index is zero
// or less, no bits are set.
- ArrayAttr masks = m.mask_dim_sizes();
+ ArrayAttr masks = m.getMaskDimSizes();
assert(masks.size() == 1);
int64_t i = masks[0].cast<IntegerAttr>().getInt();
int64_t u = m.getType().getDimSize(0);
bool mlir::vector::checkSameValueRAW(vector::TransferWriteOp defWrite,
vector::TransferReadOp read) {
- return !defWrite.hasOutOfBoundsDim() && !defWrite.mask() && !read.mask() &&
- defWrite.indices() == read.indices() &&
+ return !defWrite.hasOutOfBoundsDim() && !defWrite.getMask() &&
+ !read.getMask() && defWrite.getIndices() == read.getIndices() &&
defWrite.getVectorType() == read.getVectorType() &&
- defWrite.permutation_map() == read.permutation_map();
+ defWrite.getPermutationMap() == read.getPermutationMap();
}
bool mlir::vector::checkSameValueWAW(vector::TransferWriteOp write,
vector::TransferWriteOp priorWrite) {
- return priorWrite.indices() == write.indices() &&
- priorWrite.mask() == write.mask() &&
+ return priorWrite.getIndices() == write.getIndices() &&
+ priorWrite.getMask() == write.getMask() &&
priorWrite.getVectorType() == write.getVectorType() &&
- priorWrite.permutation_map() == write.permutation_map();
+ priorWrite.getPermutationMap() == write.getPermutationMap();
}
bool mlir::vector::isDisjointTransferIndices(
DictionaryAttr attributes, RegionRange,
SmallVectorImpl<Type> &inferredReturnTypes) {
MultiDimReductionOp::Adaptor op(operands, attributes);
- auto vectorType = op.source().getType().cast<VectorType>();
+ auto vectorType = op.getSource().getType().cast<VectorType>();
SmallVector<int64_t> targetShape;
for (auto it : llvm::enumerate(vectorType.getShape()))
- if (!llvm::any_of(op.reduction_dims().getValue(), [&](Attribute attr) {
+ if (!llvm::any_of(op.getReductionDims().getValue(), [&](Attribute attr) {
return attr.cast<IntegerAttr>().getValue() == it.index();
}))
targetShape.push_back(it.value());
OpFoldResult MultiDimReductionOp::fold(ArrayRef<Attribute> operands) {
// Single parallel dim, this is a noop.
if (getSourceVectorType().getRank() == 1 && !isReducedDim(0))
- return source();
+ return getSource();
return {};
}
return emitOpError("unsupported reduction rank: ") << rank;
// Verify supported reduction kind.
- Type eltType = dest().getType();
- if (!isSupportedCombiningKind(kind(), eltType))
+ Type eltType = getDest().getType();
+ if (!isSupportedCombiningKind(getKind(), eltType))
return emitOpError("unsupported reduction type '")
- << eltType << "' for kind '" << stringifyCombiningKind(kind())
+ << eltType << "' for kind '" << stringifyCombiningKind(getKind())
<< "'";
// Verify optional accumulator.
- if (acc()) {
- if (kind() != CombiningKind::ADD && kind() != CombiningKind::MUL)
+ if (getAcc()) {
+ if (getKind() != CombiningKind::ADD && getKind() != CombiningKind::MUL)
return emitOpError("no accumulator for reduction kind: ")
- << stringifyCombiningKind(kind());
+ << stringifyCombiningKind(getKind());
if (!eltType.isa<FloatType>())
return emitOpError("no accumulator for type: ") << eltType;
}
void ReductionOp::print(OpAsmPrinter &p) {
p << " ";
- kindAttr().print(p);
- p << ", " << vector();
- if (acc())
- p << ", " << acc();
- p << " : " << vector().getType() << " into " << dest().getType();
+ getKindAttr().print(p);
+ p << ", " << getVector();
+ if (getAcc())
+ p << ", " << getAcc();
+ p << " : " << getVector().getType() << " into " << getDest().getType();
}
Value mlir::vector::getVectorReductionOp(arith::AtomicRMWKind op,
attrs.push_back(attr);
auto dictAttr = DictionaryAttr::get(getContext(), attrs);
- p << " " << dictAttr << " " << lhs() << ", ";
- p << rhs() << ", " << acc();
- if (masks().size() == 2)
- p << ", " << masks();
+ p << " " << dictAttr << " " << getLhs() << ", ";
+ p << getRhs() << ", " << getAcc();
+ if (getMasks().size() == 2)
+ p << ", " << getMasks();
p.printOptionalAttrDict((*this)->getAttrs(), attrNames);
- p << " : " << lhs().getType() << ", " << rhs().getType() << " into "
+ p << " : " << getLhs().getType() << ", " << getRhs().getType() << " into "
<< getResultType();
}
auto resType = getResultType();
// Verify that an indexing map was specified for each vector operand.
- if (indexing_maps().size() != 3)
+ if (getIndexingMaps().size() != 3)
return emitOpError("expected an indexing map for each vector operand");
// Verify that each index map has 'numIterators' inputs, no symbols, and
// that the number of map outputs equals the rank of its associated
// vector operand.
- unsigned numIterators = iterator_types().getValue().size();
- for (const auto &it : llvm::enumerate(indexing_maps())) {
+ unsigned numIterators = getIteratorTypes().getValue().size();
+ for (const auto &it : llvm::enumerate(getIndexingMaps())) {
auto index = it.index();
auto map = it.value();
if (map.getNumSymbols() != 0)
// Verify supported combining kind.
auto vectorType = resType.dyn_cast<VectorType>();
auto elementType = vectorType ? vectorType.getElementType() : resType;
- if (!isSupportedCombiningKind(kind(), elementType))
+ if (!isSupportedCombiningKind(getKind(), elementType))
return emitOpError("unsupported contraction type");
return success();
auto resVectorType = getResultType().dyn_cast<VectorType>();
SmallVector<AffineMap, 4> indexingMaps(getIndexingMaps());
SmallVector<int64_t, 2> iterationShape;
- for (const auto &it : llvm::enumerate(iterator_types())) {
+ for (const auto &it : llvm::enumerate(getIteratorTypes())) {
// Search lhs/rhs map results for 'targetExpr'.
auto targetExpr = getAffineDimExpr(it.index(), getContext());
auto iteratorTypeName = it.value().cast<StringAttr>().getValue();
void ContractionOp::getIterationIndexMap(
std::vector<DenseMap<int64_t, int64_t>> &iterationIndexMap) {
- unsigned numMaps = indexing_maps().size();
+ unsigned numMaps = getIndexingMaps().size();
iterationIndexMap.resize(numMaps);
- for (const auto &it : llvm::enumerate(indexing_maps())) {
+ for (const auto &it : llvm::enumerate(getIndexingMaps())) {
auto index = it.index();
auto map = it.value();
for (unsigned i = 0, e = map.getNumResults(); i < e; ++i) {
std::vector<std::pair<int64_t, int64_t>> ContractionOp::getContractingDimMap() {
SmallVector<AffineMap, 4> indexingMaps(getIndexingMaps());
- return getDimMap(indexingMaps, iterator_types(),
+ return getDimMap(indexingMaps, getIteratorTypes(),
getReductionIteratorTypeName(), getContext());
}
std::vector<std::pair<int64_t, int64_t>> ContractionOp::getBatchDimMap() {
SmallVector<AffineMap, 4> indexingMaps(getIndexingMaps());
- return getDimMap(indexingMaps, iterator_types(),
+ return getDimMap(indexingMaps, getIteratorTypes(),
getParallelIteratorTypeName(), getContext());
}
if (!contractionOp)
return vector::ContractionOp();
if (auto maybeZero = dyn_cast_or_null<arith::ConstantOp>(
- contractionOp.acc().getDefiningOp())) {
+ contractionOp.getAcc().getDefiningOp())) {
if (maybeZero.getValue() ==
- rewriter.getZeroAttr(contractionOp.acc().getType())) {
+ rewriter.getZeroAttr(contractionOp.getAcc().getType())) {
BlockAndValueMapping bvm;
- bvm.map(contractionOp.acc(), otherOperand);
+ bvm.map(contractionOp.getAcc(), otherOperand);
auto newContraction =
cast<vector::ContractionOp>(rewriter.clone(*contractionOp, bvm));
rewriter.replaceOp(addOp, newContraction.getResult());
LogicalResult vector::ExtractElementOp::verify() {
VectorType vectorType = getVectorType();
if (vectorType.getRank() == 0) {
- if (position())
+ if (getPosition())
return emitOpError("expected position to be empty with 0-D vector");
return success();
}
if (vectorType.getRank() != 1)
return emitOpError("unexpected >1 vector rank");
- if (!position())
+ if (!getPosition())
return emitOpError("expected position for 1-D vector");
return success();
}
RegionRange,
SmallVectorImpl<Type> &inferredReturnTypes) {
ExtractOp::Adaptor op(operands, attributes);
- auto vectorType = op.vector().getType().cast<VectorType>();
- if (static_cast<int64_t>(op.position().size()) == vectorType.getRank()) {
+ auto vectorType = op.getVector().getType().cast<VectorType>();
+ if (static_cast<int64_t>(op.getPosition().size()) == vectorType.getRank()) {
inferredReturnTypes.push_back(vectorType.getElementType());
} else {
- auto n = std::min<size_t>(op.position().size(), vectorType.getRank() - 1);
+ auto n =
+ std::min<size_t>(op.getPosition().size(), vectorType.getRank() - 1);
inferredReturnTypes.push_back(VectorType::get(
vectorType.getShape().drop_front(n), vectorType.getElementType()));
}
}
LogicalResult vector::ExtractOp::verify() {
- auto positionAttr = position().getValue();
+ auto positionAttr = getPosition().getValue();
if (positionAttr.size() > static_cast<unsigned>(getVectorType().getRank()))
return emitOpError(
"expected position attribute of rank smaller than vector rank");
/// Fold the result of chains of ExtractOp in place by simply concatenating the
/// positions.
static LogicalResult foldExtractOpFromExtractChain(ExtractOp extractOp) {
- if (!extractOp.vector().getDefiningOp<ExtractOp>())
+ if (!extractOp.getVector().getDefiningOp<ExtractOp>())
return failure();
SmallVector<int64_t, 4> globalPosition;
ExtractOp currentOp = extractOp;
- auto extrPos = extractVector<int64_t>(currentOp.position());
+ auto extrPos = extractVector<int64_t>(currentOp.getPosition());
globalPosition.append(extrPos.rbegin(), extrPos.rend());
- while (ExtractOp nextOp = currentOp.vector().getDefiningOp<ExtractOp>()) {
+ while (ExtractOp nextOp = currentOp.getVector().getDefiningOp<ExtractOp>()) {
currentOp = nextOp;
- auto extrPos = extractVector<int64_t>(currentOp.position());
+ auto extrPos = extractVector<int64_t>(currentOp.getPosition());
globalPosition.append(extrPos.rbegin(), extrPos.rend());
}
- extractOp.setOperand(currentOp.vector());
+ extractOp.setOperand(currentOp.getVector());
// OpBuilder is only used as a helper to build an I64ArrayAttr.
OpBuilder b(extractOp.getContext());
std::reverse(globalPosition.begin(), globalPosition.end());
ExtractFromInsertTransposeChainState::ExtractFromInsertTransposeChainState(
ExtractOp e)
: extractOp(e), vectorRank(extractOp.getVectorType().getRank()),
- extractedRank(extractOp.position().size()) {
+ extractedRank(extractOp.getPosition().size()) {
assert(vectorRank >= extractedRank && "extracted pos overflow");
sentinels.reserve(vectorRank - extractedRank);
for (int64_t i = 0, e = vectorRank - extractedRank; i < e; ++i)
sentinels.push_back(-(i + 1));
- extractPosition = extractVector<int64_t>(extractOp.position());
+ extractPosition = extractVector<int64_t>(extractOp.getPosition());
llvm::append_range(extractPosition, sentinels);
}
LogicalResult ExtractFromInsertTransposeChainState::handleTransposeOp() {
if (!nextTransposeOp)
return failure();
- auto permutation = extractVector<unsigned>(nextTransposeOp.transp());
+ auto permutation = extractVector<unsigned>(nextTransposeOp.getTransp());
AffineMap m = inversePermutation(
AffineMap::getPermutationMap(permutation, extractOp.getContext()));
extractPosition = applyPermutationMap(m, makeArrayRef(extractPosition));
LogicalResult
ExtractFromInsertTransposeChainState::handleInsertOpWithMatchingPos(
Value &res) {
- auto insertedPos = extractVector<int64_t>(nextInsertOp.position());
+ auto insertedPos = extractVector<int64_t>(nextInsertOp.getPosition());
if (makeArrayRef(insertedPos) !=
llvm::makeArrayRef(extractPosition).take_front(extractedRank))
return failure();
// Case 2.a. early-exit fold.
- res = nextInsertOp.source();
+ res = nextInsertOp.getSource();
// Case 2.b. if internal transposition is present, canFold will be false.
return success();
}
/// This method updates the internal state.
LogicalResult
ExtractFromInsertTransposeChainState::handleInsertOpWithPrefixPos(Value &res) {
- auto insertedPos = extractVector<int64_t>(nextInsertOp.position());
+ auto insertedPos = extractVector<int64_t>(nextInsertOp.getPosition());
if (!isContainedWithin(insertedPos, extractPosition))
return failure();
// Set leading dims to zero.
extractPosition.begin() + insertedPos.size());
extractedRank = extractPosition.size() - sentinels.size();
// Case 3.a. early-exit fold (break and delegate to post-while path).
- res = nextInsertOp.source();
+ res = nextInsertOp.getSource();
// Case 3.b. if internal transposition is present, canFold will be false.
return success();
}
Value ExtractFromInsertTransposeChainState::tryToFoldExtractOpInPlace(
Value source) {
// If we can't fold (either internal transposition, or nothing to fold), bail.
- bool nothingToFold = (source == extractOp.vector());
+ bool nothingToFold = (source == extractOp.getVector());
if (nothingToFold || !canFold())
return Value();
// Otherwise, fold by updating the op inplace and return its result.
OpBuilder b(extractOp.getContext());
extractOp->setAttr(
- extractOp.positionAttrName(),
+ extractOp.getPositionAttrName(),
b.getI64ArrayAttr(
makeArrayRef(extractPosition).take_front(extractedRank)));
- extractOp.vectorMutable().assign(source);
+ extractOp.getVectorMutable().assign(source);
return extractOp.getResult();
}
/// Iterate over producing insert and transpose ops until we find a fold.
Value ExtractFromInsertTransposeChainState::fold() {
- Value valueToExtractFrom = extractOp.vector();
+ Value valueToExtractFrom = extractOp.getVector();
updateStateForNextIteration(valueToExtractFrom);
while (nextInsertOp || nextTransposeOp) {
// Case 1. If we hit a transpose, just compose the map and iterate.
// Invariant: insert + transpose do not change rank, we can always compose.
if (succeeded(handleTransposeOp())) {
- valueToExtractFrom = nextTransposeOp.vector();
+ valueToExtractFrom = nextTransposeOp.getVector();
updateStateForNextIteration(valueToExtractFrom);
continue;
}
// Case 4: extractPositionRef intersects insertedPosRef on non-sentinel
// values. This is a more difficult case and we bail.
- auto insertedPos = extractVector<int64_t>(nextInsertOp.position());
+ auto insertedPos = extractVector<int64_t>(nextInsertOp.getPosition());
if (isContainedWithin(extractPosition, insertedPos) ||
intersectsWhereNonNegative(extractPosition, insertedPos))
return Value();
// Case 5: No intersection, we forward the extract to insertOp.dest().
- valueToExtractFrom = nextInsertOp.dest();
+ valueToExtractFrom = nextInsertOp.getDest();
updateStateForNextIteration(valueToExtractFrom);
}
// If after all this we can fold, go for it.
/// Fold extractOp with scalar result coming from BroadcastOp or SplatOp.
static Value foldExtractFromBroadcast(ExtractOp extractOp) {
- Operation *defOp = extractOp.vector().getDefiningOp();
+ Operation *defOp = extractOp.getVector().getDefiningOp();
if (!defOp || !isa<vector::BroadcastOp, SplatOp>(defOp))
return Value();
Value source = defOp->getOperand(0);
unsigned broadcastSrcRank = getRank(source.getType());
unsigned extractResultRank = getRank(extractOp.getType());
if (extractResultRank < broadcastSrcRank) {
- auto extractPos = extractVector<int64_t>(extractOp.position());
+ auto extractPos = extractVector<int64_t>(extractOp.getPosition());
unsigned rankDiff = broadcastSrcRank - extractResultRank;
extractPos.erase(
extractPos.begin(),
// Fold extractOp with source coming from ShapeCast op.
static Value foldExtractFromShapeCast(ExtractOp extractOp) {
- auto shapeCastOp = extractOp.vector().getDefiningOp<vector::ShapeCastOp>();
+ auto shapeCastOp = extractOp.getVector().getDefiningOp<vector::ShapeCastOp>();
if (!shapeCastOp)
return Value();
// Get the nth dimension size starting from lowest dimension.
}
// Extract the strides associated with the extract op vector source. Then use
// this to calculate a linearized position for the extract.
- auto extractedPos = extractVector<int64_t>(extractOp.position());
+ auto extractedPos = extractVector<int64_t>(extractOp.getPosition());
std::reverse(extractedPos.begin(), extractedPos.end());
SmallVector<int64_t, 4> strides;
int64_t stride = 1;
OpBuilder b(extractOp.getContext());
extractOp->setAttr(ExtractOp::getPositionAttrStrName(),
b.getI64ArrayAttr(newPosition));
- extractOp.setOperand(shapeCastOp.source());
+ extractOp.setOperand(shapeCastOp.getSource());
return extractOp.getResult();
}
/// Fold an ExtractOp from ExtractStridedSliceOp.
static Value foldExtractFromExtractStrided(ExtractOp extractOp) {
auto extractStridedSliceOp =
- extractOp.vector().getDefiningOp<vector::ExtractStridedSliceOp>();
+ extractOp.getVector().getDefiningOp<vector::ExtractStridedSliceOp>();
if (!extractStridedSliceOp)
return Value();
// Return if 'extractStridedSliceOp' has non-unit strides.
return Value();
// Trim offsets for dimensions fully extracted.
- auto sliceOffsets = extractVector<int64_t>(extractStridedSliceOp.offsets());
+ auto sliceOffsets =
+ extractVector<int64_t>(extractStridedSliceOp.getOffsets());
while (!sliceOffsets.empty()) {
size_t lastOffset = sliceOffsets.size() - 1;
if (sliceOffsets.back() != 0 ||
if (destinationRank >
extractStridedSliceOp.getVectorType().getRank() - sliceOffsets.size())
return Value();
- auto extractedPos = extractVector<int64_t>(extractOp.position());
+ auto extractedPos = extractVector<int64_t>(extractOp.getPosition());
assert(extractedPos.size() >= sliceOffsets.size());
for (size_t i = 0, e = sliceOffsets.size(); i < e; i++)
extractedPos[i] = extractedPos[i] + sliceOffsets[i];
- extractOp.vectorMutable().assign(extractStridedSliceOp.vector());
+ extractOp.getVectorMutable().assign(extractStridedSliceOp.getVector());
// OpBuilder is only used as a helper to build an I64ArrayAttr.
OpBuilder b(extractOp.getContext());
extractOp->setAttr(ExtractOp::getPositionAttrStrName(),
int64_t destinationRank = op.getType().isa<VectorType>()
? op.getType().cast<VectorType>().getRank()
: 0;
- auto insertOp = op.vector().getDefiningOp<InsertStridedSliceOp>();
+ auto insertOp = op.getVector().getDefiningOp<InsertStridedSliceOp>();
while (insertOp) {
int64_t insertRankDiff = insertOp.getDestVectorType().getRank() -
insertOp.getSourceVectorType().getRank();
if (destinationRank > insertOp.getSourceVectorType().getRank())
return Value();
- auto insertOffsets = extractVector<int64_t>(insertOp.offsets());
- auto extractOffsets = extractVector<int64_t>(op.position());
+ auto insertOffsets = extractVector<int64_t>(insertOp.getOffsets());
+ auto extractOffsets = extractVector<int64_t>(op.getPosition());
- if (llvm::any_of(insertOp.strides(), [](Attribute attr) {
+ if (llvm::any_of(insertOp.getStrides(), [](Attribute attr) {
return attr.cast<IntegerAttr>().getInt() != 1;
}))
return Value();
insertRankDiff))
return Value();
}
- op.vectorMutable().assign(insertOp.source());
+ op.getVectorMutable().assign(insertOp.getSource());
// OpBuilder is only used as a helper to build an I64ArrayAttr.
OpBuilder b(op.getContext());
op->setAttr(ExtractOp::getPositionAttrStrName(),
}
// If the chunk extracted is disjoint from the chunk inserted, keep
// looking in the insert chain.
- insertOp = insertOp.dest().getDefiningOp<InsertStridedSliceOp>();
+ insertOp = insertOp.getDest().getDefiningOp<InsertStridedSliceOp>();
}
return Value();
}
OpFoldResult ExtractOp::fold(ArrayRef<Attribute>) {
- if (position().empty())
- return vector();
+ if (getPosition().empty())
+ return getVector();
if (succeeded(foldExtractOpFromExtractChain(*this)))
return getResult();
if (auto res = ExtractFromInsertTransposeChainState(*this).fold())
LogicalResult matchAndRewrite(ExtractOp extractOp,
PatternRewriter &rewriter) const override {
- Operation *defOp = extractOp.vector().getDefiningOp();
+ Operation *defOp = extractOp.getVector().getDefiningOp();
if (!defOp || !isa<vector::BroadcastOp, SplatOp>(defOp))
return failure();
Value source = defOp->getOperand(0);
PatternRewriter &rewriter) const override {
// Return if 'extractStridedSliceOp' operand is not defined by a
// ConstantOp.
- auto constantOp = extractOp.
vector().getDefiningOp<arith::ConstantOp>();
+ auto constantOp = extractOp.
getVector().getDefiningOp<arith::ConstantOp>();
if (!constantOp)
return failure();
auto dense = constantOp.getValue().dyn_cast<SplatElementsAttr>();
if (getSourceVectorType().getDimSize(i) != getResultType().getDimSize(i))
numId++;
}
- if (numId != ids().size())
+ if (numId != getIds().size())
return emitOpError("expected number of ids must match the number of "
"dimensions distributed");
return success();
}
OpFoldResult ExtractMapOp::fold(ArrayRef<Attribute> operands) {
- auto insert = vector().getDefiningOp<vector::InsertMapOp>();
- if (insert == nullptr || getType() != insert.vector().getType() ||
- ids() != insert.ids())
+ auto insert = getVector().getDefiningOp<vector::InsertMapOp>();
+ if (insert == nullptr || getType() != insert.getVector().getType() ||
+ getIds() != insert.getIds())
return {};
- return insert.vector();
+ return insert.getVector();
}
void ExtractMapOp::getMultiplicity(SmallVectorImpl<int64_t> &multiplicity) {
OpFoldResult BroadcastOp::fold(ArrayRef<Attribute> operands) {
if (getSourceType() == getVectorType())
- return source();
+ return getSource();
if (!operands[0])
return {};
auto vectorType = getVectorType();
LogicalResult matchAndRewrite(BroadcastOp broadcastOp,
PatternRewriter &rewriter) const override {
- auto srcBroadcast = broadcastOp.source().getDefiningOp<BroadcastOp>();
+ auto srcBroadcast = broadcastOp.getSource().getDefiningOp<BroadcastOp>();
if (!srcBroadcast)
return failure();
rewriter.replaceOpWithNewOp<BroadcastOp>(
- broadcastOp, broadcastOp.getVectorType(), srcBroadcast.source());
+ broadcastOp, broadcastOp.getVectorType(), srcBroadcast.getSource());
return success();
}
};
return emitOpError("dimension mismatch");
}
// Verify mask length.
- auto maskAttr = mask().getValue();
+ auto maskAttr = getMask().getValue();
int64_t maskLength = maskAttr.size();
if (maskLength <= 0)
return emitOpError("invalid mask length");
RegionRange,
SmallVectorImpl<Type> &inferredReturnTypes) {
ShuffleOp::Adaptor op(operands, attributes);
- auto v1Type = op.v1().getType().cast<VectorType>();
+ auto v1Type = op.getV1().getType().cast<VectorType>();
// Construct resulting type: leading dimension matches mask length,
// all trailing dimensions match the operands.
SmallVector<int64_t, 4> shape;
shape.reserve(v1Type.getRank());
- shape.push_back(std::max<size_t>(1, op.mask().size()));
+ shape.push_back(std::max<size_t>(1, op.getMask().size()));
llvm::append_range(shape, v1Type.getShape().drop_front());
inferredReturnTypes.push_back(
VectorType::get(shape, v1Type.getElementType()));
SmallVector<Attribute> results;
auto lhsElements = lhs.cast<DenseElementsAttr>().getValues<Attribute>();
auto rhsElements = rhs.cast<DenseElementsAttr>().getValues<Attribute>();
- for (const auto &index : this->mask().getAsValueRange<IntegerAttr>()) {
+ for (const auto &index : this->getMask().getAsValueRange<IntegerAttr>()) {
int64_t i = index.getZExtValue();
if (i >= lhsSize) {
results.push_back(rhsElements[i - lhsSize]);
LogicalResult InsertElementOp::verify() {
auto dstVectorType = getDestVectorType();
if (dstVectorType.getRank() == 0) {
- if (position())
+ if (getPosition())
return emitOpError("expected position to be empty with 0-D vector");
return success();
}
if (dstVectorType.getRank() != 1)
return emitOpError("unexpected >1 vector rank");
- if (!position())
+ if (!getPosition())
return emitOpError("expected position for 1-D vector");
return success();
}
}
LogicalResult InsertOp::verify() {
- auto positionAttr = position().getValue();
+ auto positionAttr = getPosition().getValue();
auto destVectorType = getDestVectorType();
if (positionAttr.size() > static_cast<unsigned>(destVectorType.getRank()))
return emitOpError(
srcVecType.getNumElements())
return failure();
rewriter.replaceOpWithNewOp<BroadcastOp>(
- insertOp, insertOp.getDestVectorType(), insertOp.source());
+ insertOp, insertOp.getDestVectorType(), insertOp.getSource());
return success();
}
};
// value. This happens when the source and destination vectors have identical
// sizes.
OpFoldResult vector::InsertOp::fold(ArrayRef<Attribute> operands) {
- if (position().empty())
- return source();
+ if (getPosition().empty())
+ return getSource();
return {};
}
if (getResultType().getDimSize(i) != getSourceVectorType().getDimSize(i))
numId++;
}
- if (numId != ids().size())
+ if (numId != getIds().size())
return emitOpError("expected number of ids must match the number of "
"dimensions distributed");
return success();
LogicalResult InsertStridedSliceOp::verify() {
auto sourceVectorType = getSourceVectorType();
auto destVectorType = getDestVectorType();
- auto offsets = offsetsAttr();
- auto strides = stridesAttr();
+ auto offsets = getOffsetsAttr();
+ auto strides = getStridesAttr();
if (offsets.size() != static_cast<unsigned>(destVectorType.getRank()))
return emitOpError(
"expected offsets of same size as destination vector rank");
OpFoldResult InsertStridedSliceOp::fold(ArrayRef<Attribute> operands) {
if (getSourceVectorType() == getDestVectorType())
- return source();
+ return getSource();
return {};
}
}
void OuterProductOp::print(OpAsmPrinter &p) {
- p << " " << lhs() << ", " << rhs();
- if (!acc().empty()) {
- p << ", " << acc();
+ p << " " << getLhs() << ", " << getRhs();
+ if (!getAcc().empty()) {
+ p << ", " << getAcc();
p.printOptionalAttrDict((*this)->getAttrs());
}
- p << " : " << lhs().getType() << ", " << rhs().getType();
+ p << " : " << getLhs().getType() << ", " << getRhs().getType();
}
ParseResult OuterProductOp::parse(OpAsmParser &parser, OperationState &result) {
return emitOpError("expected operand #3 of same type as result type");
// Verify supported combining kind.
- if (!isSupportedCombiningKind(kind(), vRES.getElementType()))
+ if (!isSupportedCombiningKind(getKind(), vRES.getElementType()))
return emitOpError("unsupported outerproduct type");
return success();
auto isDefByConstant = [](Value operand) {
return isa_and_nonnull<arith::ConstantIndexOp>(operand.getDefiningOp());
};
- if (llvm::all_of(input_shape(), isDefByConstant) &&
- llvm::all_of(output_shape(), isDefByConstant)) {
+ if (llvm::all_of(getInputShape(), isDefByConstant) &&
+ llvm::all_of(getOutputShape(), isDefByConstant)) {
int64_t numInputElements = 1;
- for (auto operand : input_shape())
+ for (auto operand : getInputShape())
numInputElements *=
cast<arith::ConstantIndexOp>(operand.getDefiningOp()).value();
int64_t numOutputElements = 1;
- for (auto operand : output_shape())
+ for (auto operand : getOutputShape())
numOutputElements *=
cast<arith::ConstantIndexOp>(operand.getDefiningOp()).value();
if (numInputElements != numOutputElements)
}
void ReshapeOp::getFixedVectorSizes(SmallVectorImpl<int64_t> &results) {
- populateFromInt64AttrArray(fixed_vector_sizes(), results);
+ populateFromInt64AttrArray(getFixedVectorSizes(), results);
}
//===----------------------------------------------------------------------===//
LogicalResult ExtractStridedSliceOp::verify() {
auto type = getVectorType();
- auto offsets = offsetsAttr();
- auto sizes = sizesAttr();
- auto strides = stridesAttr();
+ auto offsets = getOffsetsAttr();
+ auto sizes = getSizesAttr();
+ auto strides = getStridesAttr();
if (offsets.size() != sizes.size() || offsets.size() != strides.size())
return emitOpError("expected offsets, sizes and strides attributes of same size");
auto getElement = [](ArrayAttr array, int idx) {
return array[idx].cast<IntegerAttr>().getInt();
};
- ArrayAttr extractOffsets = op.offsets();
- ArrayAttr extractStrides = op.strides();
- ArrayAttr extractSizes = op.sizes();
- auto insertOp = op.vector().getDefiningOp<InsertStridedSliceOp>();
+ ArrayAttr extractOffsets = op.getOffsets();
+ ArrayAttr extractStrides = op.getStrides();
+ ArrayAttr extractSizes = op.getSizes();
+ auto insertOp = op.getVector().getDefiningOp<InsertStridedSliceOp>();
while (insertOp) {
if (op.getVectorType().getRank() !=
insertOp.getSourceVectorType().getRank())
return failure();
- ArrayAttr insertOffsets = insertOp.offsets();
- ArrayAttr insertStrides = insertOp.strides();
+ ArrayAttr insertOffsets = insertOp.getOffsets();
+ ArrayAttr insertStrides = insertOp.getStrides();
// If the rank of extract is greater than the rank of insert, we are likely
// extracting a partial chunk of the vector inserted.
if (extractOffsets.size() > insertOffsets.size())
}
// The extract element chunk is a subset of the insert element.
if (!disjoint && !patialoverlap) {
- op.setOperand(insertOp.source());
+ op.setOperand(insertOp.getSource());
// OpBuilder is only used as a helper to build an I64ArrayAttr.
OpBuilder b(op.getContext());
op->setAttr(ExtractStridedSliceOp::getOffsetsAttrStrName(),
// If the chunk extracted is disjoint from the chunk inserted, keep looking
// in the insert chain.
if (disjoint)
- insertOp = insertOp.dest().getDefiningOp<InsertStridedSliceOp>();
+ insertOp = insertOp.getDest().getDefiningOp<InsertStridedSliceOp>();
else {
// The extracted vector partially overlap the inserted vector, we cannot
// fold.
OpFoldResult ExtractStridedSliceOp::fold(ArrayRef<Attribute> operands) {
if (getVectorType() == getResult().getType())
- return vector();
+ return getVector();
if (succeeded(foldExtractStridedOpFromInsertChain(*this)))
return getResult();
return {};
}
void ExtractStridedSliceOp::getOffsets(SmallVectorImpl<int64_t> &results) {
- populateFromInt64AttrArray(offsets(), results);
+ populateFromInt64AttrArray(getOffsets(), results);
}
namespace {
PatternRewriter &rewriter) const override {
// Return if 'extractStridedSliceOp' operand is not defined by a
// ConstantMaskOp.
- auto *defOp = extractStridedSliceOp.vector().getDefiningOp();
+ auto *defOp = extractStridedSliceOp.getVector().getDefiningOp();
auto constantMaskOp = dyn_cast_or_null<ConstantMaskOp>(defOp);
if (!constantMaskOp)
return failure();
return failure();
// Gather constant mask dimension sizes.
SmallVector<int64_t, 4> maskDimSizes;
- populateFromInt64AttrArray(constantMaskOp.mask_dim_sizes(), maskDimSizes);
+ populateFromInt64AttrArray(constantMaskOp.getMaskDimSizes(), maskDimSizes);
// Gather strided slice offsets and sizes.
SmallVector<int64_t, 4> sliceOffsets;
- populateFromInt64AttrArray(extractStridedSliceOp.offsets(), sliceOffsets);
+ populateFromInt64AttrArray(extractStridedSliceOp.getOffsets(),
+ sliceOffsets);
SmallVector<int64_t, 4> sliceSizes;
- populateFromInt64AttrArray(extractStridedSliceOp.sizes(), sliceSizes);
+ populateFromInt64AttrArray(extractStridedSliceOp.getSizes(), sliceSizes);
// Compute slice of vector mask region.
SmallVector<int64_t, 4> sliceMaskDimSizes;
// Return if 'extractStridedSliceOp' operand is not defined by a
// ConstantOp.
auto constantOp =
- extractStridedSliceOp.
vector().getDefiningOp<arith::ConstantOp>();
+ extractStridedSliceOp.
getVector().getDefiningOp<arith::ConstantOp>();
if (!constantOp)
return failure();
auto dense = constantOp.getValue().dyn_cast<SplatElementsAttr>();
LogicalResult matchAndRewrite(ExtractStridedSliceOp op,
PatternRewriter &rewriter) const override {
- auto broadcast = op.vector().getDefiningOp<BroadcastOp>();
+ auto broadcast = op.getVector().getDefiningOp<BroadcastOp>();
if (!broadcast)
return failure();
- auto srcVecType = broadcast.source().getType().dyn_cast<VectorType>();
+ auto srcVecType = broadcast.getSource().getType().dyn_cast<VectorType>();
unsigned srcRrank = srcVecType ? srcVecType.getRank() : 0;
auto dstVecType = op.getType().cast<VectorType>();
unsigned dstRank = dstVecType.getRank();
break;
}
}
- Value source = broadcast.source();
+ Value source = broadcast.getSource();
if (!lowerDimMatch) {
// The inner dimensions don't match, it means we need to extract from the
// source of the orignal broadcast and then broadcast the extracted value.
source = rewriter.create<ExtractStridedSliceOp>(
op->getLoc(), source,
- getI64SubArray(op.offsets(), /* dropFront=*/rankDiff),
- getI64SubArray(op.sizes(), /* dropFront=*/rankDiff),
- getI64SubArray(op.strides(), /* dropFront=*/rankDiff));
+ getI64SubArray(op.getOffsets(), /* dropFront=*/rankDiff),
+ getI64SubArray(op.getSizes(), /* dropFront=*/rankDiff),
+ getI64SubArray(op.getStrides(), /* dropFront=*/rankDiff));
}
rewriter.replaceOpWithNewOp<BroadcastOp>(op, op.getType(), source);
return success();
LogicalResult matchAndRewrite(ExtractStridedSliceOp op,
PatternRewriter &rewriter) const override {
- auto splat = op.vector().getDefiningOp<SplatOp>();
+ auto splat = op.getVector().getDefiningOp<SplatOp>();
if (!splat)
return failure();
- rewriter.replaceOpWithNewOp<SplatOp>(op, op.getType(), splat.input());
+ rewriter.replaceOpWithNewOp<SplatOp>(op, op.getType(), splat.getInput());
return success();
}
};
}
void TransferReadOp::print(OpAsmPrinter &p) {
- p << " " << source() << "[" << indices() << "], " << padding();
- if (mask())
- p << ", " << mask();
+ p << " " << getSource() << "[" << getIndices() << "], " << getPadding();
+ if (getMask())
+ p << ", " << getMask();
printTransferAttrs(p, *this);
p << " : " << getShapedType() << ", " << getVectorType();
}
ShapedType shapedType = getShapedType();
VectorType vectorType = getVectorType();
VectorType maskType = getMaskType();
- auto paddingType = padding().getType();
- auto permutationMap = permutation_map();
+ auto paddingType = getPadding().getType();
+ auto permutationMap = getPermutationMap();
auto sourceElementType = shapedType.getElementType();
- if (static_cast<int64_t>(indices().size()) != shapedType.getRank())
+ if (static_cast<int64_t>(getIndices().size()) != shapedType.getRank())
return emitOpError("requires ") << shapedType.getRank() << " indices";
if (failed(verifyTransferOp(cast<VectorTransferOpInterface>(getOperation()),
shapedType, vectorType, maskType, permutationMap,
- in_bounds() ? *in_bounds() : ArrayAttr())))
+ getInBounds() ? *getInBounds() : ArrayAttr())))
return failure();
if (auto sourceVectorElementType = sourceElementType.dyn_cast<VectorType>()) {
// `op.indices()[indicesIdx] + vectorType < dim(op.source(), indicesIdx)`
if (op.getShapedType().isDynamicDim(indicesIdx))
return false;
- Value index = op.indices()[indicesIdx];
+ Value index = op.getIndices()[indicesIdx];
auto cstOp = index.getDefiningOp<arith::ConstantIndexOp>();
if (!cstOp)
return false;
// TODO: Be less conservative.
if (op.getTransferRank() == 0)
return failure();
- AffineMap permutationMap = op.permutation_map();
+ AffineMap permutationMap = op.getPermutationMap();
bool changed = false;
SmallVector<bool, 4> newInBounds;
newInBounds.reserve(op.getTransferRank());
static Value foldRAW(TransferReadOp readOp) {
if (!readOp.getShapedType().isa<RankedTensorType>())
return {};
- auto defWrite = readOp.source().getDefiningOp<vector::TransferWriteOp>();
+ auto defWrite = readOp.getSource().getDefiningOp<vector::TransferWriteOp>();
while (defWrite) {
if (checkSameValueRAW(defWrite, readOp))
- return defWrite.vector();
+ return defWrite.getVector();
if (!isDisjointTransferIndices(
cast<VectorTransferOpInterface>(defWrite.getOperation()),
cast<VectorTransferOpInterface>(readOp.getOperation())))
break;
- defWrite = defWrite.source().getDefiningOp<vector::TransferWriteOp>();
+ defWrite = defWrite.getSource().getDefiningOp<vector::TransferWriteOp>();
}
return {};
}
SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
&effects) {
if (getShapedType().isa<MemRefType>())
- effects.emplace_back(MemoryEffects::Read::get(), source(),
+ effects.emplace_back(MemoryEffects::Read::get(), getSource(),
SideEffects::DefaultResource::get());
}
return failure();
if (xferOp.hasOutOfBoundsDim())
return failure();
- if (!xferOp.permutation_map().isIdentity())
+ if (!xferOp.getPermutationMap().isIdentity())
return failure();
- if (xferOp.mask())
+ if (xferOp.getMask())
return failure();
- auto extractOp = xferOp.source().getDefiningOp<tensor::ExtractSliceOp>();
+ auto extractOp = xferOp.getSource().getDefiningOp<tensor::ExtractSliceOp>();
if (!extractOp)
return failure();
if (!extractOp.hasUnitStride())
newIndices.push_back(getValueOrCreateConstantIndexOp(
rewriter, extractOp.getLoc(), offset));
}
- for (const auto &it : llvm::enumerate(xferOp.indices())) {
+ for (const auto &it : llvm::enumerate(xferOp.getIndices())) {
OpFoldResult offset =
extractOp.getMixedOffsets()[it.index() + rankReduced];
newIndices.push_back(rewriter.create<arith::AddIOp>(
SmallVector<bool> inBounds(xferOp.getTransferRank(), true);
rewriter.replaceOpWithNewOp<TransferReadOp>(
xferOp, xferOp.getVectorType(), extractOp.source(), newIndices,
- xferOp.padding(), ArrayRef<bool>{inBounds});
+ xferOp.getPadding(), ArrayRef<bool>{inBounds});
return success();
}
}
void TransferWriteOp::print(OpAsmPrinter &p) {
- p << " " << vector() << ", " << source() << "[" << indices() << "]";
- if (mask())
- p << ", " << mask();
+ p << " " << getVector() << ", " << getSource() << "[" << getIndices() << "]";
+ if (getMask())
+ p << ", " << getMask();
printTransferAttrs(p, *this);
p << " : " << getVectorType() << ", " << getShapedType();
}
ShapedType shapedType = getShapedType();
VectorType vectorType = getVectorType();
VectorType maskType = getMaskType();
- auto permutationMap = permutation_map();
+ auto permutationMap = getPermutationMap();
- if (llvm::size(indices()) != shapedType.getRank())
+ if (llvm::size(getIndices()) != shapedType.getRank())
return emitOpError("requires ") << shapedType.getRank() << " indices";
// We do not allow broadcast dimensions on TransferWriteOps for the moment,
if (failed(verifyTransferOp(cast<VectorTransferOpInterface>(getOperation()),
shapedType, vectorType, maskType, permutationMap,
- in_bounds() ? *in_bounds() : ArrayAttr())))
+ getInBounds() ? *getInBounds() : ArrayAttr())))
return failure();
return verifyPermutationMap(permutationMap,
// TODO: support 0-d corner case.
if (write.getTransferRank() == 0)
return failure();
- auto rankedTensorType = write.source().getType().dyn_cast<RankedTensorType>();
+ auto rankedTensorType =
+ write.getSource().getType().dyn_cast<RankedTensorType>();
// If not operating on tensors, bail.
if (!rankedTensorType)
return failure();
// If no read, bail.
- auto read = write.vector().getDefiningOp<vector::TransferReadOp>();
+ auto read = write.getVector().getDefiningOp<vector::TransferReadOp>();
if (!read)
return failure();
// TODO: support 0-d corner case.
if (read.getTransferRank() == 0)
return failure();
// For now, only accept minor identity. Future: composition is minor identity.
- if (!read.permutation_map().isMinorIdentity() ||
- !write.permutation_map().isMinorIdentity())
+ if (!read.getPermutationMap().isMinorIdentity() ||
+ !write.getPermutationMap().isMinorIdentity())
return failure();
// Bail on mismatching ranks.
if (read.getTransferRank() != write.getTransferRank())
if (read.hasOutOfBoundsDim() || write.hasOutOfBoundsDim())
return failure();
// Tensor types must be the same.
- if (read.source().getType() != rankedTensorType)
+ if (read.getSource().getType() != rankedTensorType)
return failure();
// Vector types must be the same.
if (read.getVectorType() != write.getVectorType())
auto cstOp = v.getDefiningOp<arith::ConstantIndexOp>();
return !cstOp || cstOp.value() != 0;
};
- if (llvm::any_of(read.indices(), isNotConstantZero) ||
- llvm::any_of(write.indices(), isNotConstantZero))
+ if (llvm::any_of(read.getIndices(), isNotConstantZero) ||
+ llvm::any_of(write.getIndices(), isNotConstantZero))
return failure();
// Success.
- results.push_back(read.source());
+ results.push_back(read.getSource());
return success();
}
static bool checkSameValueWAR(vector::TransferReadOp read,
vector::TransferWriteOp write) {
- return read.source() == write.source() && read.indices() == write.indices() &&
- read.permutation_map() == write.permutation_map() &&
- read.getVectorType() == write.getVectorType() && !read.mask() &&
- !write.mask();
+ return read.getSource() == write.getSource() &&
+ read.getIndices() == write.getIndices() &&
+ read.getPermutationMap() == write.getPermutationMap() &&
+ read.getVectorType() == write.getVectorType() && !read.getMask() &&
+ !write.getMask();
}
/// Fold transfer_write write after read:
/// ```
/// ```
static LogicalResult foldWAR(TransferWriteOp write,
SmallVectorImpl<OpFoldResult> &results) {
- if (!write.source().getType().isa<RankedTensorType>())
+ if (!write.getSource().getType().isa<RankedTensorType>())
return failure();
- auto read = write.vector().getDefiningOp<vector::TransferReadOp>();
+ auto read = write.getVector().getDefiningOp<vector::TransferReadOp>();
if (!read)
return failure();
if (!checkSameValueWAR(read, write))
return failure();
- results.push_back(read.source());
+ results.push_back(read.getSource());
return success();
}
SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
&effects) {
if (getShapedType().isa<MemRefType>())
- effects.emplace_back(MemoryEffects::Write::get(), source(),
+ effects.emplace_back(MemoryEffects::Write::get(), getSource(),
SideEffects::DefaultResource::get());
}
return failure();
vector::TransferWriteOp writeToModify = writeOp;
- auto defWrite = writeOp.source().getDefiningOp<vector::TransferWriteOp>();
+ auto defWrite =
+ writeOp.getSource().getDefiningOp<vector::TransferWriteOp>();
while (defWrite) {
if (checkSameValueWAW(writeOp, defWrite)) {
- writeToModify.sourceMutable().assign(defWrite.source());
+ writeToModify.getSourceMutable().assign(defWrite.getSource());
return success();
}
if (!isDisjointTransferIndices(
if (!defWrite->hasOneUse())
break;
writeToModify = defWrite;
- defWrite = defWrite.source().getDefiningOp<vector::TransferWriteOp>();
+ defWrite = defWrite.getSource().getDefiningOp<vector::TransferWriteOp>();
}
return failure();
}
return failure();
if (xferOp.getVectorType().getRank() != xferOp.getShapedType().getRank())
return failure();
- if (xferOp.mask())
+ if (xferOp.getMask())
return failure();
// Fold only if the TransferWriteOp completely overwrites the `source` with
// a vector. I.e., the result of the TransferWriteOp is a new tensor whose
if (!llvm::equal(xferOp.getVectorType().getShape(),
xferOp.getShapedType().getShape()))
return failure();
- if (!xferOp.permutation_map().isIdentity())
+ if (!xferOp.getPermutationMap().isIdentity())
return failure();
// Bail on illegal rank-reduction: we need to check that the rank-reduced
SmallVector<Value> indices = getValueOrCreateConstantIndexOp(
rewriter, insertOp.getLoc(), insertOp.getMixedOffsets());
SmallVector<bool> inBounds(xferOp.getTransferRank(), true);
- rewriter.replaceOpWithNewOp<TransferWriteOp>(insertOp, xferOp.vector(),
+ rewriter.replaceOpWithNewOp<TransferWriteOp>(insertOp, xferOp.getVector(),
insertOp.dest(), indices,
ArrayRef<bool>{inBounds});
return success();
if (resVecTy.getElementType() != memElemTy)
return emitOpError("base and result element types should match");
- if (llvm::size(indices()) != memRefTy.getRank())
+ if (llvm::size(getIndices()) != memRefTy.getRank())
return emitOpError("requires ") << memRefTy.getRank() << " indices";
return success();
}
if (valueVecTy.getElementType() != memElemTy)
return emitOpError("base and valueToStore element type should match");
- if (llvm::size(indices()) != memRefTy.getRank())
+ if (llvm::size(getIndices()) != memRefTy.getRank())
return emitOpError("requires ") << memRefTy.getRank() << " indices";
return success();
}
if (resVType.getElementType() != memType.getElementType())
return emitOpError("base and result element type should match");
- if (llvm::size(indices()) != memType.getRank())
+ if (llvm::size(getIndices()) != memType.getRank())
return emitOpError("requires ") << memType.getRank() << " indices";
if (resVType.getDimSize(0) != maskVType.getDimSize(0))
return emitOpError("expected result dim to match mask dim");
using OpRewritePattern<MaskedLoadOp>::OpRewritePattern;
LogicalResult matchAndRewrite(MaskedLoadOp load,
PatternRewriter &rewriter) const override {
- switch (get1DMaskFormat(load.mask())) {
+ switch (get1DMaskFormat(load.getMask())) {
case MaskFormat::AllTrue:
- rewriter.replaceOpWithNewOp<vector::LoadOp>(load, load.getType(),
- load.base(), load.indices());
+ rewriter.replaceOpWithNewOp<vector::LoadOp>(
+ load, load.getType(), load.getBase(), load.getIndices());
return success();
case MaskFormat::AllFalse:
- rewriter.replaceOp(load, load.pass_thru());
+ rewriter.replaceOp(load, load.getPassThru());
return success();
case MaskFormat::Unknown:
return failure();
if (valueVType.getElementType() != memType.getElementType())
return emitOpError("base and valueToStore element type should match");
- if (llvm::size(indices()) != memType.getRank())
+ if (llvm::size(getIndices()) != memType.getRank())
return emitOpError("requires ") << memType.getRank() << " indices";
if (valueVType.getDimSize(0) != maskVType.getDimSize(0))
return emitOpError("expected valueToStore dim to match mask dim");
using OpRewritePattern<MaskedStoreOp>::OpRewritePattern;
LogicalResult matchAndRewrite(MaskedStoreOp store,
PatternRewriter &rewriter) const override {
- switch (get1DMaskFormat(store.mask())) {
+ switch (get1DMaskFormat(store.getMask())) {
case MaskFormat::AllTrue:
rewriter.replaceOpWithNewOp<vector::StoreOp>(
- store, store.valueToStore(), store.base(), store.indices());
+ store, store.getValueToStore(), store.getBase(), store.getIndices());
return success();
case MaskFormat::AllFalse:
rewriter.eraseOp(store);
if (resVType.getElementType() != memType.getElementType())
return emitOpError("base and result element type should match");
- if (llvm::size(indices()) != memType.getRank())
+ if (llvm::size(getIndices()) != memType.getRank())
return emitOpError("requires ") << memType.getRank() << " indices";
if (resVType.getDimSize(0) != indVType.getDimSize(0))
return emitOpError("expected result dim to match indices dim");
using OpRewritePattern<GatherOp>::OpRewritePattern;
LogicalResult matchAndRewrite(GatherOp gather,
PatternRewriter &rewriter) const override {
- switch (get1DMaskFormat(gather.mask())) {
+ switch (get1DMaskFormat(gather.getMask())) {
case MaskFormat::AllTrue:
return failure(); // no unmasked equivalent
case MaskFormat::AllFalse:
- rewriter.replaceOp(gather, gather.pass_thru());
+ rewriter.replaceOp(gather, gather.getPassThru());
return success();
case MaskFormat::Unknown:
return failure();
if (valueVType.getElementType() != memType.getElementType())
return emitOpError("base and valueToStore element type should match");
- if (llvm::size(indices()) != memType.getRank())
+ if (llvm::size(getIndices()) != memType.getRank())
return emitOpError("requires ") << memType.getRank() << " indices";
if (valueVType.getDimSize(0) != indVType.getDimSize(0))
return emitOpError("expected valueToStore dim to match indices dim");
using OpRewritePattern<ScatterOp>::OpRewritePattern;
LogicalResult matchAndRewrite(ScatterOp scatter,
PatternRewriter &rewriter) const override {
- switch (get1DMaskFormat(scatter.mask())) {
+ switch (get1DMaskFormat(scatter.getMask())) {
case MaskFormat::AllTrue:
return failure(); // no unmasked equivalent
case MaskFormat::AllFalse:
if (resVType.getElementType() != memType.getElementType())
return emitOpError("base and result element type should match");
- if (llvm::size(indices()) != memType.getRank())
+ if (llvm::size(getIndices()) != memType.getRank())
return emitOpError("requires ") << memType.getRank() << " indices";
if (resVType.getDimSize(0) != maskVType.getDimSize(0))
return emitOpError("expected result dim to match mask dim");
using OpRewritePattern<ExpandLoadOp>::OpRewritePattern;
LogicalResult matchAndRewrite(ExpandLoadOp expand,
PatternRewriter &rewriter) const override {
- switch (get1DMaskFormat(expand.mask())) {
+ switch (get1DMaskFormat(expand.getMask())) {
case MaskFormat::AllTrue:
rewriter.replaceOpWithNewOp<vector::LoadOp>(
- expand, expand.getType(), expand.base(), expand.indices());
+ expand, expand.getType(), expand.getBase(), expand.getIndices());
return success();
case MaskFormat::AllFalse:
- rewriter.replaceOp(expand, expand.pass_thru());
+ rewriter.replaceOp(expand, expand.getPassThru());
return success();
case MaskFormat::Unknown:
return failure();
if (valueVType.getElementType() != memType.getElementType())
return emitOpError("base and valueToStore element type should match");
- if (llvm::size(indices()) != memType.getRank())
+ if (llvm::size(getIndices()) != memType.getRank())
return emitOpError("requires ") << memType.getRank() << " indices";
if (valueVType.getDimSize(0) != maskVType.getDimSize(0))
return emitOpError("expected valueToStore dim to match mask dim");
using OpRewritePattern<CompressStoreOp>::OpRewritePattern;
LogicalResult matchAndRewrite(CompressStoreOp compress,
PatternRewriter &rewriter) const override {
- switch (get1DMaskFormat(compress.mask())) {
+ switch (get1DMaskFormat(compress.getMask())) {
case MaskFormat::AllTrue:
rewriter.replaceOpWithNewOp<vector::StoreOp>(
- compress, compress.valueToStore(), compress.base(),
- compress.indices());
+ compress, compress.getValueToStore(), compress.getBase(),
+ compress.getIndices());
return success();
case MaskFormat::AllFalse:
rewriter.eraseOp(compress);
}
LogicalResult ShapeCastOp::verify() {
- auto sourceVectorType = source().getType().dyn_cast_or_null<VectorType>();
- auto resultVectorType = result().getType().dyn_cast_or_null<VectorType>();
+ auto sourceVectorType = getSource().getType().dyn_cast_or_null<VectorType>();
+ auto resultVectorType = getResult().getType().dyn_cast_or_null<VectorType>();
// Check if source/result are of vector type.
if (sourceVectorType && resultVectorType)
OpFoldResult ShapeCastOp::fold(ArrayRef<Attribute> operands) {
// Nop shape cast.
- if (source().getType() == result().getType())
- return source();
+ if (getSource().getType() == getResult().getType())
+ return getSource();
// Canceling shape casts.
- if (auto otherOp = source().getDefiningOp<ShapeCastOp>()) {
- if (result().getType() == otherOp.source().getType())
- return otherOp.source();
+ if (auto otherOp = getSource().getDefiningOp<ShapeCastOp>()) {
+ if (getResult().getType() == otherOp.getSource().getType())
+ return otherOp.getSource();
// Only allows valid transitive folding.
- VectorType srcType = otherOp.source().getType().cast<VectorType>();
+ VectorType srcType = otherOp.getSource().getType().cast<VectorType>();
VectorType resultType = getResult().getType().cast<VectorType>();
if (srcType.getRank() < resultType.getRank()) {
if (!isValidShapeCast(srcType.getShape(), resultType.getShape()))
return {};
}
- setOperand(otherOp.source());
+ setOperand(otherOp.getSource());
return getResult();
}
return {};
LogicalResult matchAndRewrite(ShapeCastOp shapeCastOp,
PatternRewriter &rewriter) const override {
- auto constantOp = shapeCastOp.source().getDefiningOp<arith::ConstantOp>();
+ auto constantOp =
+ shapeCastOp.getSource().getDefiningOp<arith::ConstantOp>();
if (!constantOp)
return failure();
// Only handle splat for now.
OpFoldResult BitCastOp::fold(ArrayRef<Attribute> operands) {
// Nop cast.
- if (source().getType() == result().getType())
- return source();
+ if (getSource().getType() == getResult().getType())
+ return getSource();
// Canceling bitcasts.
- if (auto otherOp = source().getDefiningOp<BitCastOp>())
- if (result().getType() == otherOp.source().getType())
- return otherOp.source();
+ if (auto otherOp = getSource().getDefiningOp<BitCastOp>())
+ if (getResult().getType() == otherOp.getSource().getType())
+ return otherOp.getSource();
Attribute sourceConstant = operands.front();
if (!sourceConstant)
return {};
}
- return vector();
+ return getVector();
}
LogicalResult vector::TransposeOp::verify() {
if (vectorType.getRank() != rank)
return emitOpError("vector result rank mismatch: ") << rank;
// Verify transposition array.
- auto transpAttr = transp().getValue();
+ auto transpAttr = getTransp().getValue();
int64_t size = transpAttr.size();
if (rank != size)
return emitOpError("transposition length mismatch: ") << size;
// Return if the input of 'transposeOp' is not defined by another transpose.
vector::TransposeOp parentTransposeOp =
- transposeOp.vector().getDefiningOp<vector::TransposeOp>();
+ transposeOp.getVector().getDefiningOp<vector::TransposeOp>();
if (!parentTransposeOp)
return failure();
// Replace 'transposeOp' with a new transpose operation.
rewriter.replaceOpWithNewOp<vector::TransposeOp>(
transposeOp, transposeOp.getResult().getType(),
- parentTransposeOp.vector(),
+ parentTransposeOp.getVector(),
vector::getVectorSubscriptAttr(rewriter, permutation));
return success();
}
}
void vector::TransposeOp::getTransp(SmallVectorImpl<int64_t> &results) {
- populateFromInt64AttrArray(transp(), results);
+ populateFromInt64AttrArray(getTransp(), results);
}
//===----------------------------------------------------------------------===//
auto resultType = getResult().getType().cast<VectorType>();
// Check the corner case of 0-D vectors first.
if (resultType.getRank() == 0) {
- if (mask_dim_sizes().size() != 1)
+ if (getMaskDimSizes().size() != 1)
return emitError("array attr must have length 1 for 0-D vectors");
- auto dim = mask_dim_sizes()[0].cast<IntegerAttr>().getInt();
+ auto dim = getMaskDimSizes()[0].cast<IntegerAttr>().getInt();
if (dim != 0 && dim != 1)
return emitError("mask dim size must be either 0 or 1 for 0-D vectors");
return success();
}
// Verify that array attr size matches the rank of the vector result.
- if (static_cast<int64_t>(mask_dim_sizes().size()) != resultType.getRank())
+ if (static_cast<int64_t>(getMaskDimSizes().size()) != resultType.getRank())
return emitOpError(
"must specify array attr of size equal vector result rank");
// Verify that each array attr element is in bounds of corresponding vector
// result dimension size.
auto resultShape = resultType.getShape();
SmallVector<int64_t, 4> maskDimSizes;
- for (const auto &it : llvm::enumerate(mask_dim_sizes())) {
+ for (const auto &it : llvm::enumerate(getMaskDimSizes())) {
int64_t attrValue = it.value().cast<IntegerAttr>().getInt();
if (attrValue < 0 || attrValue > resultShape[it.index()])
return emitOpError(
// `vector.constant_mask`. In the future, a convention could be established
// to decide if a specific dimension value could be considered as "all set".
if (resultType.isScalable() &&
- mask_dim_sizes()[0].cast<IntegerAttr>().getInt() != 0)
+ getMaskDimSizes()[0].cast<IntegerAttr>().getInt() != 0)
return emitOpError("expected mask dim sizes for scalable masks to be 0");
return success();
}
VectorType initialType = getInitialValueType();
// Check reduction dimension < rank.
int64_t srcRank = srcType.getRank();
- int64_t reductionDim = reduction_dim();
+ int64_t reductionDim = getReductionDim();
if (reductionDim >= srcRank)
return emitOpError("reduction dimension ")
<< reductionDim << " has to be less than " << srcRank;
Value buffer =
*state.getBuffer(rewriter, readOp->getOpOperand(0) /*source*/);
replaceOpWithNewBufferizedOp<vector::TransferReadOp>(
- rewriter, readOp, readOp.getVectorType(), buffer, readOp.indices(),
- readOp.permutation_map(), readOp.padding(), readOp.mask(),
- readOp.in_boundsAttr());
+ rewriter, readOp, readOp.getVectorType(), buffer, readOp.getIndices(),
+ readOp.getPermutationMap(), readOp.getPadding(), readOp.getMask(),
+ readOp.getInBoundsAttr());
return success();
}
};
if (failed(resultBuffer))
return failure();
rewriter.create<vector::TransferWriteOp>(
- writeOp.getLoc(), writeOp.vector(), *resultBuffer, writeOp.indices(),
- writeOp.permutation_mapAttr(), writeOp.in_boundsAttr());
+ writeOp.getLoc(), writeOp.getVector(), *resultBuffer,
+ writeOp.getIndices(), writeOp.getPermutationMapAttr(),
+ writeOp.getInBoundsAttr());
replaceOpWithBufferizedValues(rewriter, op, *resultBuffer);
return success();
Location loc = extractOp.getLoc();
Value newSrcVector = rewriter.create<vector::ExtractOp>(
- loc, extractOp.vector(), splatZero(dropCount));
+ loc, extractOp.getVector(), splatZero(dropCount));
// The offsets/sizes/strides attribute can have a less number of elements
// than the input vector's rank: it is meant for the leading dimensions.
auto newOffsets = rewriter.getArrayAttr(
- extractOp.offsets().getValue().drop_front(dropCount));
+ extractOp.getOffsets().getValue().drop_front(dropCount));
auto newSizes = rewriter.getArrayAttr(
- extractOp.sizes().getValue().drop_front(dropCount));
+ extractOp.getSizes().getValue().drop_front(dropCount));
auto newStrides = rewriter.getArrayAttr(
- extractOp.strides().getValue().drop_front(dropCount));
+ extractOp.getStrides().getValue().drop_front(dropCount));
auto newExtractOp = rewriter.create<vector::ExtractStridedSliceOp>(
loc, newDstType, newSrcVector, newOffsets, newSizes, newStrides);
Location loc = insertOp.getLoc();
Value newSrcVector = rewriter.create<vector::ExtractOp>(
- loc, insertOp.source(), splatZero(srcDropCount));
+ loc, insertOp.getSource(), splatZero(srcDropCount));
Value newDstVector = rewriter.create<vector::ExtractOp>(
- loc, insertOp.dest(), splatZero(dstDropCount));
+ loc, insertOp.getDest(), splatZero(dstDropCount));
auto newOffsets = rewriter.getArrayAttr(
- insertOp.offsets().getValue().take_back(newDstType.getRank()));
+ insertOp.getOffsets().getValue().take_back(newDstType.getRank()));
auto newStrides = rewriter.getArrayAttr(
- insertOp.strides().getValue().take_back(newSrcType.getRank()));
+ insertOp.getStrides().getValue().take_back(newSrcType.getRank()));
auto newInsertOp = rewriter.create<vector::InsertStridedSliceOp>(
loc, newDstType, newSrcVector, newDstVector, newOffsets, newStrides);
if (read.getTransferRank() == 0)
return failure();
- if (read.mask())
+ if (read.getMask())
return failure();
- auto shapedType = read.source().getType().cast<ShapedType>();
+ auto shapedType = read.getSource().getType().cast<ShapedType>();
if (shapedType.getElementType() != read.getVectorType().getElementType())
return failure();
if (newType == oldType)
return failure();
- AffineMap oldMap = read.permutation_map();
+ AffineMap oldMap = read.getPermutationMap();
ArrayRef<AffineExpr> newResults =
oldMap.getResults().take_back(newType.getRank());
AffineMap newMap =
rewriter.getContext());
ArrayAttr inBoundsAttr;
- if (read.in_bounds())
+ if (read.getInBounds())
inBoundsAttr = rewriter.getArrayAttr(
- read.in_boundsAttr().getValue().take_back(newType.getRank()));
+ read.getInBoundsAttr().getValue().take_back(newType.getRank()));
auto newRead = rewriter.create<vector::TransferReadOp>(
- read.getLoc(), newType, read.source(), read.indices(),
- AffineMapAttr::get(newMap), read.padding(), /*mask=*/Value(),
+ read.getLoc(), newType, read.getSource(), read.getIndices(),
+ AffineMapAttr::get(newMap), read.getPadding(), /*mask=*/Value(),
inBoundsAttr);
rewriter.replaceOpWithNewOp<vector::BroadcastOp>(read, oldType, newRead);
if (write.getTransferRank() == 0)
return failure();
- if (write.mask())
+ if (write.getMask())
return failure();
- auto shapedType = write.source().getType().dyn_cast<ShapedType>();
+ auto shapedType = write.getSource().getType().dyn_cast<ShapedType>();
if (shapedType.getElementType() != write.getVectorType().getElementType())
return failure();
return failure();
int64_t dropDim = oldType.getRank() - newType.getRank();
- AffineMap oldMap = write.permutation_map();
+ AffineMap oldMap = write.getPermutationMap();
ArrayRef<AffineExpr> newResults =
oldMap.getResults().take_back(newType.getRank());
AffineMap newMap =
rewriter.getContext());
ArrayAttr inBoundsAttr;
- if (write.in_bounds())
+ if (write.getInBounds())
inBoundsAttr = rewriter.getArrayAttr(
- write.in_boundsAttr().getValue().take_back(newType.getRank()));
+ write.getInBoundsAttr().getValue().take_back(newType.getRank()));
auto newVector = rewriter.create<vector::ExtractOp>(
- write.getLoc(), write.vector(), splatZero(dropDim));
+ write.getLoc(), write.getVector(), splatZero(dropDim));
rewriter.replaceOpWithNewOp<vector::TransferWriteOp>(
- write, newVector, write.source(), write.indices(),
+ write, newVector, write.getSource(), write.getIndices(),
AffineMapAttr::get(newMap), inBoundsAttr);
return success();
if (oldAccType.getRank() < 2)
return failure();
// TODO: implement masks.
- if (llvm::size(contractOp.masks()) != 0)
+ if (llvm::size(contractOp.getMasks()) != 0)
return failure();
if (oldAccType.getShape()[0] != 1)
return failure();
auto oldIndexingMaps = contractOp.getIndexingMaps();
SmallVector<AffineMap> newIndexingMaps;
- auto oldIteratorTypes = contractOp.iterator_types();
+ auto oldIteratorTypes = contractOp.getIteratorTypes();
SmallVector<Attribute> newIteratorTypes;
int64_t dimToDrop = oldIndexingMaps[2].getDimPosition(0);
newIteratorTypes.push_back(it.value());
}
- SmallVector<Value> operands = {contractOp.lhs(), contractOp.rhs(),
- contractOp.acc()};
+ SmallVector<Value> operands = {contractOp.getLhs(), contractOp.getRhs(),
+ contractOp.getAcc()};
SmallVector<Value> newOperands;
for (const auto &it : llvm::enumerate(oldIndexingMaps)) {
auto newContractOp = rewriter.create<vector::ContractionOp>(
contractOp.getLoc(), newOperands[0], newOperands[1], newOperands[2],
rewriter.getAffineMapArrayAttr(newIndexingMaps),
- rewriter.getArrayAttr(newIteratorTypes), contractOp.kind());
+ rewriter.getArrayAttr(newIteratorTypes), contractOp.getKind());
rewriter.replaceOpWithNewOp<vector::BroadcastOp>(
contractOp, contractOp->getResultTypes()[0], newContractOp);
return success();
auto srcType = op.getSourceVectorType();
auto dstType = op.getDestVectorType();
- if (op.offsets().getValue().empty())
+ if (op.getOffsets().getValue().empty())
return failure();
auto loc = op.getLoc();
int64_t rankRest = dstType.getRank() - rankDiff;
// Extract / insert the subvector of matching rank and InsertStridedSlice
// on it.
- Value extracted =
- rewriter.create<ExtractOp>(loc, op.dest(),
- getI64SubArray(op.offsets(), /*dropFront=*/0,
- /*dropBack=*/rankRest));
+ Value extracted = rewriter.create<ExtractOp>(
+ loc, op.getDest(),
+ getI64SubArray(op.getOffsets(), /*dropFront=*/0,
+ /*dropBack=*/rankRest));
// A different pattern will kick in for InsertStridedSlice with matching
// ranks.
auto stridedSliceInnerOp = rewriter.create<InsertStridedSliceOp>(
- loc, op.source(), extracted,
- getI64SubArray(op.offsets(), /*dropFront=*/rankDiff),
- getI64SubArray(op.strides(), /*dropFront=*/0));
+ loc, op.getSource(), extracted,
+ getI64SubArray(op.getOffsets(), /*dropFront=*/rankDiff),
+ getI64SubArray(op.getStrides(), /*dropFront=*/0));
rewriter.replaceOpWithNewOp<InsertOp>(
- op, stridedSliceInnerOp.getResult(), op.dest(),
- getI64SubArray(op.offsets(), /*dropFront=*/0,
+ op, stridedSliceInnerOp.getResult(), op.getDest(),
+ getI64SubArray(op.getOffsets(), /*dropFront=*/0,
/*dropBack=*/rankRest));
return success();
}
auto srcType = op.getSourceVectorType();
auto dstType = op.getDestVectorType();
- if (op.offsets().getValue().empty())
+ if (op.getOffsets().getValue().empty())
return failure();
int64_t srcRank = srcType.getRank();
return failure();
if (srcType == dstType) {
- rewriter.replaceOp(op, op.source());
+ rewriter.replaceOp(op, op.getSource());
return success();
}
int64_t offset =
- op.offsets().getValue().front().cast<IntegerAttr>().getInt();
+ op.getOffsets().getValue().front().cast<IntegerAttr>().getInt();
int64_t size = srcType.getShape().front();
int64_t stride =
- op.strides().getValue().front().cast<IntegerAttr>().getInt();
+ op.getStrides().getValue().front().cast<IntegerAttr>().getInt();
auto loc = op.getLoc();
- Value res = op.dest();
+ Value res = op.getDest();
if (srcRank == 1) {
int nSrc = srcType.getShape().front();
SmallVector<int64_t> offsets(nDest, 0);
for (int64_t i = 0; i < nSrc; ++i)
offsets[i] = i;
- Value scaledSource =
- rewriter.create<ShuffleOp>(loc, op.source(), op.source(), offsets);
+ Value scaledSource = rewriter.create<ShuffleOp>(loc, op.getSource(),
+ op.getSource(), offsets);
// 2. Create a mask where we take the value from scaledSource of dest
// depending on the offset.
}
// 3. Replace with a ShuffleOp.
- rewriter.replaceOpWithNewOp<ShuffleOp>(op, scaledSource, op.dest(),
+ rewriter.replaceOpWithNewOp<ShuffleOp>(op, scaledSource, op.getDest(),
offsets);
return success();
for (int64_t off = offset, e = offset + size * stride, idx = 0; off < e;
off += stride, ++idx) {
// 1. extract the proper subvector (or element) from source
- Value extractedSource = extractOne(rewriter, loc, op.source(), idx);
+ Value extractedSource = extractOne(rewriter, loc, op.getSource(), idx);
if (extractedSource.getType().isa<VectorType>()) {
// 2. If we have a vector, extract the proper subvector from destination
// Otherwise we are at the element level and no need to recurse.
- Value extractedDest = extractOne(rewriter, loc, op.dest(), off);
+ Value extractedDest = extractOne(rewriter, loc, op.getDest(), off);
// 3. Reduce the problem to lowering a new InsertStridedSlice op with
// smaller rank.
extractedSource = rewriter.create<InsertStridedSliceOp>(
loc, extractedSource, extractedDest,
- getI64SubArray(op.offsets(), /* dropFront=*/1),
- getI64SubArray(op.strides(), /* dropFront=*/1));
+ getI64SubArray(op.getOffsets(), /* dropFront=*/1),
+ getI64SubArray(op.getStrides(), /* dropFront=*/1));
}
// 4. Insert the extractedSource into the res vector.
res = insertOne(rewriter, loc, extractedSource, res, off);
PatternRewriter &rewriter) const override {
auto dstType = op.getType();
- assert(!op.offsets().getValue().empty() && "Unexpected empty offsets");
+ assert(!op.getOffsets().getValue().empty() && "Unexpected empty offsets");
int64_t offset =
- op.offsets().getValue().front().cast<IntegerAttr>().getInt();
- int64_t size = op.sizes().getValue().front().cast<IntegerAttr>().getInt();
+ op.getOffsets().getValue().front().cast<IntegerAttr>().getInt();
+ int64_t size =
+ op.getSizes().getValue().front().cast<IntegerAttr>().getInt();
int64_t stride =
- op.strides().getValue().front().cast<IntegerAttr>().getInt();
+ op.getStrides().getValue().front().cast<IntegerAttr>().getInt();
auto loc = op.getLoc();
auto elemType = dstType.getElementType();
assert(elemType.isSignlessIntOrIndexOrFloat());
// Single offset can be more efficiently shuffled.
- if (op.offsets().getValue().size() == 1) {
+ if (op.getOffsets().getValue().size() == 1) {
SmallVector<int64_t, 4> offsets;
offsets.reserve(size);
for (int64_t off = offset, e = offset + size * stride; off < e;
off += stride)
offsets.push_back(off);
- rewriter.replaceOpWithNewOp<ShuffleOp>(op, dstType, op.vector(),
- op.vector(),
+ rewriter.replaceOpWithNewOp<ShuffleOp>(op, dstType, op.getVector(),
+ op.getVector(),
rewriter.getI64ArrayAttr(offsets));
return success();
}
Value res = rewriter.create<SplatOp>(loc, dstType, zero);
for (int64_t off = offset, e = offset + size * stride, idx = 0; off < e;
off += stride, ++idx) {
- Value one = extractOne(rewriter, loc, op.vector(), off);
+ Value one = extractOne(rewriter, loc, op.getVector(), off);
Value extracted = rewriter.create<ExtractStridedSliceOp>(
- loc, one, getI64SubArray(op.offsets(), /* dropFront=*/1),
- getI64SubArray(op.sizes(), /* dropFront=*/1),
- getI64SubArray(op.strides(), /* dropFront=*/1));
+ loc, one, getI64SubArray(op.getOffsets(), /* dropFront=*/1),
+ getI64SubArray(op.getSizes(), /* dropFront=*/1),
+ getI64SubArray(op.getStrides(), /* dropFront=*/1));
res = insertOne(rewriter, loc, extracted, res, idx);
}
rewriter.replaceOp(op, res);
LogicalResult matchAndRewrite(vector::MultiDimReductionOp multiReductionOp,
PatternRewriter &rewriter) const override {
- auto src = multiReductionOp.source();
+ auto src = multiReductionOp.getSource();
auto loc = multiReductionOp.getLoc();
auto srcRank = multiReductionOp.getSourceVectorType().getRank();
// Separate reduction and parallel dims
auto reductionDimsRange =
- multiReductionOp.reduction_dims().getAsValueRange<IntegerAttr>();
+ multiReductionOp.getReductionDims().getAsValueRange<IntegerAttr>();
auto reductionDims = llvm::to_vector<4>(llvm::map_range(
reductionDimsRange, [](const APInt &a) { return a.getZExtValue(); }));
llvm::SmallDenseSet<int64_t> reductionDimsSet(reductionDims.begin(),
reductionMask[i] = true;
}
rewriter.replaceOpWithNewOp<vector::MultiDimReductionOp>(
- multiReductionOp, transposeOp.result(), reductionMask,
- multiReductionOp.kind());
+ multiReductionOp, transposeOp.getResult(), reductionMask,
+ multiReductionOp.getKind());
return success();
}
auto castedType = VectorType::get(
vectorShape, multiReductionOp.getSourceVectorType().getElementType());
Value cast = rewriter.create<vector::ShapeCastOp>(
- loc, castedType, multiReductionOp.source());
+ loc, castedType, multiReductionOp.getSource());
// 5. Creates the flattened form of vector.multi_reduction with inner/outer
// most dim as reduction.
auto newOp = rewriter.create<vector::MultiDimReductionOp>(
- loc, cast, mask, multiReductionOp.kind());
+ loc, cast, mask, multiReductionOp.getKind());
// 6. If there are no parallel shapes, the result is a scalar.
// TODO: support 0-d vectors when available.
if (parallelShapes.empty()) {
- rewriter.replaceOp(multiReductionOp, newOp.dest());
+ rewriter.replaceOp(multiReductionOp, newOp.getDest());
return success();
}
parallelShapes,
multiReductionOp.getSourceVectorType().getElementType());
rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(
- multiReductionOp, outputCastedType, newOp.dest());
+ multiReductionOp, outputCastedType, newOp.getDest());
return success();
}
return failure();
Value result =
- rewriter.create<vector::ExtractOp>(loc, multiReductionOp.source(), 0)
+ rewriter.create<vector::ExtractOp>(loc, multiReductionOp.getSource(), 0)
.getResult();
for (int64_t i = 1; i < srcShape[0]; i++) {
- auto operand =
- rewriter.create<vector::ExtractOp>(loc, multiReductionOp.source(), i);
- result = makeArithReduction(rewriter, loc, multiReductionOp.kind(),
+ auto operand = rewriter.create<vector::ExtractOp>(
+ loc, multiReductionOp.getSource(), i);
+ result = makeArithReduction(rewriter, loc, multiReductionOp.getKind(),
operand, result);
}
for (int i = 0; i < outerDim; ++i) {
auto v = rewriter.create<vector::ExtractOp>(
- loc, multiReductionOp.source(), ArrayRef<int64_t>{i});
- auto reducedValue =
- rewriter.create<vector::ReductionOp>(loc, multiReductionOp.kind(), v);
+ loc, multiReductionOp.getSource(), ArrayRef<int64_t>{i});
+ auto reducedValue = rewriter.create<vector::ReductionOp>(
+ loc, multiReductionOp.getKind(), v);
result = rewriter.create<vector::InsertElementOp>(
loc, reducedValue, result,
rewriter.create<arith::ConstantIndexOp>(loc, i));
/// vector.extract(vector.multi_reduce(vector.shape_cast(v, 1xk)), 0)
Value cast = rewriter.create<vector::ShapeCastOp>(
- loc, castedType, multiReductionOp.source());
+ loc, castedType, multiReductionOp.getSource());
Value reduced = rewriter.create<vector::MultiDimReductionOp>(
- loc, cast, mask, multiReductionOp.kind());
+ loc, cast, mask, multiReductionOp.getKind());
rewriter.replaceOpWithNewOp<vector::ExtractOp>(multiReductionOp, reduced,
ArrayRef<int64_t>{0});
return success();
<< "\n");
llvm::SmallVector<Operation *, 8> reads;
Operation *firstOverwriteCandidate = nullptr;
- for (auto *user : write.source().getUsers()) {
+ for (auto *user : write.getSource().getUsers()) {
if (user == write.getOperation())
continue;
if (auto nextWrite = dyn_cast<vector::TransferWriteOp>(user)) {
<< "\n");
SmallVector<Operation *, 8> blockingWrites;
vector::TransferWriteOp lastwrite = nullptr;
- for (Operation *user : read.source().getUsers()) {
+ for (Operation *user : read.getSource().getUsers()) {
if (isa<vector::TransferReadOp>(user))
continue;
if (auto write = dyn_cast<vector::TransferWriteOp>(user)) {
LLVM_DEBUG(DBGS() << "Forward value from " << *lastwrite.getOperation()
<< " to: " << *read.getOperation() << "\n");
- read.replaceAllUsesWith(lastwrite.vector());
+ read.replaceAllUsesWith(lastwrite.getVector());
opToErase.push_back(read.getOperation());
}
LogicalResult matchAndRewrite(vector::TransferReadOp transferReadOp,
PatternRewriter &rewriter) const override {
auto loc = transferReadOp.getLoc();
- Value vector = transferReadOp.vector();
+ Value vector = transferReadOp.getVector();
VectorType vectorType = vector.getType().cast<VectorType>();
- Value source = transferReadOp.source();
+ Value source = transferReadOp.getSource();
MemRefType sourceType = source.getType().dyn_cast<MemRefType>();
// TODO: support tensor types.
if (!sourceType || !sourceType.hasStaticShape())
// TODO: generalize this pattern, relax the requirements here.
if (transferReadOp.hasOutOfBoundsDim())
return failure();
- if (!transferReadOp.permutation_map().isMinorIdentity())
+ if (!transferReadOp.getPermutationMap().isMinorIdentity())
return failure();
int reducedRank = getReducedRank(sourceType.getShape());
if (reducedRank == sourceType.getRank())
if (reducedRank != vectorType.getRank())
return failure(); // This pattern requires the vector shape to match the
// reduced source shape.
- if (llvm::any_of(transferReadOp.indices(),
+ if (llvm::any_of(transferReadOp.getIndices(),
[](Value v) { return !isZero(v); }))
return failure();
Value reducedShapeSource =
LogicalResult matchAndRewrite(vector::TransferWriteOp transferWriteOp,
PatternRewriter &rewriter) const override {
auto loc = transferWriteOp.getLoc();
- Value vector = transferWriteOp.vector();
+ Value vector = transferWriteOp.getVector();
VectorType vectorType = vector.getType().cast<VectorType>();
- Value source = transferWriteOp.source();
+ Value source = transferWriteOp.getSource();
MemRefType sourceType = source.getType().dyn_cast<MemRefType>();
// TODO: support tensor type.
if (!sourceType || !sourceType.hasStaticShape())
// TODO: generalize this pattern, relax the requirements here.
if (transferWriteOp.hasOutOfBoundsDim())
return failure();
- if (!transferWriteOp.permutation_map().isMinorIdentity())
+ if (!transferWriteOp.getPermutationMap().isMinorIdentity())
return failure();
int reducedRank = getReducedRank(sourceType.getShape());
if (reducedRank == sourceType.getRank())
if (reducedRank != vectorType.getRank())
return failure(); // This pattern requires the vector shape to match the
// reduced source shape.
- if (llvm::any_of(transferWriteOp.indices(),
+ if (llvm::any_of(transferWriteOp.getIndices(),
[](Value v) { return !isZero(v); }))
return failure();
Value reducedShapeSource =
LogicalResult matchAndRewrite(vector::TransferReadOp transferReadOp,
PatternRewriter &rewriter) const override {
auto loc = transferReadOp.getLoc();
- Value vector = transferReadOp.vector();
+ Value vector = transferReadOp.getVector();
VectorType vectorType = vector.getType().cast<VectorType>();
- Value source = transferReadOp.source();
+ Value source = transferReadOp.getSource();
MemRefType sourceType = source.getType().dyn_cast<MemRefType>();
// Contiguity check is valid on tensors only.
if (!sourceType)
// TODO: generalize this pattern, relax the requirements here.
if (transferReadOp.hasOutOfBoundsDim())
return failure();
- if (!transferReadOp.permutation_map().isMinorIdentity())
+ if (!transferReadOp.getPermutationMap().isMinorIdentity())
return failure();
- if (transferReadOp.mask())
+ if (transferReadOp.getMask())
return failure();
- if (llvm::any_of(transferReadOp.indices(),
+ if (llvm::any_of(transferReadOp.getIndices(),
[](Value v) { return !isZero(v); }))
return failure();
Value c0 = rewriter.create<arith::ConstantIndexOp>(loc, 0);
LogicalResult matchAndRewrite(vector::TransferWriteOp transferWriteOp,
PatternRewriter &rewriter) const override {
auto loc = transferWriteOp.getLoc();
- Value vector = transferWriteOp.vector();
+ Value vector = transferWriteOp.getVector();
VectorType vectorType = vector.getType().cast<VectorType>();
- Value source = transferWriteOp.source();
+ Value source = transferWriteOp.getSource();
MemRefType sourceType = source.getType().dyn_cast<MemRefType>();
// Contiguity check is valid on tensors only.
if (!sourceType)
// TODO: generalize this pattern, relax the requirements here.
if (transferWriteOp.hasOutOfBoundsDim())
return failure();
- if (!transferWriteOp.permutation_map().isMinorIdentity())
+ if (!transferWriteOp.getPermutationMap().isMinorIdentity())
return failure();
- if (transferWriteOp.mask())
+ if (transferWriteOp.getMask())
return failure();
- if (llvm::any_of(transferWriteOp.indices(),
+ if (llvm::any_of(transferWriteOp.getIndices(),
[](Value v) { return !isZero(v); }))
return failure();
Value c0 = rewriter.create<arith::ConstantIndexOp>(loc, 0);
return failure();
SmallVector<unsigned> permutation;
- AffineMap map = op.permutation_map();
+ AffineMap map = op.getPermutationMap();
if (map.getNumResults() == 0)
return failure();
if (!map.isPermutationOfMinorIdentityWithBroadcasting(permutation))
// Transpose mask operand.
Value newMask;
- if (op.mask()) {
+ if (op.getMask()) {
// Remove unused dims from the permutation map. E.g.:
// E.g.: (d0, d1, d2, d3, d4, d5) -> (d5, 0, d3, 0, d2)
// comp = (d0, d1, d2) -> (d2, 0, d1, 0 d0)
maskTransposeIndices.push_back(expr.getPosition());
}
- newMask = rewriter.create<vector::TransposeOp>(op.getLoc(), op.mask(),
+ newMask = rewriter.create<vector::TransposeOp>(op.getLoc(), op.getMask(),
maskTransposeIndices);
}
// Transpose in_bounds attribute.
ArrayAttr newInBoundsAttr =
- op.in_bounds() ? transposeInBoundsAttr(
- rewriter, op.in_bounds().getValue(), permutation)
- : ArrayAttr();
+ op.getInBounds()
+ ? transposeInBoundsAttr(rewriter, op.getInBounds().getValue(),
+ permutation)
+ : ArrayAttr();
// Generate new transfer_read operation.
VectorType newReadType =
VectorType::get(newVectorShape, op.getVectorType().getElementType());
Value newRead = rewriter.create<vector::TransferReadOp>(
- op.getLoc(), newReadType, op.source(), op.indices(),
- AffineMapAttr::get(newMap), op.padding(), newMask, newInBoundsAttr);
+ op.getLoc(), newReadType, op.getSource(), op.getIndices(),
+ AffineMapAttr::get(newMap), op.getPadding(), newMask, newInBoundsAttr);
// Transpose result of transfer_read.
SmallVector<int64_t> transposePerm(permutation.begin(), permutation.end());
return failure();
SmallVector<unsigned> permutation;
- AffineMap map = op.permutation_map();
+ AffineMap map = op.getPermutationMap();
if (map.isMinorIdentity())
return failure();
if (!map.isPermutationOfMinorIdentityWithBroadcasting(permutation))
});
// Transpose mask operand.
- Value newMask = op.mask() ? rewriter.create<vector::TransposeOp>(
- op.getLoc(), op.mask(), indices)
- : Value();
+ Value newMask = op.getMask() ? rewriter.create<vector::TransposeOp>(
+ op.getLoc(), op.getMask(), indices)
+ : Value();
// Transpose in_bounds attribute.
ArrayAttr newInBoundsAttr =
- op.in_bounds() ? transposeInBoundsAttr(
- rewriter, op.in_bounds().getValue(), permutation)
- : ArrayAttr();
+ op.getInBounds()
+ ? transposeInBoundsAttr(rewriter, op.getInBounds().getValue(),
+ permutation)
+ : ArrayAttr();
// Generate new transfer_write operation.
- Value newVec =
- rewriter.create<vector::TransposeOp>(op.getLoc(), op.vector(), indices);
+ Value newVec = rewriter.create<vector::TransposeOp>(
+ op.getLoc(), op.getVector(), indices);
auto newMap = AffineMap::getMinorIdentityMap(
map.getNumDims(), map.getNumResults(), rewriter.getContext());
rewriter.replaceOpWithNewOp<vector::TransferWriteOp>(
- op, newVec, op.source(), op.indices(), AffineMapAttr::get(newMap),
+ op, newVec, op.getSource(), op.getIndices(), AffineMapAttr::get(newMap),
newMask, newInBoundsAttr);
return success();
if (op.getTransferRank() == 0)
return failure();
- AffineMap map = op.permutation_map();
+ AffineMap map = op.getPermutationMap();
unsigned numLeadingBroadcast = 0;
for (auto expr : map.getResults()) {
auto dimExpr = expr.dyn_cast<AffineConstantExpr>();
if (reducedShapeRank == 0) {
Value newRead;
if (op.getShapedType().isa<TensorType>()) {
- newRead = rewriter.create<tensor::ExtractOp>(op.getLoc(), op.source(),
- op.indices());
+ newRead = rewriter.create<tensor::ExtractOp>(
+ op.getLoc(), op.getSource(), op.getIndices());
} else {
newRead = rewriter.create<memref::LoadOp>(
- op.getLoc(), originalVecType.getElementType(), op.source(),
- op.indices());
+ op.getLoc(), originalVecType.getElementType(), op.getSource(),
+ op.getIndices());
}
rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op, originalVecType,
newRead);
VectorType newReadType =
VectorType::get(newShape, originalVecType.getElementType());
ArrayAttr newInBoundsAttr =
- op.in_bounds()
+ op.getInBounds()
? rewriter.getArrayAttr(
- op.in_boundsAttr().getValue().take_back(reducedShapeRank))
+ op.getInBoundsAttr().getValue().take_back(reducedShapeRank))
: ArrayAttr();
Value newRead = rewriter.create<vector::TransferReadOp>(
- op.getLoc(), newReadType, op.source(), op.indices(),
- AffineMapAttr::get(newMap), op.padding(), op.mask(), newInBoundsAttr);
+ op.getLoc(), newReadType, op.getSource(), op.getIndices(),
+ AffineMapAttr::get(newMap), op.getPadding(), op.getMask(),
+ newInBoundsAttr);
rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op, originalVecType,
newRead);
return success();
MemRefType compatibleMemRefType, Value alloc) {
Location loc = xferOp.getLoc();
Value zero = b.create<arith::ConstantIndexOp>(loc, 0);
- Value memref = xferOp.source();
+ Value memref = xferOp.getSource();
return b.create<scf::IfOp>(
loc, returnTypes, inBoundsCond,
[&](OpBuilder &b, Location loc) {
if (compatibleMemRefType != xferOp.getShapedType())
res = b.create<memref::CastOp>(loc, compatibleMemRefType, memref);
scf::ValueVector viewAndIndices{res};
- viewAndIndices.insert(viewAndIndices.end(), xferOp.indices().begin(),
- xferOp.indices().end());
+ viewAndIndices.insert(viewAndIndices.end(), xferOp.getIndices().begin(),
+ xferOp.getIndices().end());
b.create<scf::YieldOp>(loc, viewAndIndices);
},
[&](OpBuilder &b, Location loc) {
- b.create<linalg::FillOp>(loc, ValueRange{xferOp.padding()},
+ b.create<linalg::FillOp>(loc, ValueRange{xferOp.getPadding()},
ValueRange{alloc});
// Take partial subview of memref which guarantees no dimension
// overflows.
Location loc = xferOp.getLoc();
scf::IfOp fullPartialIfOp;
Value zero = b.create<arith::ConstantIndexOp>(loc, 0);
- Value memref = xferOp.source();
+ Value memref = xferOp.getSource();
return b.create<scf::IfOp>(
loc, returnTypes, inBoundsCond,
[&](OpBuilder &b, Location loc) {
if (compatibleMemRefType != xferOp.getShapedType())
res = b.create<memref::CastOp>(loc, compatibleMemRefType, memref);
scf::ValueVector viewAndIndices{res};
- viewAndIndices.insert(viewAndIndices.end(), xferOp.indices().begin(),
- xferOp.indices().end());
+ viewAndIndices.insert(viewAndIndices.end(), xferOp.getIndices().begin(),
+ xferOp.getIndices().end());
b.create<scf::YieldOp>(loc, viewAndIndices);
},
[&](OpBuilder &b, Location loc) {
MemRefType compatibleMemRefType, Value alloc) {
Location loc = xferOp.getLoc();
Value zero = b.create<arith::ConstantIndexOp>(loc, 0);
- Value memref = xferOp.source();
+ Value memref = xferOp.getSource();
return b
.create<scf::IfOp>(
loc, returnTypes, inBoundsCond,
res = b.create<memref::CastOp>(loc, compatibleMemRefType, memref);
scf::ValueVector viewAndIndices{res};
viewAndIndices.insert(viewAndIndices.end(),
- xferOp.indices().begin(),
- xferOp.indices().end());
+ xferOp.getIndices().begin(),
+ xferOp.getIndices().end());
b.create<scf::YieldOp>(loc, viewAndIndices);
},
[&](OpBuilder &b, Location loc) {
b.create<scf::IfOp>(loc, notInBounds, [&](OpBuilder &b, Location loc) {
BlockAndValueMapping mapping;
Value load = b.create<memref::LoadOp>(
- loc, b.create<vector::TypeCastOp>(
- loc, MemRefType::get({}, xferOp.vector().getType()), alloc));
- mapping.map(xferOp.vector(), load);
+ loc,
+ b.create<vector::TypeCastOp>(
+ loc, MemRefType::get({}, xferOp.getVector().getType()), alloc));
+ mapping.map(xferOp.getVector(), load);
b.clone(*xferOp.getOperation(), mapping);
b.create<scf::YieldOp>(loc, ValueRange{});
});
if (!(xferReadOp || xferWriteOp))
return failure();
- if (xferWriteOp && xferWriteOp.mask())
+ if (xferWriteOp && xferWriteOp.getMask())
return failure();
- if (xferReadOp && xferReadOp.mask())
+ if (xferReadOp && xferReadOp.getMask())
return failure();
}
// The operation is cloned to prevent deleting information needed for the
// later IR creation.
BlockAndValueMapping mapping;
- mapping.map(xferWriteOp.source(), memrefAndIndices.front());
- mapping.map(xferWriteOp.indices(), memrefAndIndices.drop_front());
+ mapping.map(xferWriteOp.getSource(), memrefAndIndices.front());
+ mapping.map(xferWriteOp.getIndices(), memrefAndIndices.drop_front());
auto *clone = b.clone(*xferWriteOp, mapping);
clone->setAttr(xferWriteOp.getInBoundsAttrName(), inBoundsAttr);
PatternRewriter &rewriter) const override {
// Check if 'shapeCastOp' has vector source/result type.
auto sourceVectorType =
- shapeCastOp.source().getType().dyn_cast_or_null<VectorType>();
+ shapeCastOp.getSource().getType().dyn_cast_or_null<VectorType>();
auto resultVectorType =
- shapeCastOp.result().getType().dyn_cast_or_null<VectorType>();
+ shapeCastOp.getResult().getType().dyn_cast_or_null<VectorType>();
if (!sourceVectorType || !resultVectorType)
return failure();
// Check if shape cast op source operand is also a shape cast op.
auto sourceShapeCastOp = dyn_cast_or_null<vector::ShapeCastOp>(
- shapeCastOp.source().getDefiningOp());
+ shapeCastOp.getSource().getDefiningOp());
if (!sourceShapeCastOp)
return failure();
auto operandSourceVectorType =
- sourceShapeCastOp.source().getType().cast<VectorType>();
+ sourceShapeCastOp.getSource().getType().cast<VectorType>();
auto operandResultVectorType = sourceShapeCastOp.getType();
// Check if shape cast operations invert each other.
operandResultVectorType != sourceVectorType)
return failure();
- rewriter.replaceOp(shapeCastOp, sourceShapeCastOp.source());
+ rewriter.replaceOp(shapeCastOp, sourceShapeCastOp.getSource());
return success();
}
};
// Scalar to any vector can use splat.
if (!srcType) {
- rewriter.replaceOpWithNewOp<vector::SplatOp>(op, dstType, op.source());
+ rewriter.replaceOpWithNewOp<vector::SplatOp>(op, dstType, op.getSource());
return success();
}
if (srcRank <= 1 && dstRank == 1) {
Value ext;
if (srcRank == 0)
- ext = rewriter.create<vector::ExtractElementOp>(loc, op.source());
+ ext = rewriter.create<vector::ExtractElementOp>(loc, op.getSource());
else
- ext = rewriter.create<vector::ExtractOp>(loc, op.source(), 0);
+ ext = rewriter.create<vector::ExtractOp>(loc, op.getSource(), 0);
rewriter.replaceOpWithNewOp<vector::SplatOp>(op, dstType, ext);
return success();
}
VectorType resType =
VectorType::get(dstType.getShape().drop_front(), eltType);
Value bcst =
- rewriter.create<vector::BroadcastOp>(loc, resType, op.source());
+ rewriter.create<vector::BroadcastOp>(loc, resType, op.getSource());
Value result = rewriter.create<arith::ConstantOp>(
loc, dstType, rewriter.getZeroAttr(dstType));
for (int64_t d = 0, dim = dstType.getDimSize(0); d < dim; ++d)
// All trailing dimensions are the same. Simply pass through.
if (m == -1) {
- rewriter.replaceOp(op, op.source());
+ rewriter.replaceOp(op, op.getSource());
return success();
}
loc, dstType, rewriter.getZeroAttr(dstType));
if (m == 0) {
// Stetch at start.
- Value ext = rewriter.create<vector::ExtractOp>(loc, op.source(), 0);
+ Value ext = rewriter.create<vector::ExtractOp>(loc, op.getSource(), 0);
Value bcst = rewriter.create<vector::BroadcastOp>(loc, resType, ext);
for (int64_t d = 0, dim = dstType.getDimSize(0); d < dim; ++d)
result = rewriter.create<vector::InsertOp>(loc, bcst, result, d);
} else {
// Stetch not at start.
for (int64_t d = 0, dim = dstType.getDimSize(0); d < dim; ++d) {
- Value ext = rewriter.create<vector::ExtractOp>(loc, op.source(), d);
+ Value ext = rewriter.create<vector::ExtractOp>(loc, op.getSource(), d);
Value bcst = rewriter.create<vector::BroadcastOp>(loc, resType, ext);
result = rewriter.create<vector::InsertOp>(loc, bcst, result, d);
}
PatternRewriter &rewriter) const override {
auto loc = op.getLoc();
- Value input = op.vector();
+ Value input = op.getVector();
VectorType inputType = op.getVectorType();
VectorType resType = op.getResultType();
// Set up convenience transposition table.
SmallVector<int64_t, 4> transp;
- for (auto attr : op.transp())
+ for (auto attr : op.getTransp())
transp.push_back(attr.cast<IntegerAttr>().getInt());
if (vectorTransformOptions.vectorTransposeLowering ==
return rewriter.notifyMatchFailure(op, "Not a 2D transpose");
SmallVector<int64_t, 4> transp;
- for (auto attr : op.transp())
+ for (auto attr : op.getTransp())
transp.push_back(attr.cast<IntegerAttr>().getInt());
if (transp[0] != 1 && transp[1] != 0)
return rewriter.notifyMatchFailure(op, "Not a 2D transpose permutation");
int64_t m = srcType.getShape().front(), n = srcType.getShape().back();
Value casted = rewriter.create<vector::ShapeCastOp>(
- loc, VectorType::get({m * n}, srcType.getElementType()), op.vector());
+ loc, VectorType::get({m * n}, srcType.getElementType()),
+ op.getVector());
SmallVector<int64_t> mask;
mask.reserve(m * n);
for (int64_t j = 0; j < n; ++j)
VectorType resType = op.getVectorType();
Type eltType = resType.getElementType();
bool isInt = eltType.isa<IntegerType, IndexType>();
- Value acc = (op.acc().empty()) ? nullptr : op.acc()[0];
- vector::CombiningKind kind = op.kind();
+ Value acc = (op.getAcc().empty()) ? nullptr : op.getAcc()[0];
+ vector::CombiningKind kind = op.getKind();
if (!rhsType) {
// Special case: AXPY operation.
- Value b = rewriter.create<vector::BroadcastOp>(loc, lhsType, op.rhs());
+ Value b = rewriter.create<vector::BroadcastOp>(loc, lhsType, op.getRhs());
Optional<Value> mult =
- isInt ? genMultI(loc, op.lhs(), b, acc, kind, rewriter)
- : genMultF(loc, op.lhs(), b, acc, kind, rewriter);
+ isInt ? genMultI(loc, op.getLhs(), b, acc, kind, rewriter)
+ : genMultF(loc, op.getLhs(), b, acc, kind, rewriter);
if (!mult.hasValue())
return failure();
rewriter.replaceOp(op, mult.getValue());
loc, resType, rewriter.getZeroAttr(resType));
for (int64_t d = 0, e = resType.getDimSize(0); d < e; ++d) {
auto pos = rewriter.getI64ArrayAttr(d);
- Value x = rewriter.create<vector::ExtractOp>(loc, eltType, op.lhs(), pos);
+ Value x =
+ rewriter.create<vector::ExtractOp>(loc, eltType, op.getLhs(), pos);
Value a = rewriter.create<vector::BroadcastOp>(loc, rhsType, x);
Value r = nullptr;
if (acc)
r = rewriter.create<vector::ExtractOp>(loc, rhsType, acc, pos);
- Optional<Value> m = isInt ? genMultI(loc, a, op.rhs(), r, kind, rewriter)
- : genMultF(loc, a, op.rhs(), r, kind, rewriter);
+ Optional<Value> m =
+ isInt ? genMultI(loc, a, op.getRhs(), r, kind, rewriter)
+ : genMultF(loc, a, op.getRhs(), r, kind, rewriter);
if (!m.hasValue())
return failure();
result = rewriter.create<vector::InsertOp>(loc, resType, m.getValue(),
auto loc = op.getLoc();
auto dstType = op.getType();
auto eltType = dstType.getElementType();
- auto dimSizes = op.mask_dim_sizes();
+ auto dimSizes = op.getMaskDimSizes();
int64_t rank = dstType.getRank();
if (rank == 0) {
loc, resultVectorType, rewriter.getZeroAttr(resultVectorType));
unsigned mostMinorVectorSize = sourceVectorType.getShape()[1];
for (int64_t i = 0, e = sourceVectorType.getShape().front(); i != e; ++i) {
- Value vec = rewriter.create<vector::ExtractOp>(loc, op.source(), i);
+ Value vec = rewriter.create<vector::ExtractOp>(loc, op.getSource(), i);
desc = rewriter.create<vector::InsertStridedSliceOp>(
loc, vec, desc,
/*offsets=*/i * mostMinorVectorSize, /*strides=*/1);
unsigned mostMinorVectorSize = resultVectorType.getShape()[1];
for (int64_t i = 0, e = resultVectorType.getShape().front(); i != e; ++i) {
Value vec = rewriter.create<vector::ExtractStridedSliceOp>(
- loc, op.source(), /*offsets=*/i * mostMinorVectorSize,
+ loc, op.getSource(), /*offsets=*/i * mostMinorVectorSize,
/*sizes=*/mostMinorVectorSize,
/*strides=*/1);
desc = rewriter.create<vector::InsertOp>(loc, vec, desc, i);
incIdx(srcIdx, sourceVectorType, srcRank - 1);
incIdx(resIdx, resultVectorType, resRank - 1);
}
- Value e = rewriter.create<vector::ExtractOp>(loc, op.source(), srcIdx);
+ Value e = rewriter.create<vector::ExtractOp>(loc, op.getSource(), srcIdx);
result = rewriter.create<vector::InsertOp>(loc, e, result, resIdx);
}
rewriter.replaceOp(op, result);
LogicalResult matchAndRewrite(vector::MultiDimReductionOp reduceOp,
PatternRewriter &rewriter) const override {
- if (reduceOp.kind() != vector::CombiningKind::ADD)
+ if (reduceOp.getKind() != vector::CombiningKind::ADD)
return failure();
- Operation *mulOp = reduceOp.source().getDefiningOp();
+ Operation *mulOp = reduceOp.getSource().getDefiningOp();
if (!mulOp || !isa<arith::MulIOp, arith::MulFOp>(mulOp))
return failure();
SmallVector<bool> reductionMask = reduceOp.getReductionMask();
PatternRewriter &rewriter) const override {
SmallVector<AffineMap, 4> maps =
llvm::to_vector<4>(contractOp.getIndexingMaps());
- Value lhs = contractOp.lhs();
- Value rhs = contractOp.rhs();
+ Value lhs = contractOp.getLhs();
+ Value rhs = contractOp.getRhs();
size_t index = 0;
bool changed = false;
for (Value *operand : {&lhs, &rhs}) {
SmallVector<int64_t> perm;
transposeOp.getTransp(perm);
AffineMap permutationMap = AffineMap::getPermutationMap(
- extractVector<unsigned>(transposeOp.transp()),
+ extractVector<unsigned>(transposeOp.getTransp()),
contractOp.getContext());
map = inversePermutation(permutationMap).compose(map);
- *operand = transposeOp.vector();
+ *operand = transposeOp.getVector();
changed = true;
}
if (!changed)
return failure();
rewriter.replaceOpWithNewOp<vector::ContractionOp>(
- contractOp, lhs, rhs, contractOp.acc(),
- rewriter.getAffineMapArrayAttr(maps), contractOp.iterator_types());
+ contractOp, lhs, rhs, contractOp.getAcc(),
+ rewriter.getAffineMapArrayAttr(maps), contractOp.getIteratorTypes());
return success();
}
};
PatternRewriter &rewriter) const override {
SmallVector<AffineMap, 4> maps =
llvm::to_vector<4>(contractOp.getIndexingMaps());
- Value lhs = contractOp.lhs();
- Value rhs = contractOp.rhs();
+ Value lhs = contractOp.getLhs();
+ Value rhs = contractOp.getRhs();
size_t index = 0;
bool changed = false;
for (Value *operand : {&lhs, &rhs}) {
AffineMap::get(broadcast.getVectorType().getRank(), 0, originalDims,
contractOp.getContext());
map = broadcastMap.compose(map);
- *operand = broadcast.source();
+ *operand = broadcast.getSource();
changed = true;
}
if (!changed)
return failure();
rewriter.replaceOpWithNewOp<vector::ContractionOp>(
- contractOp, lhs, rhs, contractOp.acc(),
- rewriter.getAffineMapArrayAttr(maps), contractOp.iterator_types());
+ contractOp, lhs, rhs, contractOp.getAcc(),
+ rewriter.getAffineMapArrayAttr(maps), contractOp.getIteratorTypes());
return success();
}
};
Type castResTy = getElementTypeOrSelf(op->getResult(0));
if (auto vecTy = bcastOp.getSourceType().dyn_cast<VectorType>())
castResTy = VectorType::get(vecTy.getShape(), castResTy);
- auto castOp = rewriter.create(op->getLoc(), op->getName().getIdentifier(),
- bcastOp.source(), castResTy, op->getAttrs());
+ auto castOp =
+ rewriter.create(op->getLoc(), op->getName().getIdentifier(),
+ bcastOp.getSource(), castResTy, op->getAttrs());
rewriter.replaceOpWithNewOp<vector::BroadcastOp>(
op, op->getResult(0).getType(), castOp->getResult(0));
return success();
auto castResTy = transpOp.getVectorType();
castResTy = VectorType::get(castResTy.getShape(),
getElementTypeOrSelf(op->getResult(0)));
- auto castOp = rewriter.create(op->getLoc(), op->getName().getIdentifier(),
- transpOp.vector(), castResTy, op->getAttrs());
+ auto castOp =
+ rewriter.create(op->getLoc(), op->getName().getIdentifier(),
+ transpOp.getVector(), castResTy, op->getAttrs());
rewriter.replaceOpWithNewOp<vector::TransposeOp>(
op, op->getResult(0).getType(), castOp->getResult(0),
transpOp.getTransp());
ContractionOpToMatmulOpLowering::matchAndRewrite(vector::ContractionOp op,
PatternRewriter &rew) const {
// TODO: implement masks
- if (llvm::size(op.masks()) != 0)
+ if (llvm::size(op.getMasks()) != 0)
return failure();
if (vectorTransformOptions.vectorContractLowering !=
vector::VectorContractLowering::Matmul)
if (failed(filter(op)))
return failure();
- auto iteratorTypes = op.iterator_types().getValue();
+ auto iteratorTypes = op.getIteratorTypes().getValue();
if (!isParallelIterator(iteratorTypes[0]) ||
!isParallelIterator(iteratorTypes[1]) ||
!isReductionIterator(iteratorTypes[2]))
AffineExpr m, n, k;
bindDims(rew.getContext(), m, n, k);
// LHS must be A(m, k) or A(k, m).
- Value lhs = op.lhs();
- auto lhsMap = op.indexing_maps()[0];
+ Value lhs = op.getLhs();
+ auto lhsMap = op.getIndexingMaps()[0];
if (lhsMap == AffineMap::get(3, 0, {k, m}, ctx))
lhs = rew.create<vector::TransposeOp>(loc, lhs, ArrayRef<int64_t>{1, 0});
else if (lhsMap != AffineMap::get(3, 0, {m, k}, ctx))
return failure();
// RHS must be B(k, n) or B(n, k).
- Value rhs = op.rhs();
- auto rhsMap = op.indexing_maps()[1];
+ Value rhs = op.getRhs();
+ auto rhsMap = op.getIndexingMaps()[1];
if (rhsMap == AffineMap::get(3, 0, {n, k}, ctx))
rhs = rew.create<vector::TransposeOp>(loc, rhs, ArrayRef<int64_t>{1, 0});
else if (rhsMap != AffineMap::get(3, 0, {k, n}, ctx))
mul = rew.create<vector::ShapeCastOp>(
loc,
VectorType::get({lhsRows, rhsColumns},
- getElementTypeOrSelf(op.acc().getType())),
+ getElementTypeOrSelf(op.getAcc().getType())),
mul);
// ACC must be C(m, n) or C(n, m).
- auto accMap = op.indexing_maps()[2];
+ auto accMap = op.getIndexingMaps()[2];
if (accMap == AffineMap::get(3, 0, {n, m}, ctx))
mul = rew.create<vector::TransposeOp>(loc, mul, ArrayRef<int64_t>{1, 0});
else if (accMap != AffineMap::get(3, 0, {m, n}, ctx))
Value res =
elementType.isa<IntegerType>()
- ? static_cast<Value>(rew.create<arith::AddIOp>(loc, op.acc(), mul))
- : static_cast<Value>(rew.create<arith::AddFOp>(loc, op.acc(), mul));
+ ? static_cast<Value>(rew.create<arith::AddIOp>(loc, op.getAcc(), mul))
+ : static_cast<Value>(
+ rew.create<arith::AddFOp>(loc, op.getAcc(), mul));
rew.replaceOp(op, res);
return success();
/// This unrolls outer-products along the reduction dimension.
struct UnrolledOuterProductGenerator
: public StructuredGenerator<vector::ContractionOp> {
-
UnrolledOuterProductGenerator(OpBuilder &builder, vector::ContractionOp op)
: StructuredGenerator<vector::ContractionOp>(builder, op),
- kind(op.kind()), lhs(op.lhs()), rhs(op.rhs()), res(op.acc()),
- lhsType(op.getLhsType()) {}
+ kind(op.getKind()), lhs(op.getLhs()), rhs(op.getRhs()),
+ res(op.getAcc()), lhsType(op.getLhsType()) {}
Value t(Value v) {
static constexpr std::array<int64_t, 2> perm = {1, 0};
LogicalResult ContractionOpToOuterProductOpLowering::matchAndRewrite(
vector::ContractionOp op, PatternRewriter &rewriter) const {
// TODO: implement masks
- if (llvm::size(op.masks()) != 0)
+ if (llvm::size(op.getMasks()) != 0)
return failure();
if (vectorTransformOptions.vectorContractLowering !=
ContractionOpToDotLowering::matchAndRewrite(vector::ContractionOp op,
PatternRewriter &rewriter) const {
// TODO: implement masks
- if (llvm::size(op.masks()) != 0)
+ if (llvm::size(op.getMasks()) != 0)
return failure();
if (failed(filter(op)))
vector::VectorContractLowering::Dot)
return failure();
- auto iteratorTypes = op.iterator_types().getValue();
+ auto iteratorTypes = op.getIteratorTypes().getValue();
static constexpr std::array<int64_t, 2> perm = {1, 0};
Location loc = op.getLoc();
- Value lhs = op.lhs(), rhs = op.rhs();
+ Value lhs = op.getLhs(), rhs = op.getRhs();
using MapList = ArrayRef<ArrayRef<AffineExpr>>;
auto infer = [](MapList m) { return AffineMap::inferFromExprList(m); };
res = rewriter.create<vector::InsertOp>(op.getLoc(), reduced, res, pos);
}
}
- if (auto acc = op.acc())
+ if (auto acc = op.getAcc())
res = createAdd(op.getLoc(), res, acc, isInt, rewriter);
rewriter.replaceOp(op, res);
return success();
ContractionOpLowering::matchAndRewrite(vector::ContractionOp op,
PatternRewriter &rewriter) const {
// TODO: implement masks.
- if (llvm::size(op.masks()) != 0)
+ if (llvm::size(op.getMasks()) != 0)
return failure();
if (failed(filter(op)))
adjustMap(iMap[2], iterIndex, rewriter)};
auto lowAffine = rewriter.getAffineMapArrayAttr(lowIndexingMaps);
auto lowIter =
- rewriter.getArrayAttr(adjustIter(op.iterator_types(), iterIndex));
+ rewriter.getArrayAttr(adjustIter(op.getIteratorTypes(), iterIndex));
// Unroll into a series of lower dimensional vector.contract ops.
Location loc = op.getLoc();
Value result = rewriter.create<arith::ConstantOp>(
loc, resType, rewriter.getZeroAttr(resType));
for (int64_t d = 0; d < dimSize; ++d) {
- auto lhs = reshapeLoad(loc, op.lhs(), lhsType, lhsIndex, d, rewriter);
- auto rhs = reshapeLoad(loc, op.rhs(), rhsType, rhsIndex, d, rewriter);
- auto acc = reshapeLoad(loc, op.acc(), resType, resIndex, d, rewriter);
+ auto lhs = reshapeLoad(loc, op.getLhs(), lhsType, lhsIndex, d, rewriter);
+ auto rhs = reshapeLoad(loc, op.getRhs(), rhsType, rhsIndex, d, rewriter);
+ auto acc = reshapeLoad(loc, op.getAcc(), resType, resIndex, d, rewriter);
Value lowContract = rewriter.create<vector::ContractionOp>(
loc, lhs, rhs, acc, lowAffine, lowIter);
result =
// Base case.
if (lhsType.getRank() == 1) {
assert(rhsType.getRank() == 1 && "corrupt contraction");
- Value m = createMul(loc, op.lhs(), op.rhs(), isInt, rewriter);
+ Value m = createMul(loc, op.getLhs(), op.getRhs(), isInt, rewriter);
auto kind = vector::CombiningKind::ADD;
Value res = rewriter.create<vector::ReductionOp>(loc, kind, m);
- if (auto acc = op.acc())
+ if (auto acc = op.getAcc())
res = createAdd(op.getLoc(), res, acc, isInt, rewriter);
return res;
}
adjustMap(iMap[2], iterIndex, rewriter)};
auto lowAffine = rewriter.getAffineMapArrayAttr(lowIndexingMaps);
auto lowIter =
- rewriter.getArrayAttr(adjustIter(op.iterator_types(), iterIndex));
+ rewriter.getArrayAttr(adjustIter(op.getIteratorTypes(), iterIndex));
// Unroll into a series of lower dimensional vector.contract ops.
// By feeding the initial accumulator into the first contraction,
// and the result of each contraction into the next, eventually
// the sum of all reductions is computed.
- Value result = op.acc();
+ Value result = op.getAcc();
for (int64_t d = 0; d < dimSize; ++d) {
- auto lhs = reshapeLoad(loc, op.lhs(), lhsType, lhsIndex, d, rewriter);
- auto rhs = reshapeLoad(loc, op.rhs(), rhsType, rhsIndex, d, rewriter);
+ auto lhs = reshapeLoad(loc, op.getLhs(), lhsType, lhsIndex, d, rewriter);
+ auto rhs = reshapeLoad(loc, op.getRhs(), rhsType, rhsIndex, d, rewriter);
result = rewriter.create<vector::ContractionOp>(loc, lhs, rhs, result,
lowAffine, lowIter);
}
// Permutations are handled by VectorToSCF or
// populateVectorTransferPermutationMapLoweringPatterns.
// We let the 0-d corner case pass-through as it is supported.
- if (!read.permutation_map().isMinorIdentityWithBroadcasting(
+ if (!read.getPermutationMap().isMinorIdentityWithBroadcasting(
&broadcastedDims))
return failure();
// Create vector load op.
Operation *loadOp;
- if (read.mask()) {
+ if (read.getMask()) {
Value fill = rewriter.create<vector::SplatOp>(
- read.getLoc(), unbroadcastedVectorType, read.padding());
+ read.getLoc(), unbroadcastedVectorType, read.getPadding());
loadOp = rewriter.create<vector::MaskedLoadOp>(
- read.getLoc(), unbroadcastedVectorType, read.source(), read.indices(),
- read.mask(), fill);
+ read.getLoc(), unbroadcastedVectorType, read.getSource(),
+ read.getIndices(), read.getMask(), fill);
} else {
- loadOp = rewriter.create<vector::LoadOp>(read.getLoc(),
- unbroadcastedVectorType,
- read.source(), read.indices());
+ loadOp = rewriter.create<vector::LoadOp>(
+ read.getLoc(), unbroadcastedVectorType, read.getSource(),
+ read.getIndices());
}
// Insert a broadcasting op if required.
if (vecType.getNumElements() != 1)
return failure();
auto memrefLoad = rewriter.create<memref::LoadOp>(
- loadOp.getLoc(), loadOp.base(), loadOp.indices());
+ loadOp.getLoc(), loadOp.getBase(), loadOp.getIndices());
rewriter.replaceOpWithNewOp<vector::BroadcastOp>(loadOp, vecType,
memrefLoad);
return success();
if (vecType.getRank() == 0) {
// TODO: Unifiy once ExtractOp supports 0-d vectors.
extracted = rewriter.create<vector::ExtractElementOp>(
- storeOp.getLoc(), storeOp.valueToStore());
+ storeOp.getLoc(), storeOp.getValueToStore());
} else {
SmallVector<int64_t> indices(vecType.getRank(), 0);
extracted = rewriter.create<vector::ExtractOp>(
- storeOp.getLoc(), storeOp.valueToStore(), indices);
+ storeOp.getLoc(), storeOp.getValueToStore(), indices);
}
rewriter.replaceOpWithNewOp<memref::StoreOp>(
- storeOp, extracted, storeOp.base(), storeOp.indices());
+ storeOp, extracted, storeOp.getBase(), storeOp.getIndices());
return success();
}
};
// Permutations are handled by VectorToSCF or
// populateVectorTransferPermutationMapLoweringPatterns.
if ( // pass-through for the 0-d corner case.
- !write.permutation_map().isMinorIdentity())
+ !write.getPermutationMap().isMinorIdentity())
return failure();
auto memRefType = write.getShapedType().dyn_cast<MemRefType>();
// Out-of-bounds dims are handled by MaterializeTransferMask.
if (write.hasOutOfBoundsDim())
return failure();
- if (write.mask()) {
+ if (write.getMask()) {
rewriter.replaceOpWithNewOp<vector::MaskedStoreOp>(
- write, write.source(), write.indices(), write.mask(), write.vector());
+ write, write.getSource(), write.getIndices(), write.getMask(),
+ write.getVector());
} else {
rewriter.replaceOpWithNewOp<vector::StoreOp>(
- write, write.vector(), write.source(), write.indices());
+ write, write.getVector(), write.getSource(), write.getIndices());
}
return success();
}
if (extractOp.getVectorType().getRank() != 1)
return failure();
- auto castOp = extractOp.vector().getDefiningOp<vector::BitCastOp>();
+ auto castOp = extractOp.getVector().getDefiningOp<vector::BitCastOp>();
if (!castOp)
return failure();
return (*attr.getAsValueRange<IntegerAttr>().begin()).getZExtValue();
};
- uint64_t index = getFirstIntValue(extractOp.position());
+ uint64_t index = getFirstIntValue(extractOp.getPosition());
// Get the single scalar (as a vector) in the source value that packs the
// desired scalar. E.g. extract vector<1xf32> from vector<4xf32>
VectorType oneScalarType =
VectorType::get({1}, castSrcType.getElementType());
Value packedValue = rewriter.create<vector::ExtractOp>(
- extractOp.getLoc(), oneScalarType, castOp.source(),
+ extractOp.getLoc(), oneScalarType, castOp.getSource(),
rewriter.getI64ArrayAttr(index / expandRatio));
// Cast it to a vector with the desired scalar's type.
LogicalResult matchAndRewrite(vector::ExtractStridedSliceOp extractOp,
PatternRewriter &rewriter) const override {
- auto castOp = extractOp.vector().getDefiningOp<vector::BitCastOp>();
+ auto castOp = extractOp.getVector().getDefiningOp<vector::BitCastOp>();
if (!castOp)
return failure();
return failure();
// Only accept all one strides for now.
- if (llvm::any_of(extractOp.strides().getAsValueRange<IntegerAttr>(),
+ if (llvm::any_of(extractOp.getStrides().getAsValueRange<IntegerAttr>(),
[](const APInt &val) { return !val.isOneValue(); }))
return failure();
// are selecting the full range for the last bitcasted dimension; other
// dimensions aren't affected. Otherwise, we need to scale down the last
// dimension's offset given we are extracting from less elements now.
- ArrayAttr newOffsets = extractOp.offsets();
+ ArrayAttr newOffsets = extractOp.getOffsets();
if (newOffsets.size() == rank) {
SmallVector<int64_t, 4> offsets = getIntValueVector(newOffsets);
if (offsets.back() % expandRatio != 0)
}
// Similarly for sizes.
- ArrayAttr newSizes = extractOp.sizes();
+ ArrayAttr newSizes = extractOp.getSizes();
if (newSizes.size() == rank) {
SmallVector<int64_t, 4> sizes = getIntValueVector(newSizes);
if (sizes.back() % expandRatio != 0)
VectorType::get(dims, castSrcType.getElementType());
auto newExtractOp = rewriter.create<vector::ExtractStridedSliceOp>(
- extractOp.getLoc(), newExtractType, castOp.source(), newOffsets,
- newSizes, extractOp.strides());
+ extractOp.getLoc(), newExtractType, castOp.getSource(), newOffsets,
+ newSizes, extractOp.getStrides());
rewriter.replaceOpWithNewOp<vector::BitCastOp>(
extractOp, extractOp.getType(), newExtractOp);
int64_t shrinkRatio = castSrcLastDim / castDstLastDim;
auto insertOp =
- bitcastOp.source().getDefiningOp<vector::InsertStridedSliceOp>();
+ bitcastOp.getSource().getDefiningOp<vector::InsertStridedSliceOp>();
if (!insertOp)
return failure();
// Only accept all one strides for now.
- if (llvm::any_of(insertOp.strides().getAsValueRange<IntegerAttr>(),
+ if (llvm::any_of(insertOp.getStrides().getAsValueRange<IntegerAttr>(),
[](const APInt &val) { return !val.isOneValue(); }))
return failure();
if (rank != insertOp.getDestVectorType().getRank())
return failure();
- ArrayAttr newOffsets = insertOp.offsets();
+ ArrayAttr newOffsets = insertOp.getOffsets();
assert(newOffsets.size() == rank);
SmallVector<int64_t, 4> offsets = getIntValueVector(newOffsets);
if (offsets.back() % shrinkRatio != 0)
VectorType::get(srcDims, castDstType.getElementType());
auto newCastSrcOp = rewriter.create<vector::BitCastOp>(
- bitcastOp.getLoc(), newCastSrcType, insertOp.source());
+ bitcastOp.getLoc(), newCastSrcType, insertOp.getSource());
SmallVector<int64_t, 4> dstDims =
llvm::to_vector<4>(insertOp.getDestVectorType().getShape());
VectorType::get(dstDims, castDstType.getElementType());
auto newCastDstOp = rewriter.create<vector::BitCastOp>(
- bitcastOp.getLoc(), newCastDstType, insertOp.dest());
+ bitcastOp.getLoc(), newCastDstType, insertOp.getDest());
rewriter.replaceOpWithNewOp<vector::InsertStridedSliceOp>(
bitcastOp, bitcastOp.getType(), newCastSrcOp, newCastDstOp, newOffsets,
- insertOp.strides());
+ insertOp.getStrides());
return success();
}
return failure();
if (xferOp.getVectorType().getRank() > 1 ||
- llvm::size(xferOp.indices()) == 0)
+ llvm::size(xferOp.getIndices()) == 0)
return failure();
Location loc = xferOp->getLoc();
//
// TODO: when the leaf transfer rank is k > 1, we need the last `k`
// dimensions here.
- unsigned lastIndex = llvm::size(xferOp.indices()) - 1;
- Value off = xferOp.indices()[lastIndex];
+ unsigned lastIndex = llvm::size(xferOp.getIndices()) - 1;
+ Value off = xferOp.getIndices()[lastIndex];
Value dim =
- vector::createOrFoldDimOp(rewriter, loc, xferOp.source(), lastIndex);
+ vector::createOrFoldDimOp(rewriter, loc, xferOp.getSource(), lastIndex);
Value b = rewriter.create<arith::SubIOp>(loc, dim.getType(), dim, off);
Value mask = rewriter.create<vector::CreateMaskOp>(
loc,
VectorType::get(vtp.getShape(), rewriter.getI1Type(),
vtp.getNumScalableDims()),
b);
- if (xferOp.mask()) {
+ if (xferOp.getMask()) {
// Intersect the in-bounds with the mask specified as an op parameter.
- mask = rewriter.create<arith::AndIOp>(loc, mask, xferOp.mask());
+ mask = rewriter.create<arith::AndIOp>(loc, mask, xferOp.getMask());
}
rewriter.updateRootInPlace(xferOp, [&]() {
- xferOp.maskMutable().assign(mask);
- xferOp.in_boundsAttr(rewriter.getBoolArrayAttr({true}));
+ xferOp.getMaskMutable().assign(mask);
+ xferOp.setInBoundsAttr(rewriter.getBoolArrayAttr({true}));
});
return success();
return failure();
// TODO: support mask.
- if (readOp.mask())
+ if (readOp.getMask())
return failure();
- auto srcType = readOp.source().getType().dyn_cast<MemRefType>();
+ auto srcType = readOp.getSource().getType().dyn_cast<MemRefType>();
if (!srcType || !srcType.hasStaticShape())
return failure();
- if (!readOp.permutation_map().isMinorIdentity())
+ if (!readOp.getPermutationMap().isMinorIdentity())
return failure();
auto targetType = readOp.getVectorType();
SmallVector<int64_t> strides(srcType.getRank(), 1);
ArrayAttr inBoundsAttr =
- readOp.in_bounds()
+ readOp.getInBounds()
? rewriter.getArrayAttr(
- readOp.in_boundsAttr().getValue().drop_back(dimsToDrop))
+ readOp.getInBoundsAttr().getValue().drop_back(dimsToDrop))
: ArrayAttr();
Value rankedReducedView = rewriter.create<memref::SubViewOp>(
- loc, resultMemrefType, readOp.source(), offsets, srcType.getShape(),
+ loc, resultMemrefType, readOp.getSource(), offsets, srcType.getShape(),
strides);
auto permMap = getTransferMinorIdentityMap(
rankedReducedView.getType().cast<ShapedType>(), resultTargetVecType);
Value result = rewriter.create<vector::TransferReadOp>(
loc, resultTargetVecType, rankedReducedView,
- readOp.indices().drop_back(dimsToDrop), AffineMapAttr::get(permMap),
- readOp.padding(),
+ readOp.getIndices().drop_back(dimsToDrop), AffineMapAttr::get(permMap),
+ readOp.getPadding(),
// TODO: support mask.
/*mask=*/Value(), inBoundsAttr);
rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(readOp, targetType,
ArrayRef<int64_t> destShape = destType.getShape();
auto elType = destType.getElementType();
bool isInt = elType.isIntOrIndex();
- if (!isValidKind(isInt, scanOp.kind()))
+ if (!isValidKind(isInt, scanOp.getKind()))
return failure();
VectorType resType = VectorType::get(destShape, elType);
Value result = rewriter.create<arith::ConstantOp>(
loc, resType, rewriter.getZeroAttr(resType));
- int64_t reductionDim = scanOp.reduction_dim();
- bool inclusive = scanOp.inclusive();
+ int64_t reductionDim = scanOp.getReductionDim();
+ bool inclusive = scanOp.getInclusive();
int64_t destRank = destType.getRank();
VectorType initialValueType = scanOp.getInitialValueType();
int64_t initialValueRank = initialValueType.getRank();
offsets[reductionDim] = i;
ArrayAttr scanOffsets = rewriter.getI64ArrayAttr(offsets);
Value input = rewriter.create<vector::ExtractStridedSliceOp>(
- loc, reductionType, scanOp.source(), scanOffsets, scanSizes,
+ loc, reductionType, scanOp.getSource(), scanOffsets, scanSizes,
scanStrides);
Value output;
if (i == 0) {
if (initialValueRank == 0) {
// ShapeCastOp cannot handle 0-D vectors
output = rewriter.create<vector::BroadcastOp>(
- loc, input.getType(), scanOp.initial_value());
+ loc, input.getType(), scanOp.getInitialValue());
} else {
output = rewriter.create<vector::ShapeCastOp>(
- loc, input.getType(), scanOp.initial_value());
+ loc, input.getType(), scanOp.getInitialValue());
}
}
} else {
Value y = inclusive ? input : lastInput;
- output = genOperator(loc, lastOutput, y, scanOp.kind(), rewriter);
+ output = genOperator(loc, lastOutput, y, scanOp.getKind(), rewriter);
assert(output != nullptr);
}
result = rewriter.create<vector::InsertStridedSliceOp>(
// TODO: support 0-d corner case.
if (readOp.getTransferRank() == 0)
return failure();
- if (readOp.mask())
+ if (readOp.getMask())
return failure();
auto targetShape = getTargetShape(options, readOp);
if (!targetShape)
loc, sourceVectorType, rewriter.getZeroAttr(sourceVectorType));
auto targetType =
VectorType::get(*targetShape, sourceVectorType.getElementType());
- SmallVector<Value, 4> originalIndices(readOp.indices().begin(),
- readOp.indices().end());
+ SmallVector<Value, 4> originalIndices(readOp.getIndices().begin(),
+ readOp.getIndices().end());
for (int64_t i = 0; i < sliceCount; i++) {
SmallVector<Value, 4> indices =
sliceTransferIndices(i, originalSize, *targetShape, originalIndices,
- readOp.permutation_map(), loc, rewriter);
+ readOp.getPermutationMap(), loc, rewriter);
auto slicedRead = rewriter.create<vector::TransferReadOp>(
- loc, targetType, readOp.source(), indices,
- readOp.permutation_mapAttr(), readOp.padding(), readOp.mask(),
- readOp.in_boundsAttr());
+ loc, targetType, readOp.getSource(), indices,
+ readOp.getPermutationMapAttr(), readOp.getPadding(), readOp.getMask(),
+ readOp.getInBoundsAttr());
SmallVector<int64_t, 4> elementOffsets =
getVectorOffset(originalSize, *targetShape, i);
if (writeOp.getTransferRank() == 0)
return failure();
- if (writeOp.mask())
+ if (writeOp.getMask())
return failure();
auto targetShape = getTargetShape(options, writeOp);
if (!targetShape)
SmallVector<int64_t, 4> ratio = *shapeRatio(originalSize, *targetShape);
// Compute shape ratio of 'shape' and 'sizes'.
int64_t sliceCount = computeMaxLinearIndex(ratio);
- SmallVector<Value, 4> originalIndices(writeOp.indices().begin(),
- writeOp.indices().end());
+ SmallVector<Value, 4> originalIndices(writeOp.getIndices().begin(),
+ writeOp.getIndices().end());
Value resultTensor;
for (int64_t i = 0; i < sliceCount; i++) {
SmallVector<int64_t, 4> elementOffsets =
getVectorOffset(originalSize, *targetShape, i);
Value slicedVector = rewriter.create<vector::ExtractStridedSliceOp>(
- loc, writeOp.vector(), elementOffsets, *targetShape, strides);
+ loc, writeOp.getVector(), elementOffsets, *targetShape, strides);
SmallVector<Value, 4> indices =
sliceTransferIndices(i, originalSize, *targetShape, originalIndices,
- writeOp.permutation_map(), loc, rewriter);
+ writeOp.getPermutationMap(), loc, rewriter);
Operation *slicedWrite = rewriter.create<vector::TransferWriteOp>(
- loc, slicedVector, resultTensor ? resultTensor : writeOp.source(),
- indices, writeOp.permutation_mapAttr(), writeOp.in_boundsAttr());
+ loc, slicedVector, resultTensor ? resultTensor : writeOp.getSource(),
+ indices, writeOp.getPermutationMapAttr(), writeOp.getInBoundsAttr());
// For the tensor case update the destination for the next transfer write.
if (!slicedWrite->getResults().empty())
resultTensor = slicedWrite->getResult(0);
AffineMap lhsPermutationMap = contractOp.getIndexingMaps()[0];
SmallVector<int64_t> lhsOffets =
applyPermutationMap(lhsPermutationMap, ArrayRef<int64_t>(offsets));
- extractOperand(0, contractOp.lhs(), lhsPermutationMap, lhsOffets);
+ extractOperand(0, contractOp.getLhs(), lhsPermutationMap, lhsOffets);
// If there is a mask associated to lhs, extract it as well.
if (slicesOperands.size() > 3)
- extractOperand(3, contractOp.masks()[0], lhsPermutationMap, lhsOffets);
+ extractOperand(3, contractOp.getMasks()[0], lhsPermutationMap,
+ lhsOffets);
// Extract the new rhs operand.
AffineMap rhsPermutationMap = contractOp.getIndexingMaps()[1];
SmallVector<int64_t> rhsOffets =
applyPermutationMap(rhsPermutationMap, ArrayRef<int64_t>(offsets));
- extractOperand(1, contractOp.rhs(), rhsPermutationMap, rhsOffets);
+ extractOperand(1, contractOp.getRhs(), rhsPermutationMap, rhsOffets);
// If there is a mask associated to rhs, extract it as well.
if (slicesOperands.size() > 4)
- extractOperand(4, contractOp.masks()[1], rhsPermutationMap, rhsOffets);
+ extractOperand(4, contractOp.getMasks()[1], rhsPermutationMap,
+ rhsOffets);
AffineMap accPermutationMap = contractOp.getIndexingMaps()[2];
SmallVector<int64_t> accOffets =
if (accIt != accCache.end())
slicesOperands[2] = accIt->second;
else
- extractOperand(2, contractOp.acc(), accPermutationMap, accOffets);
+ extractOperand(2, contractOp.getAcc(), accPermutationMap, accOffets);
SmallVector<int64_t> dstShape =
applyPermutationMap(dstAffineMap, ArrayRef<int64_t>(*targetShape));
// reduction loop keeps updating the accumulator.
auto accIt = accCache.find(destOffset);
if (accIt != accCache.end())
- result = makeArithReduction(rewriter, loc, reductionOp.kind(), result,
- accIt->second);
+ result = makeArithReduction(rewriter, loc, reductionOp.getKind(),
+ result, accIt->second);
accCache[destOffset] = result;
}
// Assemble back the accumulator into a single vector.
using OpRewritePattern<vector::ExtractMapOp>::OpRewritePattern;
LogicalResult matchAndRewrite(vector::ExtractMapOp extract,
PatternRewriter &rewriter) const override {
- Operation *definedOp = extract.vector().getDefiningOp();
+ Operation *definedOp = extract.getVector().getDefiningOp();
if (!definedOp || !OpTrait::hasElementwiseMappableTraits(definedOp) ||
definedOp->getNumResults() != 1)
return failure();
loc,
VectorType::get(extract.getResultType().getShape(),
vecType.getElementType()),
- operand.get(), extract.ids()));
+ operand.get(), extract.getIds()));
}
Operation *newOp = cloneOpWithOperandsAndTypes(
rewriter, loc, definedOp, extractOperands, extract.getResultType());
using OpRewritePattern<vector::ExtractMapOp>::OpRewritePattern;
LogicalResult matchAndRewrite(vector::ExtractMapOp extract,
PatternRewriter &rewriter) const override {
- Operation *definedOp = extract.vector().getDefiningOp();
+ Operation *definedOp = extract.getVector().getDefiningOp();
auto contract = dyn_cast_or_null<vector::ContractionOp>(definedOp);
if (!contract)
return failure();
VectorType newVecType =
VectorType::get(operandShape, vecType.getElementType());
extractOperands.push_back(rewriter.create<vector::ExtractMapOp>(
- loc, newVecType, operand, extract.ids()));
+ loc, newVecType, operand, extract.getIds()));
}
Operation *newOp =
cloneOpWithOperandsAndTypes(rewriter, loc, definedOp, extractOperands,
dyn_cast<vector::ExtractMapOp>(*read.getResult().getUsers().begin());
if (!extract)
return failure();
- if (read.mask())
+ if (read.getMask())
return failure();
- SmallVector<Value, 4> indices(read.indices().begin(), read.indices().end());
- AffineMap indexMap = extract.map().compose(read.permutation_map());
+ SmallVector<Value, 4> indices(read.getIndices().begin(),
+ read.getIndices().end());
+ AffineMap indexMap = extract.map().compose(read.getPermutationMap());
unsigned idCount = 0;
ImplicitLocOpBuilder lb(read.getLoc(), rewriter);
for (auto it :
extract.getResultType().getDimSize(vectorPos), read.getContext());
indices[indexPos] = makeComposedAffineApply(
rewriter, read.getLoc(), d0 + scale * d1,
- {indices[indexPos], extract.ids()[idCount++]});
+ {indices[indexPos], extract.getIds()[idCount++]});
}
Value newRead = lb.create<vector::TransferReadOp>(
- extract.getType(), read.source(), indices, read.permutation_mapAttr(),
- read.padding(), read.mask(), read.in_boundsAttr());
+ extract.getType(), read.getSource(), indices,
+ read.getPermutationMapAttr(), read.getPadding(), read.getMask(),
+ read.getInBoundsAttr());
Value dest = lb.create<arith::ConstantOp>(
read.getType(), rewriter.getZeroAttr(read.getType()));
- newRead = lb.create<vector::InsertMapOp>(newRead, dest, extract.ids());
+ newRead = lb.create<vector::InsertMapOp>(newRead, dest, extract.getIds());
rewriter.replaceOp(read, newRead);
return success();
}
if (write.getTransferRank() == 0)
return failure();
- auto insert = write.vector().getDefiningOp<vector::InsertMapOp>();
+ auto insert = write.getVector().getDefiningOp<vector::InsertMapOp>();
if (!insert)
return failure();
- if (write.mask())
+ if (write.getMask())
return failure();
- SmallVector<Value, 4> indices(write.indices().begin(),
- write.indices().end());
- AffineMap indexMap = insert.map().compose(write.permutation_map());
+ SmallVector<Value, 4> indices(write.getIndices().begin(),
+ write.getIndices().end());
+ AffineMap indexMap = insert.map().compose(write.getPermutationMap());
unsigned idCount = 0;
Location loc = write.getLoc();
for (auto it :
auto scale = getAffineConstantExpr(
insert.getSourceVectorType().getDimSize(vectorPos),
write.getContext());
- indices[indexPos] =
- makeComposedAffineApply(rewriter, loc, d0 + scale * d1,
- {indices[indexPos], insert.ids()[idCount++]});
+ indices[indexPos] = makeComposedAffineApply(
+ rewriter, loc, d0 + scale * d1,
+ {indices[indexPos], insert.getIds()[idCount++]});
}
rewriter.create<vector::TransferWriteOp>(
- loc, insert.vector(), write.source(), indices,
- write.permutation_mapAttr(), write.in_boundsAttr());
+ loc, insert.getVector(), write.getSource(), indices,
+ write.getPermutationMapAttr(), write.getInBoundsAttr());
rewriter.eraseOp(write);
return success();
}
getVectorOffset(originalSize, *targetShape, i);
SmallVector<int64_t> strides(offsets.size(), 1);
Value slicedOperand = rewriter.create<vector::ExtractStridedSliceOp>(
- loc, reductionOp.vector(), offsets, *targetShape, strides);
+ loc, reductionOp.getVector(), offsets, *targetShape, strides);
Operation *newOp = cloneOpWithOperandsAndTypes(
rewriter, loc, reductionOp, slicedOperand, reductionOp.getType());
Value result = newOp->getResult(0);
accumulator = result;
} else {
// On subsequent reduction, combine with the accumulator.
- accumulator = makeArithReduction(rewriter, loc, reductionOp.kind(),
+ accumulator = makeArithReduction(rewriter, loc, reductionOp.getKind(),
accumulator, result);
}
}
return rewriter.notifyMatchFailure(op, "Unsupported vector type");
SmallVector<int64_t, 4> transp;
- for (auto attr : op.transp())
+ for (auto attr : op.getTransp())
transp.push_back(attr.cast<IntegerAttr>().getInt());
// Check whether the two source vector dimensions that are greater than one
VectorType::get({n * m}, op.getVectorType().getElementType());
auto reshInputType = VectorType::get({m, n}, srcType.getElementType());
auto reshInput =
- ib.create<vector::ShapeCastOp>(flattenedType, op.vector());
+ ib.create<vector::ShapeCastOp>(flattenedType, op.getVector());
reshInput = ib.create<vector::ShapeCastOp>(reshInputType, reshInput);
// Extract 1-D vectors from the higher-order dimension of the input
dstVec.getShape().end());
}
if (auto writeOp = dyn_cast<vector::TransferWriteOp>(op)) {
- auto insert = writeOp.vector().getDefiningOp<InsertStridedSliceOp>();
+ auto insert = writeOp.getVector().getDefiningOp<InsertStridedSliceOp>();
if (!insert)
return llvm::None;
ArrayRef<int64_t> shape = insert.getSourceVectorType().getShape();
projects / platform / upstream / llvm.git / commitdiff