1 // This file is part of OpenCV project.
2 // It is subject to the license terms in the LICENSE file found in the top-level directory
3 // of this distribution and at http://opencv.org/license.html.
5 // Copyright (C) 2018-2020 Intel Corporation
8 #ifndef OPENCV_GAPI_GKERNEL_HPP
9 #define OPENCV_GAPI_GKERNEL_HPP
13 #include <string> // string
14 #include <type_traits> // false_type, true_type
15 #include <unordered_map> // map (for GKernelPackage)
16 #include <utility> // tuple
18 #include <opencv2/gapi/gcommon.hpp> // CompileArgTag
19 #include <opencv2/gapi/util/util.hpp> // Seq
20 #include <opencv2/gapi/gcall.hpp>
21 #include <opencv2/gapi/garg.hpp> // GArg
22 #include <opencv2/gapi/gmetaarg.hpp> // GMetaArg
23 #include <opencv2/gapi/gtype_traits.hpp> // GTypeTraits
24 #include <opencv2/gapi/util/compiler_hints.hpp> //suppress_unused_warning
25 #include <opencv2/gapi/gtransform.hpp>
29 using GShapes = std::vector<GShape>;
30 using GKinds = std::vector<cv::detail::OpaqueKind>;
32 // GKernel describes kernel API to the system
33 // FIXME: add attributes of a kernel, (e.g. number and types
35 struct GAPI_EXPORTS GKernel
37 using M = std::function<GMetaArgs(const GMetaArgs &, const GArgs &)>;
39 std::string name; // kernel ID, defined by its API (signature)
40 std::string tag; // some (implementation-specific) tag
41 M outMeta; // generic adaptor to API::outMeta(...)
42 GShapes outShapes; // types (shapes) kernel's outputs
43 GKinds inKinds; // kinds of kernel's inputs (fixme: below)
45 // TODO: It's questionable if inKinds should really be here. Instead,
46 // this information could come from meta.
48 // GKernelImpl describes particular kernel implementation to the system
49 struct GAPI_EXPORTS GKernelImpl
51 util::any opaque; // backend-specific opaque info
52 GKernel::M outMeta; // for deserialized graphs, the outMeta is taken here
55 template<typename, typename> class GKernelTypeM;
59 ////////////////////////////////////////////////////////////////////////////
60 // yield() is used in graph construction time as a generic method to obtain
61 // lazy "return value" of G-API operations
65 template<typename T> struct Yield;
66 template<> struct Yield<cv::GMat>
68 static inline cv::GMat yield(cv::GCall &call, int i) { return call.yield(i); }
70 template<> struct Yield<cv::GMatP>
72 static inline cv::GMatP yield(cv::GCall &call, int i) { return call.yieldP(i); }
74 template<> struct Yield<cv::GScalar>
76 static inline cv::GScalar yield(cv::GCall &call, int i) { return call.yieldScalar(i); }
78 template<typename U> struct Yield<cv::GArray<U> >
80 static inline cv::GArray<U> yield(cv::GCall &call, int i) { return call.yieldArray<U>(i); }
82 template<typename U> struct Yield<cv::GOpaque<U> >
84 static inline cv::GOpaque<U> yield(cv::GCall &call, int i) { return call.yieldOpaque<U>(i); }
86 } // anonymous namespace
88 ////////////////////////////////////////////////////////////////////////////
89 // Helper classes which brings outputMeta() marshalling to kernel
92 // 1. MetaType establishes G#Type -> G#Meta mapping between G-API dynamic
93 // types and its metadata descriptor types.
94 // This mapping is used to transform types to call outMeta() callback.
95 template<typename T> struct MetaType;
96 template<> struct MetaType<cv::GMat> { using type = GMatDesc; };
97 template<> struct MetaType<cv::GMatP> { using type = GMatDesc; };
98 template<> struct MetaType<cv::GFrame> { using type = GFrameDesc; };
99 template<> struct MetaType<cv::GScalar> { using type = GScalarDesc; };
100 template<typename U> struct MetaType<cv::GArray<U> > { using type = GArrayDesc; };
101 template<typename U> struct MetaType<cv::GOpaque<U> > { using type = GOpaqueDesc; };
102 template<typename T> struct MetaType { using type = T; }; // opaque args passed as-is
103 // FIXME: Move it to type traits?
105 // 2. Hacky test based on MetaType to check if we operate on G-* type or not
106 template<typename T> using is_nongapi_type = std::is_same<T, typename MetaType<T>::type>;
108 // 3. Two ways to transform input arguments to its meta - for G-* and non-G* types:
110 typename std::enable_if<!is_nongapi_type<T>::value, typename MetaType<T>::type>
111 ::type get_in_meta(const GMetaArgs &in_meta, const GArgs &, int idx)
113 return util::get<typename MetaType<T>::type>(in_meta.at(idx));
117 typename std::enable_if<is_nongapi_type<T>::value, T>
118 ::type get_in_meta(const GMetaArgs &, const GArgs &in_args, int idx)
120 return in_args.at(idx).template get<T>();
123 // 4. The MetaHelper itself: an entity which generates outMeta() call
124 // based on kernel signature, with arguments properly substituted.
125 // 4.1 - case for multiple return values
126 // FIXME: probably can be simplified with std::apply or analogue.
127 template<typename, typename, typename>
130 template<typename K, typename... Ins, typename... Outs>
131 struct MetaHelper<K, std::tuple<Ins...>, std::tuple<Outs...> >
133 template<int... IIs, int... OIs>
134 static GMetaArgs getOutMeta_impl(const GMetaArgs &in_meta,
135 const GArgs &in_args,
140 using R = std::tuple<typename MetaType<Outs>::type...>;
141 const R r = K::outMeta( get_in_meta<Ins>(in_meta, in_args, IIs)... );
142 return GMetaArgs{ GMetaArg(std::get<OIs>(r))... };
144 // FIXME: help users identify how outMeta must look like (via default impl w/static_assert?)
146 static GMetaArgs getOutMeta(const GMetaArgs &in_meta,
147 const GArgs &in_args)
149 return getOutMeta_impl(in_meta,
151 typename detail::MkSeq<sizeof...(Ins)>::type(),
152 typename detail::MkSeq<sizeof...(Outs)>::type());
156 // 4.1 - case for a single return value
157 // FIXME: How to avoid duplication here?
158 template<typename K, typename... Ins, typename Out>
159 struct MetaHelper<K, std::tuple<Ins...>, Out >
162 static GMetaArgs getOutMeta_impl(const GMetaArgs &in_meta,
163 const GArgs &in_args,
167 using R = typename MetaType<Out>::type;
168 const R r = K::outMeta( get_in_meta<Ins>(in_meta, in_args, IIs)... );
169 return GMetaArgs{ GMetaArg(r) };
171 // FIXME: help users identify how outMeta must look like (via default impl w/static_assert?)
173 static GMetaArgs getOutMeta(const GMetaArgs &in_meta,
174 const GArgs &in_args)
176 return getOutMeta_impl(in_meta,
178 typename detail::MkSeq<sizeof...(Ins)>::type());
182 ////////////////////////////////////////////////////////////////////////////
183 // Helper class to introduce tags to calls. By default there's no tag
185 static constexpr const char *tag() { return ""; }
188 } // namespace detail
190 // GKernelType and GKernelTypeM are base classes which implement typed ::on()
191 // method based on kernel signature. GKernelTypeM stands for multiple-return-value kernels
193 // G_TYPED_KERNEL and G_TYPED_KERNEL_M macros inherit user classes from GKernelType and
194 // GKernelTypeM respectively.
196 template<typename K, typename... R, typename... Args>
197 class GKernelTypeM<K, std::function<std::tuple<R...>(Args...)> >
198 : public detail::MetaHelper<K, std::tuple<Args...>, std::tuple<R...>>
199 , public detail::NoTag
202 static std::tuple<R...> yield(cv::GCall &call, detail::Seq<IIs...>)
204 return std::make_tuple(detail::Yield<R>::yield(call, IIs)...);
208 using InArgs = std::tuple<Args...>;
209 using OutArgs = std::tuple<R...>;
211 // TODO: Args&&... here?
212 static std::tuple<R...> on(Args... args)
214 cv::GCall call(GKernel{ K::id()
217 , {detail::GTypeTraits<R>::shape...}
218 , {detail::GTypeTraits<Args>::op_kind...}});
219 call.pass(args...); // TODO: std::forward() here?
220 return yield(call, typename detail::MkSeq<sizeof...(R)>::type());
224 template<typename, typename> class GKernelType;
226 template<typename K, typename R, typename... Args>
227 class GKernelType<K, std::function<R(Args...)> >
228 : public detail::MetaHelper<K, std::tuple<Args...>, R>
229 , public detail::NoTag
232 using InArgs = std::tuple<Args...>;
233 using OutArgs = std::tuple<R>;
235 static_assert(!cv::detail::contains<GFrame, OutArgs>::value, "Values of GFrame type can't be used as operation outputs");
237 static R on(Args... args)
239 cv::GCall call(GKernel{ K::id()
242 , {detail::GTypeTraits<R>::shape}
243 , {detail::GTypeTraits<Args>::op_kind...}});
245 return detail::Yield<R>::yield(call, 0);
250 // This tiny class eliminates the semantic difference between
251 // GKernelType and GKernelTypeM.
252 template<typename, typename> class KernelTypeMedium;
254 template<typename K, typename... R, typename... Args>
255 class KernelTypeMedium<K, std::function<std::tuple<R...>(Args...)>> :
256 public cv::GKernelTypeM<K, std::function<std::tuple<R...>(Args...)>> {};
258 template<typename K, typename R, typename... Args>
259 class KernelTypeMedium<K, std::function<R(Args...)>> :
260 public cv::GKernelType<K, std::function<R(Args...)>> {};
261 } // namespace detail
266 // FIXME: I don't know a better way so far. Feel free to suggest one
267 // The problem is that every typed kernel should have ::id() but body
268 // of the class is defined by user (with outMeta, other stuff)
271 #define G_ID_HELPER_CLASS(Class) Class##IdHelper
273 #define G_ID_HELPER_BODY(Class, Id) \
274 struct G_ID_HELPER_CLASS(Class) \
276 static constexpr const char * id() {return Id;} \
280 #define GET_G_TYPED_KERNEL(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, NAME, ...) NAME
281 #define COMBINE_SIGNATURE(...) __VA_ARGS__
282 // Ensure correct __VA_ARGS__ expansion on Windows
283 #define __WRAP_VAARGS(x) x
286 * Helper for G_TYPED_KERNEL declares a new G-API Operation. See [Kernel API](@ref gapi_kernel_api)
289 * @param Class type name for this operation.
290 * @param API an `std::function<>`-like signature for the operation;
291 * return type is a single value or a tuple of multiple values.
292 * @param Id string identifier for the operation. Must be unique.
294 #define G_TYPED_KERNEL_HELPER(Class, API, Id) \
295 G_ID_HELPER_BODY(Class, Id) \
296 struct Class final: public cv::detail::KernelTypeMedium<Class, std::function API >, \
297 public G_ID_HELPER_CLASS(Class)
298 // {body} is to be defined by user
300 #define G_TYPED_KERNEL_HELPER_2(Class, _1, _2, Id) \
301 G_TYPED_KERNEL_HELPER(Class, COMBINE_SIGNATURE(_1, _2), Id)
303 #define G_TYPED_KERNEL_HELPER_3(Class, _1, _2, _3, Id) \
304 G_TYPED_KERNEL_HELPER(Class, COMBINE_SIGNATURE(_1, _2, _3), Id)
306 #define G_TYPED_KERNEL_HELPER_4(Class, _1, _2, _3, _4, Id) \
307 G_TYPED_KERNEL_HELPER(Class, COMBINE_SIGNATURE(_1, _2, _3, _4), Id)
309 #define G_TYPED_KERNEL_HELPER_5(Class, _1, _2, _3, _4, _5, Id) \
310 G_TYPED_KERNEL_HELPER(Class, COMBINE_SIGNATURE(_1, _2, _3, _4, _5), Id)
312 #define G_TYPED_KERNEL_HELPER_6(Class, _1, _2, _3, _4, _5, _6, Id) \
313 G_TYPED_KERNEL_HELPER(Class, COMBINE_SIGNATURE(_1, _2, _3, _4, _5, _6), Id)
315 #define G_TYPED_KERNEL_HELPER_7(Class, _1, _2, _3, _4, _5, _6, _7, Id) \
316 G_TYPED_KERNEL_HELPER(Class, COMBINE_SIGNATURE(_1, _2, _3, _4, _5, _6, _7), Id)
318 #define G_TYPED_KERNEL_HELPER_8(Class, _1, _2, _3, _4, _5, _6, _7, _8, Id) \
319 G_TYPED_KERNEL_HELPER(Class, COMBINE_SIGNATURE(_1, _2, _3, _4, _5, _6, _7, _8), Id)
321 #define G_TYPED_KERNEL_HELPER_9(Class, _1, _2, _3, _4, _5, _6, _7, _8, _9, Id) \
322 G_TYPED_KERNEL_HELPER(Class, COMBINE_SIGNATURE(_1, _2, _3, _4, _5, _6, _7, _8, _9), Id)
324 #define G_TYPED_KERNEL_HELPER_10(Class, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, Id) \
325 G_TYPED_KERNEL_HELPER(Class, COMBINE_SIGNATURE(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10), Id)
328 * Declares a new G-API Operation. See [Kernel API](@ref gapi_kernel_api)
331 * @param Class type name for this operation.
333 #define G_TYPED_KERNEL(Class, ...) __WRAP_VAARGS(GET_G_TYPED_KERNEL(__VA_ARGS__, \
334 G_TYPED_KERNEL_HELPER_10, \
335 G_TYPED_KERNEL_HELPER_9, \
336 G_TYPED_KERNEL_HELPER_8, \
337 G_TYPED_KERNEL_HELPER_7, \
338 G_TYPED_KERNEL_HELPER_6, \
339 G_TYPED_KERNEL_HELPER_5, \
340 G_TYPED_KERNEL_HELPER_4, \
341 G_TYPED_KERNEL_HELPER_3, \
342 G_TYPED_KERNEL_HELPER_2, \
343 G_TYPED_KERNEL_HELPER)(Class, __VA_ARGS__)) \
346 * Declares a new G-API Operation. See [Kernel API](@ref gapi_kernel_api) for more details.
348 * @deprecated This macro is deprecated in favor of `G_TYPED_KERNEL` that is used for declaring any
351 * @param Class type name for this operation.
353 #define G_TYPED_KERNEL_M G_TYPED_KERNEL
355 #define G_API_OP G_TYPED_KERNEL
356 #define G_API_OP_M G_API_OP
362 // Prework: model "Device" API before it gets to G-API headers.
363 // FIXME: Don't mix with internal Backends class!
364 class GAPI_EXPORTS GBackend
369 // TODO: make it template (call `new` within??)
371 explicit GBackend(std::shared_ptr<Priv> &&p);
374 const Priv& priv() const;
375 std::size_t hash() const;
377 bool operator== (const GBackend &rhs) const;
380 std::shared_ptr<Priv> m_priv;
383 inline bool operator != (const GBackend &lhs, const GBackend &rhs)
385 return !(lhs == rhs);
392 template<> struct hash<cv::gapi::GBackend>
394 std::size_t operator() (const cv::gapi::GBackend &b) const
407 virtual cv::GKernelImpl impl() const = 0;
408 virtual cv::gapi::GBackend backend() const = 0;
409 const char* id() const { return m_id; }
411 virtual ~GFunctor() = default;
413 GFunctor(const char* id) : m_id(id) { };
418 /** \addtogroup gapi_compile_args
422 // FIXME: Hide implementation
424 * @brief A container class for heterogeneous kernel
425 * implementation collections and graph transformations.
427 * GKernelPackage is a special container class which stores kernel
428 * _implementations_ and graph _transformations_. Objects of this class
429 * are created and passed to cv::GComputation::compile() to specify
430 * which kernels to use and which transformations to apply in the
431 * compiled graph. GKernelPackage may contain kernels of
432 * different backends, e.g. be heterogeneous.
434 * The most easy way to create a kernel package is to use function
435 * cv::gapi::kernels(). This template functions takes kernel
436 * implementations in form of type list (variadic template) and
437 * generates a kernel package atop of that.
439 * Kernel packages can be also generated programmatically, starting
440 * with an empty package (created with the default constructor)
441 * and then by populating it with kernels via call to
442 * GKernelPackage::include(). Note this method is also a template
443 * one since G-API kernel and transformation implementations are _types_,
446 * Finally, two kernel packages can be combined into a new one
447 * with function cv::gapi::combine().
449 class GAPI_EXPORTS_W_SIMPLE GKernelPackage
453 using M = std::unordered_map<std::string, std::pair<GBackend, GKernelImpl>>;
459 std::vector<GTransform> m_transformations;
463 // Check if package contains ANY implementation of a kernel API
464 // by API textual id.
465 bool includesAPI(const std::string &id) const;
468 // Remove ALL implementations of the given API (identified by ID)
469 void removeAPI(const std::string &id);
472 // Partial include() specialization for kernels
473 template <typename KImpl>
474 typename std::enable_if<(std::is_base_of<cv::detail::KernelTag, KImpl>::value), void>::type
477 auto backend = KImpl::backend();
478 auto kernel_id = KImpl::API::id();
479 auto kernel_impl = GKernelImpl{KImpl::kernel(), &KImpl::API::getOutMeta};
480 removeAPI(kernel_id);
482 m_id_kernels[kernel_id] = std::make_pair(backend, kernel_impl);
486 // Partial include() specialization for transformations
487 template <typename TImpl>
488 typename std::enable_if<(std::is_base_of<cv::detail::TransformTag, TImpl>::value), void>::type
491 m_transformations.emplace_back(TImpl::transformation());
495 void include(const GFunctor& functor)
497 m_id_kernels[functor.id()] = std::make_pair(functor.backend(), functor.impl());
500 * @brief Returns total number of kernels
501 * in the package (across all backends included)
503 * @return a number of kernels in the package
505 std::size_t size() const;
508 * @brief Returns vector of transformations included in the package
510 * @return vector of transformations included in the package
512 const std::vector<GTransform>& get_transformations() const;
515 * @brief Test if a particular kernel _implementation_ KImpl is
516 * included in this kernel package.
520 * @note cannot be applied to transformations
522 * @return true if there is such kernel, false otherwise.
524 template<typename KImpl>
525 bool includes() const
527 static_assert(std::is_base_of<cv::detail::KernelTag, KImpl>::value,
528 "includes() can be applied to kernels only");
530 auto kernel_it = m_id_kernels.find(KImpl::API::id());
531 return kernel_it != m_id_kernels.end() &&
532 kernel_it->second.first == KImpl::backend();
536 * @brief Remove all kernels associated with the given backend
539 * Does nothing if there's no kernels of this backend in the package.
541 * @param backend backend which kernels to remove
543 void remove(const GBackend& backend);
546 * @brief Remove all kernels implementing the given API from
549 * Does nothing if there's no kernels implementing the given interface.
551 template<typename KAPI>
554 removeAPI(KAPI::id());
557 // FIXME: Rename to includes() and distinguish API/impl case by
560 * Check if package contains ANY implementation of a kernel API
563 template<typename KAPI>
564 bool includesAPI() const
566 return includesAPI(KAPI::id());
569 // FIXME: The below comment is wrong, and who needs this function?
571 * @brief Find a kernel (by its API)
573 * Returns implementation corresponding id.
574 * Throws if nothing found.
576 * @return Backend which hosts matching kernel implementation.
579 template<typename KAPI>
580 GBackend lookup() const
582 return lookup(KAPI::id()).first;
586 std::pair<cv::gapi::GBackend, cv::GKernelImpl>
587 lookup(const std::string &id) const;
589 // FIXME: No overwrites allowed?
591 * @brief Put a new kernel implementation or a new transformation
592 * KImpl into the package.
594 template<typename KImpl>
597 includeHelper<KImpl>();
601 * @brief Lists all backends which are included into package
603 * @return vector of backends
605 std::vector<GBackend> backends() const;
607 // TODO: Doxygen bug -- it wants me to place this comment
610 * @brief Create a new package based on `lhs` and `rhs`.
612 * @param lhs "Left-hand-side" package in the process
613 * @param rhs "Right-hand-side" package in the process
614 * @return a new kernel package.
616 friend GAPI_EXPORTS GKernelPackage combine(const GKernelPackage &lhs,
617 const GKernelPackage &rhs);
621 * @brief Create a kernel package object containing kernels
622 * and transformations specified in variadic template argument.
624 * In G-API, kernel implementations and transformations are _types_.
625 * Every backend has its own kernel API (like GAPI_OCV_KERNEL() and
626 * GAPI_FLUID_KERNEL()) but all of that APIs define a new type for
627 * each kernel implementation.
629 * Use this function to pass kernel implementations (defined in
630 * either way) and transformations to the system. Example:
632 * @snippet modules/gapi/samples/api_ref_snippets.cpp kernels_snippet
634 * Note that kernels() itself is a function returning object, not
635 * a type, so having `()` at the end is important -- it must be a
638 template<typename... KK> GKernelPackage kernels()
640 // FIXME: currently there is no check that transformations' signatures are unique
641 // and won't be any intersection in graph compilation stage
642 static_assert(cv::detail::all_unique<typename KK::API...>::value, "Kernels API must be unique");
646 // For those who wonder - below is a trick to call a number of
647 // methods based on parameter pack (zeroes just help hiding these
648 // calls into a sequence which helps to expand this parameter pack).
649 // Just note that `f(),a` always equals to `a` (with f() called!)
650 // and parentheses are used to hide function call in the expanded sequence.
651 // Leading 0 helps to handle case when KK is an empty list (kernels<>()).
652 int unused[] = { 0, (pkg.include<KK>(), 0)... };
653 cv::util::suppress_unused_warning(unused);
657 template<typename... FF>
658 GKernelPackage kernels(FF&... functors)
661 int unused[] = { 0, (pkg.include(functors), 0)... };
662 cv::util::suppress_unused_warning(unused);
668 // FYI - this function is already commented above
669 GAPI_EXPORTS GKernelPackage combine(const GKernelPackage &lhs,
670 const GKernelPackage &rhs);
673 * @brief Combines multiple G-API kernel packages into one
677 * This function successively combines the passed kernel packages using a right fold.
678 * Calling `combine(a, b, c)` is equal to `combine(a, combine(b, c))`.
680 * @return The resulting kernel package
682 template<typename... Ps>
683 GKernelPackage combine(const GKernelPackage &a, const GKernelPackage &b, Ps&&... rest)
685 return combine(a, combine(b, rest...));
688 /** \addtogroup gapi_compile_args
692 * @brief cv::use_only() is a special combinator which hints G-API to use only
693 * kernels specified in cv::GComputation::compile() (and not to extend kernels available by
694 * default with that package).
696 struct GAPI_EXPORTS use_only
706 template<> struct CompileArgTag<cv::gapi::GKernelPackage>
708 static const char* tag() { return "gapi.kernel_package"; }
711 template<> struct CompileArgTag<cv::gapi::use_only>
713 static const char* tag() { return "gapi.use_only"; }
715 } // namespace detail
719 #endif // OPENCV_GAPI_GKERNEL_HPP