modules/superres/src/btv_l1_ocl.cpp

   1 /*M///////////////////////////////////////////////////////////////////////////////////////
   2 //
   3 //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
   4 //
   5 //  By downloading, copying, installing or using the software you agree to this license.
   6 //  If you do not agree to this license, do not download, install,
   7 //  copy or use the software.
   8 //
   9 //
  10 //                           License Agreement
  11 //                For Open Source Computer Vision Library
  12 //
  13 // Copyright (C) 2010-2012, Multicoreware, Inc., all rights reserved.
  14 // Copyright (C) 2010-2012, Advanced Micro Devices, Inc., all rights reserved.
  15 // Third party copyrights are property of their respective owners.
  16 //
  17 // @Authors
  18 //              Jin Ma, jin@multicorewareinc.com
  19 // Redistribution and use in source and binary forms, with or without modification,
  20 // are permitted provided that the following conditions are met:
  21 //
  22 //   * Redistribution's of source code must retain the above copyright notice,
  23 //     this list of conditions and the following disclaimer.
  24 //
  25 //   * Redistribution's in binary form must reproduce the above copyright notice,
  26 //     this list of conditions and the following disclaimer in the documentation
  27 //     and/or other materials provided with the distribution.
  28 //
  29 //   * The name of the copyright holders may not be used to endorse or promote products
  30 //     derived from this software without specific prior written permission.
  31 //
  32 // This software is provided by the copyright holders and contributors "as is" and
  33 // any express or implied warranties, including, but not limited to, the implied
  34 // warranties of merchantability and fitness for a particular purpose are disclaimed.
  35 // In no event shall the Intel Corporation or contributors be liable for any direct,
  36 // indirect, incidental, special, exemplary, or consequential damages
  37 // (including, but not limited to, procurement of substitute goods or services;
  38 // loss of use, data, or profits; or business interruption) however caused
  39 // and on any theory of liability, whether in contract, strict liability,
  40 // or tort (including negligence or otherwise) arising in any way out of
  41 // the use of this software, even if advised of the possibility of such damage.
  42 //
  43 //M*/
  44
  45 // S. Farsiu , D. Robinson, M. Elad, P. Milanfar. Fast and robust multiframe super resolution.
  46 // Dennis Mitzel, Thomas Pock, Thomas Schoenemann, Daniel Cremers. Video Super Resolution using Duality Based TV-L1 Optical Flow.
  47
  48 #include "precomp.hpp"
  49
  50 #if !defined(HAVE_OPENCL) || !defined(HAVE_OPENCV_OCL)
  51
  52 cv::Ptr<cv::superres::SuperResolution> cv::superres::createSuperResolution_BTVL1_OCL()
  53 {
  54     CV_Error(CV_StsNotImplemented, "The called functionality is disabled for current build or platform");
  55     return Ptr<SuperResolution>();
  56 }
  57
  58 #else
  59
  60 using namespace std;
  61 using namespace cv;
  62 using namespace cv::ocl;
  63 using namespace cv::superres;
  64 using namespace cv::superres::detail;
  65
  66 namespace cv
  67 {
  68     namespace ocl
  69     {
  70         extern const char* superres_btvl1;
  71
  72         float* btvWeights_ = NULL;
  73         size_t btvWeights_size = 0;
  74     }
  75 }
  76
  77 namespace btv_l1_device_ocl
  78 {
  79     void buildMotionMaps(const oclMat& forwardMotionX, const oclMat& forwardMotionY,
  80         const oclMat& backwardMotionX, const oclMat& bacwardMotionY,
  81         oclMat& forwardMapX, oclMat& forwardMapY,
  82         oclMat& backwardMapX, oclMat& backwardMapY);
  83
  84     void upscale(const oclMat& src, oclMat& dst, int scale);
  85
  86     float diffSign(float a, float b);
  87
  88     Point3f diffSign(Point3f a, Point3f b);
  89
  90     void diffSign(const oclMat& src1, const oclMat& src2, oclMat& dst);
  91
  92     void calcBtvRegularization(const oclMat& src, oclMat& dst, int ksize);
  93 }
  94
  95 void btv_l1_device_ocl::buildMotionMaps(const oclMat& forwardMotionX, const oclMat& forwardMotionY,
  96     const oclMat& backwardMotionX, const oclMat& backwardMotionY,
  97     oclMat& forwardMapX, oclMat& forwardMapY,
  98     oclMat& backwardMapX, oclMat& backwardMapY)
  99 {
 100     Context* clCxt = Context::getContext();
 101
 102     size_t local_thread[] = {32, 8, 1};
 103     size_t global_thread[] = {forwardMapX.cols, forwardMapX.rows, 1};
 104
 105     int forwardMotionX_step = (int)(forwardMotionX.step/forwardMotionX.elemSize());
 106     int forwardMotionY_step = (int)(forwardMotionY.step/forwardMotionY.elemSize());
 107     int backwardMotionX_step = (int)(backwardMotionX.step/backwardMotionX.elemSize());
 108     int backwardMotionY_step = (int)(backwardMotionY.step/backwardMotionY.elemSize());
 109     int forwardMapX_step = (int)(forwardMapX.step/forwardMapX.elemSize());
 110     int forwardMapY_step = (int)(forwardMapY.step/forwardMapY.elemSize());
 111     int backwardMapX_step = (int)(backwardMapX.step/backwardMapX.elemSize());
 112     int backwardMapY_step = (int)(backwardMapY.step/backwardMapY.elemSize());
 113
 114     String kernel_name = "buildMotionMapsKernel";
 115     vector< pair<size_t, const void*> > args;
 116
 117     args.push_back(make_pair(sizeof(cl_mem), (void*)&forwardMotionX.data));
 118     args.push_back(make_pair(sizeof(cl_mem), (void*)&forwardMotionY.data));
 119     args.push_back(make_pair(sizeof(cl_mem), (void*)&backwardMotionX.data));
 120     args.push_back(make_pair(sizeof(cl_mem), (void*)&backwardMotionY.data));
 121     args.push_back(make_pair(sizeof(cl_mem), (void*)&forwardMapX.data));
 122     args.push_back(make_pair(sizeof(cl_mem), (void*)&forwardMapY.data));
 123     args.push_back(make_pair(sizeof(cl_mem), (void*)&backwardMapX.data));
 124     args.push_back(make_pair(sizeof(cl_mem), (void*)&backwardMapY.data));
 125
 126     args.push_back(make_pair(sizeof(cl_int), (void*)&forwardMotionX.rows));
 127     args.push_back(make_pair(sizeof(cl_int), (void*)&forwardMotionY.cols));
 128
 129     args.push_back(make_pair(sizeof(cl_int), (void*)&forwardMotionX_step));
 130     args.push_back(make_pair(sizeof(cl_int), (void*)&forwardMotionY_step));
 131     args.push_back(make_pair(sizeof(cl_int), (void*)&backwardMotionX_step));
 132     args.push_back(make_pair(sizeof(cl_int), (void*)&backwardMotionY_step));
 133     args.push_back(make_pair(sizeof(cl_int), (void*)&forwardMapX_step));
 134     args.push_back(make_pair(sizeof(cl_int), (void*)&forwardMapY_step));
 135     args.push_back(make_pair(sizeof(cl_int), (void*)&backwardMapX_step));
 136     args.push_back(make_pair(sizeof(cl_int), (void*)&backwardMapY_step));
 137
 138     openCLExecuteKernel(clCxt, &superres_btvl1, kernel_name, global_thread, local_thread, args, -1, -1);
 139 }
 140
 141 void btv_l1_device_ocl::upscale(const oclMat& src, oclMat& dst, int scale)
 142 {
 143     Context* clCxt = Context::getContext();
 144
 145     size_t local_thread[] = {32, 8, 1};
 146     size_t global_thread[] = {src.cols, src.rows, 1};
 147
 148     int src_step = (int)(src.step/src.elemSize());
 149     int dst_step = (int)(dst.step/dst.elemSize());
 150
 151     String kernel_name = "upscaleKernel";
 152     vector< pair<size_t, const void*> > args;
 153
 154     int cn = src.oclchannels();
 155
 156     args.push_back(make_pair(sizeof(cl_mem), (void*)&src.data));
 157     args.push_back(make_pair(sizeof(cl_mem), (void*)&dst.data));
 158     args.push_back(make_pair(sizeof(cl_int), (void*)&src_step));
 159     args.push_back(make_pair(sizeof(cl_int), (void*)&dst_step));
 160     args.push_back(make_pair(sizeof(cl_int), (void*)&src.rows));
 161     args.push_back(make_pair(sizeof(cl_int), (void*)&src.cols));
 162     args.push_back(make_pair(sizeof(cl_int), (void*)&scale));
 163     args.push_back(make_pair(sizeof(cl_int), (void*)&cn));
 164
 165     openCLExecuteKernel(clCxt, &superres_btvl1, kernel_name, global_thread, local_thread, args, -1, -1);
 166
 167 }
 168
 169 float btv_l1_device_ocl::diffSign(float a, float b)
 170 {
 171     return a > b ? 1.0f : a < b ? -1.0f : 0.0f;
 172 }
 173
 174 Point3f btv_l1_device_ocl::diffSign(Point3f a, Point3f b)
 175 {
 176     return Point3f(
 177         a.x > b.x ? 1.0f : a.x < b.x ? -1.0f : 0.0f,
 178         a.y > b.y ? 1.0f : a.y < b.y ? -1.0f : 0.0f,
 179         a.z > b.z ? 1.0f : a.z < b.z ? -1.0f : 0.0f
 180         );
 181 }
 182
 183 void btv_l1_device_ocl::diffSign(const oclMat& src1, const oclMat& src2, oclMat& dst)
 184 {
 185     Context* clCxt = Context::getContext();
 186
 187     oclMat src1_ = src1.reshape(1);
 188     oclMat src2_ = src2.reshape(1);
 189     oclMat dst_ = dst.reshape(1);
 190
 191     int src1_step = (int)(src1_.step/src1_.elemSize());
 192     int src2_step = (int)(src2_.step/src2_.elemSize());
 193     int dst_step = (int)(dst_.step/dst_.elemSize());
 194
 195     size_t local_thread[] = {32, 8, 1};
 196     size_t global_thread[] = {src1_.cols, src1_.rows, 1};
 197
 198     String kernel_name = "diffSignKernel";
 199     vector< pair<size_t, const void*> > args;
 200
 201     args.push_back(make_pair(sizeof(cl_mem), (void*)&src1_.data));
 202     args.push_back(make_pair(sizeof(cl_mem), (void*)&src2_.data));
 203     args.push_back(make_pair(sizeof(cl_mem), (void*)&dst_.data));
 204
 205     args.push_back(make_pair(sizeof(cl_int), (void*)&src1_.rows));
 206     args.push_back(make_pair(sizeof(cl_int), (void*)&src1_.cols));
 207     args.push_back(make_pair(sizeof(cl_int), (void*)&dst_step));
 208     args.push_back(make_pair(sizeof(cl_int), (void*)&src1_step));
 209     args.push_back(make_pair(sizeof(cl_int), (void*)&src2_step));
 210
 211     openCLExecuteKernel(clCxt, &superres_btvl1, kernel_name, global_thread, local_thread, args, -1, -1);
 212 }
 213
 214 void btv_l1_device_ocl::calcBtvRegularization(const oclMat& src, oclMat& dst, int ksize)
 215 {
 216     Context* clCxt = Context::getContext();
 217
 218     oclMat src_ = src.reshape(1);
 219     oclMat dst_ = dst.reshape(1);
 220
 221     size_t local_thread[] = {32, 8, 1};
 222     size_t global_thread[] = {src.cols, src.rows, 1};
 223
 224     int src_step = (int)(src_.step/src_.elemSize());
 225     int dst_step = (int)(dst_.step/dst_.elemSize());
 226
 227     String kernel_name = "calcBtvRegularizationKernel";
 228     vector< pair<size_t, const void*> > args;
 229
 230     int cn = src.oclchannels();
 231
 232     cl_mem c_btvRegWeights;
 233     size_t count = btvWeights_size * sizeof(float);
 234     c_btvRegWeights = openCLCreateBuffer(clCxt, CL_MEM_READ_ONLY, count);
 235     int cl_safe_check = clEnqueueWriteBuffer((cl_command_queue)clCxt->oclCommandQueue(), c_btvRegWeights, 1, 0, count, btvWeights_, 0, NULL, NULL);
 236     CV_Assert(cl_safe_check == CL_SUCCESS);
 237
 238     args.push_back(make_pair(sizeof(cl_mem), (void*)&src_.data));
 239     args.push_back(make_pair(sizeof(cl_mem), (void*)&dst_.data));
 240     args.push_back(make_pair(sizeof(cl_int), (void*)&src_step));
 241     args.push_back(make_pair(sizeof(cl_int), (void*)&dst_step));
 242     args.push_back(make_pair(sizeof(cl_int), (void*)&src.rows));
 243     args.push_back(make_pair(sizeof(cl_int), (void*)&src.cols));
 244     args.push_back(make_pair(sizeof(cl_int), (void*)&ksize));
 245     args.push_back(make_pair(sizeof(cl_int), (void*)&cn));
 246     args.push_back(make_pair(sizeof(cl_mem), (void*)&c_btvRegWeights));
 247
 248     openCLExecuteKernel(clCxt, &superres_btvl1, kernel_name, global_thread, local_thread, args, -1, -1);
 249     cl_safe_check = clReleaseMemObject(c_btvRegWeights);
 250     CV_Assert(cl_safe_check == CL_SUCCESS);
 251 }
 252
 253 namespace
 254 {
 255     void calcRelativeMotions(const vector<pair<oclMat, oclMat> >& forwardMotions, const vector<pair<oclMat, oclMat> >& backwardMotions,
 256         vector<pair<oclMat, oclMat> >& relForwardMotions, vector<pair<oclMat, oclMat> >& relBackwardMotions,
 257         int baseIdx, Size size)
 258     {
 259         const int count = static_cast<int>(forwardMotions.size());
 260
 261         relForwardMotions.resize(count);
 262         relForwardMotions[baseIdx].first.create(size, CV_32FC1);
 263         relForwardMotions[baseIdx].first.setTo(Scalar::all(0));
 264         relForwardMotions[baseIdx].second.create(size, CV_32FC1);
 265         relForwardMotions[baseIdx].second.setTo(Scalar::all(0));
 266
 267         relBackwardMotions.resize(count);
 268         relBackwardMotions[baseIdx].first.create(size, CV_32FC1);
 269         relBackwardMotions[baseIdx].first.setTo(Scalar::all(0));
 270         relBackwardMotions[baseIdx].second.create(size, CV_32FC1);
 271         relBackwardMotions[baseIdx].second.setTo(Scalar::all(0));
 272
 273         for (int i = baseIdx - 1; i >= 0; --i)
 274         {
 275             ocl::add(relForwardMotions[i + 1].first, forwardMotions[i].first, relForwardMotions[i].first);
 276             ocl::add(relForwardMotions[i + 1].second, forwardMotions[i].second, relForwardMotions[i].second);
 277
 278             ocl::add(relBackwardMotions[i + 1].first, backwardMotions[i + 1].first, relBackwardMotions[i].first);
 279             ocl::add(relBackwardMotions[i + 1].second, backwardMotions[i + 1].second, relBackwardMotions[i].second);
 280         }
 281
 282         for (int i = baseIdx + 1; i < count; ++i)
 283         {
 284             ocl::add(relForwardMotions[i - 1].first, backwardMotions[i].first, relForwardMotions[i].first);
 285             ocl::add(relForwardMotions[i - 1].second, backwardMotions[i].second, relForwardMotions[i].second);
 286
 287             ocl::add(relBackwardMotions[i - 1].first, forwardMotions[i - 1].first, relBackwardMotions[i].first);
 288             ocl::add(relBackwardMotions[i - 1].second, forwardMotions[i - 1].second, relBackwardMotions[i].second);
 289         }
 290     }
 291
 292     void upscaleMotions(const vector<pair<oclMat, oclMat> >& lowResMotions, vector<pair<oclMat, oclMat> >& highResMotions, int scale)
 293     {
 294         highResMotions.resize(lowResMotions.size());
 295
 296         for (size_t i = 0; i < lowResMotions.size(); ++i)
 297         {
 298             ocl::resize(lowResMotions[i].first, highResMotions[i].first, Size(), scale, scale, INTER_LINEAR);
 299             ocl::resize(lowResMotions[i].second, highResMotions[i].second, Size(), scale, scale, INTER_LINEAR);
 300
 301             ocl::multiply(scale, highResMotions[i].first, highResMotions[i].first);
 302             ocl::multiply(scale, highResMotions[i].second, highResMotions[i].second);
 303         }
 304     }
 305
 306     void buildMotionMaps(const pair<oclMat, oclMat>& forwardMotion, const pair<oclMat, oclMat>& backwardMotion,
 307         pair<oclMat, oclMat>& forwardMap, pair<oclMat, oclMat>& backwardMap)
 308     {
 309         forwardMap.first.create(forwardMotion.first.size(), CV_32FC1);
 310         forwardMap.second.create(forwardMotion.first.size(), CV_32FC1);
 311
 312         backwardMap.first.create(forwardMotion.first.size(), CV_32FC1);
 313         backwardMap.second.create(forwardMotion.first.size(), CV_32FC1);
 314
 315         btv_l1_device_ocl::buildMotionMaps(forwardMotion.first, forwardMotion.second,
 316             backwardMotion.first, backwardMotion.second,
 317             forwardMap.first, forwardMap.second,
 318             backwardMap.first, backwardMap.second);
 319     }
 320
 321     void upscale(const oclMat& src, oclMat& dst, int scale)
 322     {
 323         CV_Assert( src.channels() == 1 || src.channels() == 3 || src.channels() == 4 );
 324
 325         dst.create(src.rows * scale, src.cols * scale, src.type());
 326         dst.setTo(Scalar::all(0));
 327
 328         btv_l1_device_ocl::upscale(src, dst, scale);
 329     }
 330
 331     void diffSign(const oclMat& src1, const oclMat& src2, oclMat& dst)
 332     {
 333         dst.create(src1.size(), src1.type());
 334
 335         btv_l1_device_ocl::diffSign(src1, src2, dst);
 336     }
 337
 338     void calcBtvWeights(int btvKernelSize, double alpha, vector<float>& btvWeights)
 339     {
 340         const size_t size = btvKernelSize * btvKernelSize;
 341
 342         btvWeights.resize(size);
 343
 344         const int ksize = (btvKernelSize - 1) / 2;
 345         const float alpha_f = static_cast<float>(alpha);
 346
 347         for (int m = 0, ind = 0; m <= ksize; ++m)
 348         {
 349             for (int l = ksize; l + m >= 0; --l, ++ind)
 350                 btvWeights[ind] = pow(alpha_f, std::abs(m) + std::abs(l));
 351         }
 352
 353         btvWeights_ = &btvWeights[0];
 354         btvWeights_size = size;
 355     }
 356
 357     void calcBtvRegularization(const oclMat& src, oclMat& dst, int btvKernelSize)
 358     {
 359         dst.create(src.size(), src.type());
 360         dst.setTo(Scalar::all(0));
 361
 362         const int ksize = (btvKernelSize - 1) / 2;
 363
 364         btv_l1_device_ocl::calcBtvRegularization(src, dst, ksize);
 365     }
 366
 367     class BTVL1_OCL_Base
 368     {
 369     public:
 370         BTVL1_OCL_Base();
 371
 372         void process(const vector<oclMat>& src, oclMat& dst,
 373             const vector<pair<oclMat, oclMat> >& forwardMotions, const vector<pair<oclMat, oclMat> >& backwardMotions,
 374             int baseIdx);
 375
 376         void collectGarbage();
 377
 378     protected:
 379         int scale_;
 380         int iterations_;
 381         double lambda_;
 382         double tau_;
 383         double alpha_;
 384         int btvKernelSize_;
 385         int blurKernelSize_;
 386         double blurSigma_;
 387         Ptr<DenseOpticalFlowExt> opticalFlow_;
 388
 389     private:
 390         vector<Ptr<cv::ocl::FilterEngine_GPU> > filters_;
 391         int curBlurKernelSize_;
 392         double curBlurSigma_;
 393         int curSrcType_;
 394
 395         vector<float> btvWeights_;
 396         int curBtvKernelSize_;
 397         double curAlpha_;
 398
 399         vector<pair<oclMat, oclMat> > lowResForwardMotions_;
 400         vector<pair<oclMat, oclMat> > lowResBackwardMotions_;
 401
 402         vector<pair<oclMat, oclMat> > highResForwardMotions_;
 403         vector<pair<oclMat, oclMat> > highResBackwardMotions_;
 404
 405         vector<pair<oclMat, oclMat> > forwardMaps_;
 406         vector<pair<oclMat, oclMat> > backwardMaps_;
 407
 408         oclMat highRes_;
 409
 410         vector<oclMat> diffTerms_;
 411         vector<oclMat> a_, b_, c_;
 412         oclMat regTerm_;
 413     };
 414
 415     BTVL1_OCL_Base::BTVL1_OCL_Base()
 416     {
 417         scale_ = 4;
 418         iterations_ = 180;
 419         lambda_ = 0.03;
 420         tau_ = 1.3;
 421         alpha_ = 0.7;
 422         btvKernelSize_ = 7;
 423         blurKernelSize_ = 5;
 424         blurSigma_ = 0.0;
 425         opticalFlow_ = createOptFlow_DualTVL1_OCL();
 426
 427         curBlurKernelSize_ = -1;
 428         curBlurSigma_ = -1.0;
 429         curSrcType_ = -1;
 430
 431         curBtvKernelSize_ = -1;
 432         curAlpha_ = -1.0;
 433     }
 434
 435     void BTVL1_OCL_Base::process(const vector<oclMat>& src, oclMat& dst,
 436         const vector<pair<oclMat, oclMat> >& forwardMotions, const vector<pair<oclMat, oclMat> >& backwardMotions,
 437         int baseIdx)
 438     {
 439         CV_Assert( scale_ > 1 );
 440         CV_Assert( iterations_ > 0 );
 441         CV_Assert( tau_ > 0.0 );
 442         CV_Assert( alpha_ > 0.0 );
 443         CV_Assert( btvKernelSize_ > 0 && btvKernelSize_ <= 16 );
 444         CV_Assert( blurKernelSize_ > 0 );
 445         CV_Assert( blurSigma_ >= 0.0 );
 446
 447         // update blur filter and btv weights
 448
 449         if (filters_.size() != src.size() || blurKernelSize_ != curBlurKernelSize_ || blurSigma_ != curBlurSigma_ || src[0].type() != curSrcType_)
 450         {
 451             filters_.resize(src.size());
 452             for (size_t i = 0; i < src.size(); ++i)
 453                 filters_[i] = cv::ocl::createGaussianFilter_GPU(src[0].type(), Size(blurKernelSize_, blurKernelSize_), blurSigma_);
 454             curBlurKernelSize_ = blurKernelSize_;
 455             curBlurSigma_ = blurSigma_;
 456             curSrcType_ = src[0].type();
 457         }
 458
 459         if (btvWeights_.empty() || btvKernelSize_ != curBtvKernelSize_ || alpha_ != curAlpha_)
 460         {
 461             calcBtvWeights(btvKernelSize_, alpha_, btvWeights_);
 462             curBtvKernelSize_ = btvKernelSize_;
 463             curAlpha_ = alpha_;
 464         }
 465
 466         // calc motions between input frames
 467
 468         calcRelativeMotions(forwardMotions, backwardMotions,
 469             lowResForwardMotions_, lowResBackwardMotions_,
 470             baseIdx, src[0].size());
 471
 472         upscaleMotions(lowResForwardMotions_, highResForwardMotions_, scale_);
 473         upscaleMotions(lowResBackwardMotions_, highResBackwardMotions_, scale_);
 474
 475         forwardMaps_.resize(highResForwardMotions_.size());
 476         backwardMaps_.resize(highResForwardMotions_.size());
 477         for (size_t i = 0; i < highResForwardMotions_.size(); ++i)
 478         {
 479             buildMotionMaps(highResForwardMotions_[i], highResBackwardMotions_[i], forwardMaps_[i], backwardMaps_[i]);
 480         }
 481         // initial estimation
 482
 483         const Size lowResSize = src[0].size();
 484         const Size highResSize(lowResSize.width * scale_, lowResSize.height * scale_);
 485
 486         ocl::resize(src[baseIdx], highRes_, highResSize, 0, 0, INTER_LINEAR);
 487
 488         // iterations
 489
 490         diffTerms_.resize(src.size());
 491         a_.resize(src.size());
 492         b_.resize(src.size());
 493         c_.resize(src.size());
 494
 495         for (int i = 0; i < iterations_; ++i)
 496         {
 497             for (size_t k = 0; k < src.size(); ++k)
 498             {
 499                 diffTerms_[k].create(highRes_.size(), highRes_.type());
 500                 a_[k].create(highRes_.size(), highRes_.type());
 501                 b_[k].create(highRes_.size(), highRes_.type());
 502                 c_[k].create(lowResSize, highRes_.type());
 503
 504                 // a = M * Ih
 505                 ocl::remap(highRes_, a_[k], backwardMaps_[k].first, backwardMaps_[k].second, INTER_NEAREST, BORDER_CONSTANT, Scalar());
 506                 // b = HM * Ih
 507                 filters_[k]->apply(a_[k], b_[k], Rect(0,0,-1,-1));
 508                 // c = DHF * Ih
 509                 ocl::resize(b_[k], c_[k], lowResSize, 0, 0, INTER_NEAREST);
 510
 511                 diffSign(src[k], c_[k], c_[k]);
 512
 513                 // a = Dt * diff
 514                 upscale(c_[k], a_[k], scale_);
 515                 // b = HtDt * diff
 516                 filters_[k]->apply(a_[k], b_[k], Rect(0,0,-1,-1));
 517                 // diffTerm = MtHtDt * diff
 518                 ocl::remap(b_[k], diffTerms_[k], forwardMaps_[k].first, forwardMaps_[k].second, INTER_NEAREST, BORDER_CONSTANT, Scalar());
 519             }
 520
 521             if (lambda_ > 0)
 522             {
 523                 calcBtvRegularization(highRes_, regTerm_, btvKernelSize_);
 524                 ocl::addWeighted(highRes_, 1.0, regTerm_, -tau_ * lambda_, 0.0, highRes_);
 525             }
 526
 527             for (size_t k = 0; k < src.size(); ++k)
 528             {
 529                 ocl::addWeighted(highRes_, 1.0, diffTerms_[k], tau_, 0.0, highRes_);
 530             }
 531         }
 532
 533         Rect inner(btvKernelSize_, btvKernelSize_, highRes_.cols - 2 * btvKernelSize_, highRes_.rows - 2 * btvKernelSize_);
 534         highRes_(inner).copyTo(dst);
 535     }
 536
 537     void BTVL1_OCL_Base::collectGarbage()
 538     {
 539         filters_.clear();
 540
 541         lowResForwardMotions_.clear();
 542         lowResBackwardMotions_.clear();
 543
 544         highResForwardMotions_.clear();
 545         highResBackwardMotions_.clear();
 546
 547         forwardMaps_.clear();
 548         backwardMaps_.clear();
 549
 550         highRes_.release();
 551
 552         diffTerms_.clear();
 553         a_.clear();
 554         b_.clear();
 555         c_.clear();
 556         regTerm_.release();
 557     }
 558
 559     ////////////////////////////////////////////////////////////
 560
 561     class BTVL1_OCL : public SuperResolution, private BTVL1_OCL_Base
 562     {
 563     public:
 564         AlgorithmInfo* info() const;
 565
 566         BTVL1_OCL();
 567
 568         void collectGarbage();
 569
 570     protected:
 571         void initImpl(Ptr<FrameSource>& frameSource);
 572         void processImpl(Ptr<FrameSource>& frameSource, OutputArray output);
 573
 574     private:
 575         int temporalAreaRadius_;
 576
 577         void readNextFrame(Ptr<FrameSource>& frameSource);
 578         void processFrame(int idx);
 579
 580         oclMat curFrame_;
 581         oclMat prevFrame_;
 582
 583         vector<oclMat> frames_;
 584         vector<pair<oclMat, oclMat> > forwardMotions_;
 585         vector<pair<oclMat, oclMat> > backwardMotions_;
 586         vector<oclMat> outputs_;
 587
 588         int storePos_;
 589         int procPos_;
 590         int outPos_;
 591
 592         vector<oclMat> srcFrames_;
 593         vector<pair<oclMat, oclMat> > srcForwardMotions_;
 594         vector<pair<oclMat, oclMat> > srcBackwardMotions_;
 595         oclMat finalOutput_;
 596     };
 597
 598     CV_INIT_ALGORITHM(BTVL1_OCL, "SuperResolution.BTVL1_OCL",
 599     obj.info()->addParam(obj, "scale", obj.scale_, false, 0, 0, "Scale factor.");
 600     obj.info()->addParam(obj, "iterations", obj.iterations_, false, 0, 0, "Iteration count.");
 601     obj.info()->addParam(obj, "tau", obj.tau_, false, 0, 0, "Asymptotic value of steepest descent method.");
 602     obj.info()->addParam(obj, "lambda", obj.lambda_, false, 0, 0, "Weight parameter to balance data term and smoothness term.");
 603     obj.info()->addParam(obj, "alpha", obj.alpha_, false, 0, 0, "Parameter of spacial distribution in Bilateral-TV.");
 604     obj.info()->addParam(obj, "btvKernelSize", obj.btvKernelSize_, false, 0, 0, "Kernel size of Bilateral-TV filter.");
 605     obj.info()->addParam(obj, "blurKernelSize", obj.blurKernelSize_, false, 0, 0, "Gaussian blur kernel size.");
 606     obj.info()->addParam(obj, "blurSigma", obj.blurSigma_, false, 0, 0, "Gaussian blur sigma.");
 607     obj.info()->addParam(obj, "temporalAreaRadius", obj.temporalAreaRadius_, false, 0, 0, "Radius of the temporal search area.");
 608     obj.info()->addParam<DenseOpticalFlowExt>(obj, "opticalFlow", obj.opticalFlow_, false, 0, 0, "Dense optical flow algorithm."));
 609
 610     BTVL1_OCL::BTVL1_OCL()
 611     {
 612         temporalAreaRadius_ = 4;
 613     }
 614
 615     void BTVL1_OCL::collectGarbage()
 616     {
 617         curFrame_.release();
 618         prevFrame_.release();
 619
 620         frames_.clear();
 621         forwardMotions_.clear();
 622         backwardMotions_.clear();
 623         outputs_.clear();
 624
 625         srcFrames_.clear();
 626         srcForwardMotions_.clear();
 627         srcBackwardMotions_.clear();
 628         finalOutput_.release();
 629
 630         SuperResolution::collectGarbage();
 631         BTVL1_OCL_Base::collectGarbage();
 632     }
 633
 634     void BTVL1_OCL::initImpl(Ptr<FrameSource>& frameSource)
 635     {
 636         const int cacheSize = 2 * temporalAreaRadius_ + 1;
 637
 638         frames_.resize(cacheSize);
 639         forwardMotions_.resize(cacheSize);
 640         backwardMotions_.resize(cacheSize);
 641         outputs_.resize(cacheSize);
 642
 643         storePos_ = -1;
 644
 645         for (int t = -temporalAreaRadius_; t <= temporalAreaRadius_; ++t)
 646             readNextFrame(frameSource);
 647
 648         for (int i = 0; i <= temporalAreaRadius_; ++i)
 649             processFrame(i);
 650
 651         procPos_ = temporalAreaRadius_;
 652         outPos_ = -1;
 653     }
 654
 655     void BTVL1_OCL::processImpl(Ptr<FrameSource>& frameSource, OutputArray _output)
 656     {
 657         if (outPos_ >= storePos_)
 658         {
 659             if(_output.kind() == _InputArray::OCL_MAT)
 660             {
 661                 getOclMatRef(_output).release();
 662             }
 663             else
 664             {
 665                 _output.release();
 666             }
 667             return;
 668         }
 669
 670         readNextFrame(frameSource);
 671
 672         if (procPos_ < storePos_)
 673         {
 674             ++procPos_;
 675             processFrame(procPos_);
 676         }
 677
 678         ++outPos_;
 679         const oclMat& curOutput = at(outPos_, outputs_);
 680
 681         if (_output.kind() == _InputArray::OCL_MAT)
 682             curOutput.convertTo(getOclMatRef(_output), CV_8U);
 683         else
 684         {
 685             curOutput.convertTo(finalOutput_, CV_8U);
 686             arrCopy(finalOutput_, _output);
 687         }
 688     }
 689
 690     void BTVL1_OCL::readNextFrame(Ptr<FrameSource>& frameSource)
 691     {
 692         curFrame_.release();
 693         frameSource->nextFrame(curFrame_);
 694
 695         if (curFrame_.empty())
 696             return;
 697
 698         ++storePos_;
 699         curFrame_.convertTo(at(storePos_, frames_), CV_32F);
 700
 701         if (storePos_ > 0)
 702         {
 703             pair<oclMat, oclMat>& forwardMotion = at(storePos_ - 1, forwardMotions_);
 704             pair<oclMat, oclMat>& backwardMotion = at(storePos_, backwardMotions_);
 705
 706             opticalFlow_->calc(prevFrame_, curFrame_, forwardMotion.first, forwardMotion.second);
 707             opticalFlow_->calc(curFrame_, prevFrame_, backwardMotion.first, backwardMotion.second);
 708         }
 709
 710         curFrame_.copyTo(prevFrame_);
 711     }
 712
 713     void BTVL1_OCL::processFrame(int idx)
 714     {
 715         const int startIdx = max(idx - temporalAreaRadius_, 0);
 716         const int procIdx = idx;
 717         const int endIdx = min(startIdx + 2 * temporalAreaRadius_, storePos_);
 718
 719         const int count = endIdx - startIdx + 1;
 720
 721         srcFrames_.resize(count);
 722         srcForwardMotions_.resize(count);
 723         srcBackwardMotions_.resize(count);
 724
 725         int baseIdx = -1;
 726
 727         for (int i = startIdx, k = 0; i <= endIdx; ++i, ++k)
 728         {
 729             if (i == procIdx)
 730                 baseIdx = k;
 731
 732             srcFrames_[k] = at(i, frames_);
 733
 734             if (i < endIdx)
 735                 srcForwardMotions_[k] = at(i, forwardMotions_);
 736             if (i > startIdx)
 737                 srcBackwardMotions_[k] = at(i, backwardMotions_);
 738         }
 739
 740         process(srcFrames_, at(idx, outputs_), srcForwardMotions_, srcBackwardMotions_, baseIdx);
 741     }
 742 }
 743
 744 Ptr<SuperResolution> cv::superres::createSuperResolution_BTVL1_OCL()
 745 {
 746     return new BTVL1_OCL;
 747 }
 748 #endif