From ef5d7278ea7b92cce5e0b7ad5cb2862e5f90b909 Mon Sep 17 00:00:00 2001 From: Maria Dimashova Date: Tue, 31 May 2011 15:30:49 +0000 Subject: [PATCH] removed evaluation testbetch to sample --- .../cpp/detector_descriptor_evaluation.cpp | 529 +++++++-------------- 1 file changed, 160 insertions(+), 369 deletions(-) rename modules/features2d/test/test_detectordescriptor_evaluation.cpp => samples/cpp/detector_descriptor_evaluation.cpp (60%) diff --git a/modules/features2d/test/test_detectordescriptor_evaluation.cpp b/samples/cpp/detector_descriptor_evaluation.cpp similarity index 60% rename from modules/features2d/test/test_detectordescriptor_evaluation.cpp rename to samples/cpp/detector_descriptor_evaluation.cpp index 4980bb9..b76bf3e 100644 --- a/modules/features2d/test/test_detectordescriptor_evaluation.cpp +++ b/samples/cpp/detector_descriptor_evaluation.cpp @@ -39,7 +39,9 @@ // //M*/ -#include "test_precomp.hpp" +#include "opencv2/imgproc/imgproc.hpp" +#include "opencv2/highgui/highgui.hpp" +#include "opencv2/features2d/features2d.hpp" #include #include @@ -49,6 +51,8 @@ using namespace std; using namespace cv; + +string data_path; /****************************************************************************************\ * Functions to evaluate affine covariant detectors and descriptors. * \****************************************************************************************/ @@ -59,8 +63,8 @@ static inline Point2f applyHomography( const Mat_& H, const Point2f& pt if( z ) { double w = 1./z; - return Point2f( (float)((H(0,0)*pt.x + H(0,1)*pt.y + H(0,2))*w), - (float)((H(1,0)*pt.x + H(1,1)*pt.y + H(1,2))*w) ); + return Point2f( (float)((H(0,0)*pt.x + H(0,1)*pt.y + H(0,2))*w), + (float)((H(1,0)*pt.x + H(1,1)*pt.y + H(1,2))*w) ); } return Point2f( numeric_limits::max(), numeric_limits::max() ); } @@ -159,24 +163,23 @@ string IS_ACTIVE_PARAMS = "isActiveParams"; string IS_SAVE_KEYPOINTS = "isSaveKeypoints"; -class BaseQualityTest : public cvtest::BaseTest +class BaseQualityEvaluator { public: - BaseQualityTest( const char* _algName ) : algName(_algName) + BaseQualityEvaluator( const char* _algName, const char* _testName ) : algName(_algName), testName(_testName) { //TODO: change this isWriteGraphicsData = true; } + void run(); + protected: virtual string getRunParamsFilename() const = 0; virtual string getResultsFilename() const = 0; virtual string getPlotPath() const = 0; - virtual void validQualityClear( int datasetIdx ) = 0; virtual void calcQualityClear( int datasetIdx ) = 0; - virtual void validQualityCreate( int datasetIdx ) = 0; - virtual bool isValidQualityEmpty( int datasetIdx ) const = 0; virtual bool isCalcQualityEmpty( int datasetIdx ) const = 0; void readAllDatasetsRunParams(); @@ -188,29 +191,21 @@ protected: virtual void readDefaultRunParams( FileNode& /*fn*/ ) {} virtual void writeDefaultRunParams( FileStorage& /*fs*/ ) const {} - virtual void readResults(); - virtual void readResults( FileNode& fn, int datasetIdx, int caseIdx ) = 0; - void writeResults() const; - virtual void writeResults( FileStorage& fs, int datasetIdx, int caseIdx ) const = 0; - bool readDataset( const string& datasetName, vector& Hs, vector& imgs ); - virtual void readAlgorithm( ) {}; - virtual void processRunParamsFile () {}; + virtual void readAlgorithm() {} + virtual void processRunParamsFile() {} virtual void runDatasetTest( const vector& /*imgs*/, const vector& /*Hs*/, int /*di*/, int& /*progress*/ ) {} - void run( int ); virtual void processResults( int datasetIdx ); - virtual int processResults( int datasetIdx, int caseIdx ) = 0; virtual void processResults(); virtual void writePlotData( int /*datasetIdx*/ ) const {} - virtual void writeAveragePlotData() const {}; - string algName; - bool isWriteParams, isWriteResults, isWriteGraphicsData; + string algName, testName; + bool isWriteParams, isWriteGraphicsData; }; -void BaseQualityTest::readAllDatasetsRunParams() +void BaseQualityEvaluator::readAllDatasetsRunParams() { string filename = getRunParamsFilename(); FileStorage fs( filename, FileStorage::READ ); @@ -218,7 +213,7 @@ void BaseQualityTest::readAllDatasetsRunParams() { isWriteParams = true; setDefaultAllDatasetsRunParams(); - ts->printf(cvtest::TS::LOG, "all runParams are default\n"); + printf("All runParams are default.\n"); } else { @@ -233,7 +228,7 @@ void BaseQualityTest::readAllDatasetsRunParams() FileNode fn = topfn[DATASET_NAMES[i]]; if( fn.empty() ) { - ts->printf( cvtest::TS::LOG, "%d-runParams is default\n", i); + printf( "%d-runParams is default.\n", i); setDefaultDatasetRunParams(i); } else @@ -242,7 +237,7 @@ void BaseQualityTest::readAllDatasetsRunParams() } } -void BaseQualityTest::writeAllDatasetsRunParams() const +void BaseQualityEvaluator::writeAllDatasetsRunParams() const { string filename = getRunParamsFilename(); FileStorage fs( filename, FileStorage::WRITE ); @@ -261,27 +256,30 @@ void BaseQualityTest::writeAllDatasetsRunParams() const fs << "}"; } else - ts->printf(cvtest::TS::LOG, "file %s for writing run params can not be opened\n", filename.c_str() ); + printf( "File %s for writing run params can not be opened.\n", filename.c_str() ); } -void BaseQualityTest::setDefaultAllDatasetsRunParams() +void BaseQualityEvaluator::setDefaultAllDatasetsRunParams() { for( int i = 0; i < DATASETS_COUNT; i++ ) setDefaultDatasetRunParams(i); } -bool BaseQualityTest::readDataset( const string& datasetName, vector& Hs, vector& imgs ) +bool BaseQualityEvaluator::readDataset( const string& datasetName, vector& Hs, vector& imgs ) { Hs.resize( TEST_CASE_COUNT ); imgs.resize( TEST_CASE_COUNT+1 ); - string dirname = string(ts->get_data_path()) + IMAGE_DATASETS_DIR + datasetName + "/"; - + string dirname = data_path + IMAGE_DATASETS_DIR + datasetName + "/"; for( int i = 0; i < (int)Hs.size(); i++ ) { stringstream filename; filename << "H1to" << i+2 << "p.xml"; FileStorage fs( dirname + filename.str(), FileStorage::READ ); if( !fs.isOpened() ) + { + cout << "filename " << dirname + filename.str() << endl; + FileStorage fs( dirname + filename.str(), FileStorage::READ ); return false; + } fs.getFirstTopLevelNode() >> Hs[i]; } @@ -290,123 +288,30 @@ bool BaseQualityTest::readDataset( const string& datasetName, vector& Hs, v stringstream filename; filename << "img" << i+1 << ".png"; imgs[i] = imread( dirname + filename.str(), 0 ); if( imgs[i].empty() ) - return false; - } - return true; -} - -void BaseQualityTest::readResults() -{ - string filename = getResultsFilename(); - FileStorage fs( filename, FileStorage::READ ); - if( fs.isOpened() ) - { - isWriteResults = false; - FileNode topfn = fs.getFirstTopLevelNode(); - for( int di = 0; di < DATASETS_COUNT; di++ ) { - FileNode datafn = topfn[DATASET_NAMES[di]]; - if( datafn.empty() ) - { - validQualityClear(di); - ts->printf( cvtest::TS::LOG, "results for %s dataset were not read\n", - DATASET_NAMES[di].c_str() ); - } - else - { - validQualityCreate(di); - for( int ci = 0; ci < TEST_CASE_COUNT; ci++ ) - { - stringstream ss; ss << "case" << ci; - FileNode casefn = datafn[ss.str()]; - CV_Assert( !casefn.empty() ); - readResults( casefn , di, ci ); - } - } - } - } - else - isWriteResults = true; -} - -void BaseQualityTest::writeResults() const -{ - string filename = getResultsFilename();; - FileStorage fs( filename, FileStorage::WRITE ); - if( fs.isOpened() ) - { - fs << "results" << "{"; - for( int di = 0; di < DATASETS_COUNT; di++ ) - { - if( isCalcQualityEmpty(di) ) - { - ts->printf(cvtest::TS::LOG, "results on %s dataset were not write because of empty\n", - DATASET_NAMES[di].c_str()); - } - else - { - fs << DATASET_NAMES[di] << "{"; - for( int ci = 0; ci < TEST_CASE_COUNT; ci++ ) - { - stringstream ss; ss << "case" << ci; - fs << ss.str() << "{"; - writeResults( fs, di, ci ); - fs << "}"; //ss.str() - } - fs << "}"; //DATASET_NAMES[di] - } + cout << "filename " << filename.str() << endl; + return false; } - fs << "}"; //results } - else - ts->printf(cvtest::TS::LOG, "results were not written because file %s can not be opened\n", filename.c_str() ); + return true; } -void BaseQualityTest::processResults( int datasetIdx ) +void BaseQualityEvaluator::processResults( int datasetIdx ) { if( isWriteGraphicsData ) writePlotData( datasetIdx ); } -void BaseQualityTest::processResults() +void BaseQualityEvaluator::processResults() { if( isWriteParams ) writeAllDatasetsRunParams(); - - if( isWriteGraphicsData ) - writeAveragePlotData(); - - int res = cvtest::TS::OK; - if( isWriteResults ) - writeResults(); - else - { - for( int di = 0; di < DATASETS_COUNT; di++ ) - { - if( isValidQualityEmpty(di) || isCalcQualityEmpty(di) ) - continue; - - ts->printf(cvtest::TS::LOG, "\nDataset: %s\n", DATASET_NAMES[di].c_str() ); - - for( int ci = 0; ci < TEST_CASE_COUNT; ci++ ) - { - ts->printf(cvtest::TS::LOG, "case%d\n", ci); - int currRes = processResults( di, ci ); - res = currRes == cvtest::TS::OK ? res : currRes; - } - } - } - - if( res != cvtest::TS::OK ) - ts->printf(cvtest::TS::LOG, "BAD ACCURACY\n"); - ts->set_failed_test_info( res ); } -void BaseQualityTest::run ( int ) +void BaseQualityEvaluator::run() { readAlgorithm (); processRunParamsFile (); - readResults(); int notReadDatasets = 0; int progress = 0; @@ -417,13 +322,15 @@ void BaseQualityTest::run ( int ) FileNode defaultParams = topfn[DEFAULT_PARAMS]; readDefaultRunParams (defaultParams); + cout << testName << endl; for(int di = 0; di < DATASETS_COUNT; di++ ) { + cout << "Dataset " << di << " [" << DATASET_NAMES[di] << "] " << flush; vector imgs, Hs; if( !readDataset( DATASET_NAMES[di], Hs, imgs ) ) { calcQualityClear (di); - ts->printf( cvtest::TS::LOG, "images or homography matrices of dataset named %s can not be read\n", + printf( "Images or homography matrices of dataset named %s can not be read\n", DATASET_NAMES[di].c_str()); notReadDatasets++; continue; @@ -434,11 +341,12 @@ void BaseQualityTest::run ( int ) runDatasetTest (imgs, Hs, di, progress); processResults( di ); + cout << endl; } if( notReadDatasets == DATASETS_COUNT ) { - ts->printf(cvtest::TS::LOG, "All datasets were not be read\n"); - ts->set_failed_test_info( cvtest::TS::FAIL_INVALID_TEST_DATA ); + printf( "All datasets were not be read\n"); + exit(-1); } else processResults(); @@ -447,13 +355,11 @@ void BaseQualityTest::run ( int ) -class DetectorQualityTest : public BaseQualityTest +class DetectorQualityEvaluator : public BaseQualityEvaluator { public: - DetectorQualityTest( const char* _detectorName ) : - BaseQualityTest( _detectorName ) + DetectorQualityEvaluator( const char* _detectorName, const char* _testName ) : BaseQualityEvaluator( _detectorName, _testName ) { - validQuality.resize(DATASETS_COUNT); calcQuality.resize(DATASETS_COUNT); isSaveKeypoints.resize(DATASETS_COUNT); isActiveParams.resize(DATASETS_COUNT); @@ -463,23 +369,13 @@ public: } protected: - using BaseQualityTest::readResults; - using BaseQualityTest::writeResults; - using BaseQualityTest::processResults; - virtual string getRunParamsFilename() const; virtual string getResultsFilename() const; virtual string getPlotPath() const; - virtual void validQualityClear( int datasetIdx ); virtual void calcQualityClear( int datasetIdx ); - virtual void validQualityCreate( int datasetIdx ); - virtual bool isValidQualityEmpty( int datasetIdx ) const; virtual bool isCalcQualityEmpty( int datasetIdx ) const; - virtual void readResults( FileNode& fn, int datasetIdx, int caseIdx ); - virtual void writeResults( FileStorage& fs, int datasetIdx, int caseIdx ) const; - virtual void readDatasetRunParams( FileNode& fn, int datasetIdx ); virtual void writeDatasetRunParams( FileStorage& fs, int datasetIdx ) const; virtual void setDefaultDatasetRunParams( int datasetIdx ); @@ -487,14 +383,12 @@ protected: virtual void writeDefaultRunParams( FileStorage &fs ) const; virtual void writePlotData( int di ) const; - virtual void writeAveragePlotData() const; void openToWriteKeypointsFile( FileStorage& fs, int datasetIdx ); - virtual void readAlgorithm( ); - virtual void processRunParamsFile () {}; + virtual void readAlgorithm(); + virtual void processRunParamsFile() {} virtual void runDatasetTest( const vector &imgs, const vector &Hs, int di, int &progress ); - virtual int processResults( int datasetIdx, int caseIdx ); Ptr specificDetector; Ptr defaultDetector; @@ -504,7 +398,6 @@ protected: float repeatability; int correspondenceCount; }; - vector > validQuality; vector > calcQuality; vector isSaveKeypoints; @@ -514,59 +407,32 @@ protected: bool isActiveParamsDefault; }; -string DetectorQualityTest::getRunParamsFilename() const +string DetectorQualityEvaluator::getRunParamsFilename() const { - return string(ts->get_data_path()) + DETECTORS_DIR + algName + PARAMS_POSTFIX; + return data_path + DETECTORS_DIR + algName + PARAMS_POSTFIX; } -string DetectorQualityTest::getResultsFilename() const +string DetectorQualityEvaluator::getResultsFilename() const { - return string(ts->get_data_path()) + DETECTORS_DIR + algName + RES_POSTFIX; + return data_path + DETECTORS_DIR + algName + RES_POSTFIX; } -string DetectorQualityTest::getPlotPath() const +string DetectorQualityEvaluator::getPlotPath() const { - return string(ts->get_data_path()) + DETECTORS_DIR + "plots/"; + return data_path + DETECTORS_DIR + "plots/"; } -void DetectorQualityTest::validQualityClear( int datasetIdx ) -{ - validQuality[datasetIdx].clear(); -} - -void DetectorQualityTest::calcQualityClear( int datasetIdx ) +void DetectorQualityEvaluator::calcQualityClear( int datasetIdx ) { calcQuality[datasetIdx].clear(); } -void DetectorQualityTest::validQualityCreate( int datasetIdx ) -{ - validQuality[datasetIdx].resize(TEST_CASE_COUNT); -} - -bool DetectorQualityTest::isValidQualityEmpty( int datasetIdx ) const -{ - return validQuality[datasetIdx].empty(); -} - -bool DetectorQualityTest::isCalcQualityEmpty( int datasetIdx ) const +bool DetectorQualityEvaluator::isCalcQualityEmpty( int datasetIdx ) const { return calcQuality[datasetIdx].empty(); } -void DetectorQualityTest::readResults( FileNode& fn, int datasetIdx, int caseIdx ) -{ - validQuality[datasetIdx][caseIdx].repeatability = fn[REPEAT]; - validQuality[datasetIdx][caseIdx].correspondenceCount = fn[CORRESP_COUNT]; -} - -void DetectorQualityTest::writeResults( FileStorage& fs, int datasetIdx, int caseIdx ) const -{ - fs << REPEAT << calcQuality[datasetIdx][caseIdx].repeatability; - fs << CORRESP_COUNT << calcQuality[datasetIdx][caseIdx].correspondenceCount; -} - -void DetectorQualityTest::readDefaultRunParams (FileNode &fn) +void DetectorQualityEvaluator::readDefaultRunParams (FileNode &fn) { if (! fn.empty() ) { @@ -575,13 +441,13 @@ void DetectorQualityTest::readDefaultRunParams (FileNode &fn) } } -void DetectorQualityTest::writeDefaultRunParams (FileStorage &fs) const +void DetectorQualityEvaluator::writeDefaultRunParams (FileStorage &fs) const { fs << IS_SAVE_KEYPOINTS << isSaveKeypointsDefault; defaultDetector->write (fs); } -void DetectorQualityTest::readDatasetRunParams( FileNode& fn, int datasetIdx ) +void DetectorQualityEvaluator::readDatasetRunParams( FileNode& fn, int datasetIdx ) { isActiveParams[datasetIdx] = (int)fn[IS_ACTIVE_PARAMS] != 0; if (isActiveParams[datasetIdx]) @@ -595,20 +461,20 @@ void DetectorQualityTest::readDatasetRunParams( FileNode& fn, int datasetIdx ) } } -void DetectorQualityTest::writeDatasetRunParams( FileStorage& fs, int datasetIdx ) const +void DetectorQualityEvaluator::writeDatasetRunParams( FileStorage& fs, int datasetIdx ) const { fs << IS_ACTIVE_PARAMS << isActiveParams[datasetIdx]; fs << IS_SAVE_KEYPOINTS << isSaveKeypoints[datasetIdx]; defaultDetector->write (fs); } -void DetectorQualityTest::setDefaultDatasetRunParams( int datasetIdx ) +void DetectorQualityEvaluator::setDefaultDatasetRunParams( int datasetIdx ) { isSaveKeypoints[datasetIdx] = isSaveKeypointsDefault; isActiveParams[datasetIdx] = isActiveParamsDefault; } -void DetectorQualityTest::writePlotData(int di ) const +void DetectorQualityEvaluator::writePlotData(int di ) const { int imgXVals[] = { 2, 3, 4, 5, 6 }; // if scale, blur or light changes int viewpointXVals[] = { 20, 30, 40, 50, 60 }; // if viewpoint changes @@ -637,36 +503,13 @@ void DetectorQualityTest::writePlotData(int di ) const } } -void DetectorQualityTest::writeAveragePlotData() const +void DetectorQualityEvaluator::openToWriteKeypointsFile( FileStorage& fs, int datasetIdx ) { - stringstream rFilename, cFilename; - rFilename << getPlotPath() << algName << "_average_repeatability.csv"; - cFilename << getPlotPath() << algName << "_average_correspondenceCount.csv"; - ofstream rfile(rFilename.str().c_str()), cfile(cFilename.str().c_str()); - float avRep = 0, avCorCount = 0; - for( int di = 0; di < DATASETS_COUNT; di++ ) - { - for( int ci = 0; ci < TEST_CASE_COUNT; ci++ ) - { - avRep += calcQuality[di][ci].repeatability; - avCorCount += calcQuality[di][ci].correspondenceCount; - } - } - avRep /= DATASETS_COUNT*TEST_CASE_COUNT; - avCorCount /= DATASETS_COUNT*TEST_CASE_COUNT; - rfile << algName << ", " << avRep << endl; - cfile << algName << ", " << cvRound(avCorCount) << endl; -} - -void DetectorQualityTest::openToWriteKeypointsFile( FileStorage& fs, int datasetIdx ) -{ - string filename = string(ts->get_data_path()) + KEYPOINTS_DIR + algName + "_"+ - DATASET_NAMES[datasetIdx] + ".xml.gz" ; + string filename = data_path + KEYPOINTS_DIR + algName + "_"+ DATASET_NAMES[datasetIdx] + ".xml.gz" ; fs.open(filename, FileStorage::WRITE); if( !fs.isOpened() ) - ts->printf( cvtest::TS::LOG, "keypoints can not be written in file %s because this file can not be opened\n", - filename.c_str()); + printf( "keypoints can not be written in file %s because this file can not be opened\n", filename.c_str() ); } inline void writeKeypoints( FileStorage& fs, const vector& keypoints, int imgIdx ) @@ -685,18 +528,39 @@ inline void readKeypoints( FileStorage& fs, vector& keypoints, int img read( fs[imgName.str()], keypoints); } -void DetectorQualityTest::readAlgorithm () +void DetectorQualityEvaluator::readAlgorithm () { defaultDetector = FeatureDetector::create( algName ); specificDetector = FeatureDetector::create( algName ); if( defaultDetector == 0 ) { - ts->printf(cvtest::TS::LOG, "Algorithm can not be read\n"); - ts->set_failed_test_info( cvtest::TS::FAIL_GENERIC); + printf( "Algorithm can not be read\n" ); + exit(-1); } } -void DetectorQualityTest::runDatasetTest (const vector &imgs, const vector &Hs, int di, int &progress) +int update_progress( const string& name, int progress, int test_case_idx, int count, double dt ) +{ + int width = 60 /*- (int)name.length()*/; + if( count > 0 ) + { + int t = cvRound( ((double)test_case_idx * width)/count ); + if( t > progress ) + { + cout << "." << flush; + progress = t; + } + } + else if( cvRound(dt) > progress ) + { + cout << "." << flush; + progress = cvRound(dt); + } + + return progress; +} + +void DetectorQualityEvaluator::runDatasetTest (const vector &imgs, const vector &Hs, int di, int &progress) { Ptr detector = isActiveParams[di] ? specificDetector : defaultDetector; FileStorage keypontsFS; @@ -709,9 +573,10 @@ void DetectorQualityTest::runDatasetTest (const vector &imgs, const vector< detector->detect( imgs[0], keypoints1 ); writeKeypoints( keypontsFS, keypoints1, 0); int progressCount = DATASETS_COUNT*TEST_CASE_COUNT; + for( int ci = 0; ci < TEST_CASE_COUNT; ci++ ) { - progress = update_progress( progress, di*TEST_CASE_COUNT + ci, progressCount, 0 ); + progress = update_progress( testName, progress, di*TEST_CASE_COUNT + ci + 1, progressCount, 0 ); vector keypoints2; float rep; evaluateFeatureDetector( imgs[0], imgs[ci+1], Hs[ci], &keypoints1, &keypoints2, @@ -722,34 +587,12 @@ void DetectorQualityTest::runDatasetTest (const vector &imgs, const vector< } } -void testLog( cvtest::TS* ts, bool isBadAccuracy ) +void testLog( bool isBadAccuracy ) { if( isBadAccuracy ) - ts->printf(cvtest::TS::LOG, " bad accuracy\n"); + printf(" bad accuracy\n"); else - ts->printf(cvtest::TS::LOG, "\n"); -} - -int DetectorQualityTest::processResults( int datasetIdx, int caseIdx ) -{ - int res = cvtest::TS::OK; - bool isBadAccuracy; - - Quality valid = validQuality[datasetIdx][caseIdx], calc = calcQuality[datasetIdx][caseIdx]; - - const int countEps = 1 + cvRound( 0.005f*(float)valid.correspondenceCount ); - const float rltvEps = 0.5f; - - ts->printf(cvtest::TS::LOG, "%s: calc=%f, valid=%f", REPEAT.c_str(), calc.repeatability, valid.repeatability ); - isBadAccuracy = (valid.repeatability - calc.repeatability) > rltvEps; - testLog( ts, isBadAccuracy ); - res = isBadAccuracy ? cvtest::TS::FAIL_BAD_ACCURACY : res; - - ts->printf(cvtest::TS::LOG, "%s: calc=%d, valid=%d", CORRESP_COUNT.c_str(), calc.correspondenceCount, valid.correspondenceCount ); - isBadAccuracy = (valid.correspondenceCount - calc.correspondenceCount) > countEps; - testLog( ts, isBadAccuracy ); - res = isBadAccuracy ? cvtest::TS::FAIL_BAD_ACCURACY : res; - return res; + printf("\n"); } /****************************************************************************************\ @@ -767,14 +610,13 @@ const string RUN_PARAMS_IS_IDENTICAL = "runParamsIsIdentical"; const string ONE_WAY_TRAIN_DIR = "detectors_descriptors_evaluation/one_way_train_images/"; const string ONE_WAY_IMAGES_LIST = "one_way_train_images.txt"; -class DescriptorQualityTest : public BaseQualityTest +class DescriptorQualityEvaluator : public BaseQualityEvaluator { public: enum{ NO_MATCH_FILTER = 0 }; - DescriptorQualityTest( const char* _descriptorName, const char* _matcherName = 0 ) : - BaseQualityTest( _descriptorName ) + DescriptorQualityEvaluator( const char* _descriptorName, const char* _testName, const char* _matcherName = 0 ) : + BaseQualityEvaluator( _descriptorName, _testName ) { - validQuality.resize(DATASETS_COUNT); calcQuality.resize(DATASETS_COUNT); calcDatasetQuality.resize(DATASETS_COUNT); commRunParams.resize(DATASETS_COUNT); @@ -788,35 +630,23 @@ public: } protected: - using BaseQualityTest::readResults; - using BaseQualityTest::writeResults; - using BaseQualityTest::processResults; - virtual string getRunParamsFilename() const; virtual string getResultsFilename() const; virtual string getPlotPath() const; - virtual void validQualityClear( int datasetIdx ); virtual void calcQualityClear( int datasetIdx ); - virtual void validQualityCreate( int datasetIdx ); - virtual bool isValidQualityEmpty( int datasetIdx ) const; virtual bool isCalcQualityEmpty( int datasetIdx ) const; - virtual void readResults( FileNode& fn, int datasetIdx, int caseIdx ); - virtual void writeResults( FileStorage& fs, int datasetIdx, int caseIdx ) const; - virtual void readDatasetRunParams( FileNode& fn, int datasetIdx ); // virtual void writeDatasetRunParams( FileStorage& fs, int datasetIdx ) const; virtual void setDefaultDatasetRunParams( int datasetIdx ); virtual void readDefaultRunParams( FileNode &fn ); virtual void writeDefaultRunParams( FileStorage &fs ) const; - virtual void readAlgorithm( ); - virtual void processRunParamsFile () {}; + virtual void readAlgorithm(); + virtual void processRunParamsFile() {} virtual void runDatasetTest( const vector &imgs, const vector &Hs, int di, int &progress ); - virtual int processResults( int datasetIdx, int caseIdx ); - virtual void writePlotData( int di ) const; void calculatePlotData( vector > &allMatches, vector > &allCorrectMatchesMask, int di ); @@ -825,7 +655,6 @@ protected: float recall; float precision; }; - vector > validQuality; vector > calcQuality; vector > calcDatasetQuality; @@ -845,59 +674,32 @@ protected: string matcherName; }; -string DescriptorQualityTest::getRunParamsFilename() const +string DescriptorQualityEvaluator::getRunParamsFilename() const { - return string(ts->get_data_path()) + DESCRIPTORS_DIR + algName + PARAMS_POSTFIX; + return data_path + DESCRIPTORS_DIR + algName + PARAMS_POSTFIX; } -string DescriptorQualityTest::getResultsFilename() const +string DescriptorQualityEvaluator::getResultsFilename() const { - return string(ts->get_data_path()) + DESCRIPTORS_DIR + algName + RES_POSTFIX; + return data_path + DESCRIPTORS_DIR + algName + RES_POSTFIX; } -string DescriptorQualityTest::getPlotPath() const +string DescriptorQualityEvaluator::getPlotPath() const { - return string(ts->get_data_path()) + DESCRIPTORS_DIR + "plots/"; + return data_path + DESCRIPTORS_DIR + "plots/"; } -void DescriptorQualityTest::validQualityClear( int datasetIdx ) -{ - validQuality[datasetIdx].clear(); -} - -void DescriptorQualityTest::calcQualityClear( int datasetIdx ) +void DescriptorQualityEvaluator::calcQualityClear( int datasetIdx ) { calcQuality[datasetIdx].clear(); } -void DescriptorQualityTest::validQualityCreate( int datasetIdx ) -{ - validQuality[datasetIdx].resize(TEST_CASE_COUNT); -} - -bool DescriptorQualityTest::isValidQualityEmpty( int datasetIdx ) const -{ - return validQuality[datasetIdx].empty(); -} - -bool DescriptorQualityTest::isCalcQualityEmpty( int datasetIdx ) const +bool DescriptorQualityEvaluator::isCalcQualityEmpty( int datasetIdx ) const { return calcQuality[datasetIdx].empty(); } -void DescriptorQualityTest::readResults( FileNode& fn, int datasetIdx, int caseIdx ) -{ - validQuality[datasetIdx][caseIdx].recall = fn[RECALL]; - validQuality[datasetIdx][caseIdx].precision = fn[PRECISION]; -} - -void DescriptorQualityTest::writeResults( FileStorage& fs, int datasetIdx, int caseIdx ) const -{ - fs << RECALL << calcQuality[datasetIdx][caseIdx].recall; - fs << PRECISION << calcQuality[datasetIdx][caseIdx].precision; -} - -void DescriptorQualityTest::readDefaultRunParams (FileNode &fn) +void DescriptorQualityEvaluator::readDefaultRunParams (FileNode &fn) { if (! fn.empty() ) { @@ -907,14 +709,14 @@ void DescriptorQualityTest::readDefaultRunParams (FileNode &fn) } } -void DescriptorQualityTest::writeDefaultRunParams (FileStorage &fs) const +void DescriptorQualityEvaluator::writeDefaultRunParams (FileStorage &fs) const { fs << PROJECT_KEYPOINTS_FROM_1IMAGE << commRunParamsDefault.projectKeypointsFrom1Image; fs << MATCH_FILTER << commRunParamsDefault.matchFilter; defaultDescMatcher->write (fs); } -void DescriptorQualityTest::readDatasetRunParams( FileNode& fn, int datasetIdx ) +void DescriptorQualityEvaluator::readDatasetRunParams( FileNode& fn, int datasetIdx ) { commRunParams[datasetIdx].isActiveParams = (int)fn[IS_ACTIVE_PARAMS] != 0; if (commRunParams[datasetIdx].isActiveParams) @@ -930,7 +732,7 @@ void DescriptorQualityTest::readDatasetRunParams( FileNode& fn, int datasetIdx ) } } -void DescriptorQualityTest::writeDatasetRunParams( FileStorage& fs, int datasetIdx ) const +void DescriptorQualityEvaluator::writeDatasetRunParams( FileStorage& fs, int datasetIdx ) const { fs << IS_ACTIVE_PARAMS << commRunParams[datasetIdx].isActiveParams; fs << KEYPOINTS_FILENAME << commRunParams[datasetIdx].keypontsFilename; @@ -940,13 +742,13 @@ void DescriptorQualityTest::writeDatasetRunParams( FileStorage& fs, int datasetI defaultDescMatcher->write (fs); } -void DescriptorQualityTest::setDefaultDatasetRunParams( int datasetIdx ) +void DescriptorQualityEvaluator::setDefaultDatasetRunParams( int datasetIdx ) { commRunParams[datasetIdx] = commRunParamsDefault; commRunParams[datasetIdx].keypontsFilename = "SURF_" + DATASET_NAMES[datasetIdx] + ".xml.gz"; } -void DescriptorQualityTest::writePlotData( int di ) const +void DescriptorQualityEvaluator::writePlotData( int di ) const { stringstream filename; filename << getPlotPath() << algName << "_" << DATASET_NAMES[di] << ".csv"; @@ -959,7 +761,7 @@ void DescriptorQualityTest::writePlotData( int di ) const fclose( file ); } -void DescriptorQualityTest::readAlgorithm( ) +void DescriptorQualityEvaluator::readAlgorithm( ) { defaultDescMatcher = GenericDescriptorMatcher::create( algName ); specificDescMatcher = GenericDescriptorMatcher::create( algName ); @@ -973,13 +775,13 @@ void DescriptorQualityTest::readAlgorithm( ) if( extractor == 0 || matcher == 0 ) { - ts->printf(cvtest::TS::LOG, "Algorithm can not be read\n"); - ts->set_failed_test_info( cvtest::TS::FAIL_GENERIC); + printf("Algorithm can not be read\n"); + exit(-1); } } } -void DescriptorQualityTest::calculatePlotData( vector > &allMatches, vector > &allCorrectMatchesMask, int di ) +void DescriptorQualityEvaluator::calculatePlotData( vector > &allMatches, vector > &allCorrectMatchesMask, int di ) { vector recallPrecisionCurve; computeRecallPrecisionCurve( allMatches, allCorrectMatchesMask, recallPrecisionCurve ); @@ -1018,14 +820,13 @@ void DescriptorQualityTest::calculatePlotData( vector > &allMatch } } -void DescriptorQualityTest::runDatasetTest (const vector &imgs, const vector &Hs, int di, int &progress) +void DescriptorQualityEvaluator::runDatasetTest (const vector &imgs, const vector &Hs, int di, int &progress) { - FileStorage keypontsFS( string(ts->get_data_path()) + KEYPOINTS_DIR + commRunParams[di].keypontsFilename, - FileStorage::READ ); + FileStorage keypontsFS( data_path + KEYPOINTS_DIR + commRunParams[di].keypontsFilename, FileStorage::READ ); if( !keypontsFS.isOpened()) { calcQuality[di].clear(); - ts->printf( cvtest::TS::LOG, "keypoints from file %s can not be read\n", commRunParams[di].keypontsFilename.c_str() ); + printf( "keypoints from file %s can not be read\n", commRunParams[di].keypontsFilename.c_str() ); return; } @@ -1041,7 +842,7 @@ void DescriptorQualityTest::runDatasetTest (const vector &imgs, const vecto vector > allCorrectMatchesMask; for( int ci = 0; ci < TEST_CASE_COUNT; ci++ ) { - progress = update_progress( progress, di*TEST_CASE_COUNT + ci, progressCount, 0 ); + progress = update_progress( testName, progress, di*TEST_CASE_COUNT + ci + 1, progressCount, 0 ); vector keypoints2; if( commRunParams[di].projectKeypointsFrom1Image ) @@ -1068,37 +869,15 @@ void DescriptorQualityTest::runDatasetTest (const vector &imgs, const vecto calculatePlotData( allMatches1to2, allCorrectMatchesMask, di ); } -int DescriptorQualityTest::processResults( int datasetIdx, int caseIdx ) -{ - const float rltvEps = 0.001f; - - int res = cvtest::TS::OK; - bool isBadAccuracy; - - Quality valid = validQuality[datasetIdx][caseIdx], calc = calcQuality[datasetIdx][caseIdx]; - - ts->printf(cvtest::TS::LOG, "%s: calc=%f, valid=%f", RECALL.c_str(), calc.recall, valid.recall ); - isBadAccuracy = (valid.recall - calc.recall) > rltvEps; - testLog( ts, isBadAccuracy ); - res = isBadAccuracy ? cvtest::TS::FAIL_BAD_ACCURACY : res; - - ts->printf(cvtest::TS::LOG, "%s: calc=%f, valid=%f", PRECISION.c_str(), calc.precision, valid.precision ); - isBadAccuracy = (valid.precision - calc.precision) > rltvEps; - testLog( ts, isBadAccuracy ); - res = isBadAccuracy ? cvtest::TS::FAIL_BAD_ACCURACY : res; - - return res; -} - //--------------------------------- Calonder descriptor test -------------------------------------------- -class CalonderDescriptorQualityTest : public DescriptorQualityTest +class CalonderDescriptorQualityEvaluator : public DescriptorQualityEvaluator { public: - CalonderDescriptorQualityTest() : - DescriptorQualityTest( "Calonder", "quality-descriptor-calonder") {} + CalonderDescriptorQualityEvaluator() : + DescriptorQualityEvaluator( "Calonder", "quality-descriptor-calonder") {} virtual void readAlgorithm( ) { - string classifierFile = string(ts->get_data_path()) + "/features2d/calonder_classifier.rtc"; + string classifierFile = data_path + "/features2d/calonder_classifier.rtc"; defaultDescMatcher = new VectorDescriptorMatch( new CalonderDescriptorExtractor( classifierFile ), new BruteForceMatcher > ); specificDescMatcher = defaultDescMatcher; @@ -1106,11 +885,11 @@ public: }; //--------------------------------- One Way descriptor test -------------------------------------------- -class OneWayDescriptorQualityTest : public DescriptorQualityTest +class OneWayDescriptorQualityTest : public DescriptorQualityEvaluator { public: OneWayDescriptorQualityTest() : - DescriptorQualityTest("ONEWAY", "quality-descriptor-one-way") + DescriptorQualityEvaluator("ONEWAY", "quality-descriptor-one-way") { } protected: @@ -1125,8 +904,8 @@ void OneWayDescriptorQualityTest::processRunParamsFile () FileNode fn = fs.getFirstTopLevelNode(); fn = fn[DEFAULT_PARAMS]; - string pcaFilename = string(ts->get_data_path()) + (string)fn["pcaFilename"]; - string trainPath = string(ts->get_data_path()) + (string)fn["trainPath"]; + string pcaFilename = data_path + (string)fn["pcaFilename"]; + string trainPath = data_path + (string)fn["trainPath"]; string trainImagesList = (string)fn["trainImagesList"]; int patch_width = fn["patchWidth"]; int patch_height = fn["patchHeight"]; @@ -1157,30 +936,42 @@ void OneWayDescriptorQualityTest::writeDatasetRunParams( FileStorage& fs, int da fs << MATCH_FILTER << commRunParams[datasetIdx].matchFilter; } -// Detectors -//DetectorQualityTest fastDetectorQuality = DetectorQualityTest( "FAST", "quality-detector-fast" ); -//DetectorQualityTest gfttDetectorQuality = DetectorQualityTest( "GFTT", "quality-detector-gftt" ); -//DetectorQualityTest harrisDetectorQuality = DetectorQualityTest( "HARRIS", "quality-detector-harris" ); -//DetectorQualityTest mserDetectorQuality = DetectorQualityTest( "MSER", "quality-detector-mser" ); -//DetectorQualityTest starDetectorQuality = DetectorQualityTest( "STAR", "quality-detector-star" ); -//DetectorQualityTest siftDetectorQuality = DetectorQualityTest( "SIFT", "quality-detector-sift" ); -//DetectorQualityTest surfDetectorQuality = DetectorQualityTest( "SURF", "quality-detector-surf" ); - -// Descriptors -//DescriptorQualityTest siftDescriptorQuality = DescriptorQualityTest( "SIFT", "quality-descriptor-sift", "BruteForce" ); -//DescriptorQualityTest surfDescriptorQuality = DescriptorQualityTest( "SURF", "quality-descriptor-surf", "BruteForce" ); -//DescriptorQualityTest fernDescriptorQualityTest( "FERN", "quality-descriptor-fern"); -//CalonderDescriptorQualityTest calonderDescriptorQualityTest; - +int main( int argc, char** argv ) +{ + if( argc != 2 ) + { + cout << "Format: " << argv[0] << " testdata path (path to testdata/cv)" << endl; + return -1; + } + data_path = argv[1]; +#ifdef WIN32 + if( *data_path.rbegin() != '\\' ) + data_path = data_path + "\\"; +#else + if( *data_path.rbegin() != '/' ) + data_path = data_path + "/"; +#endif -// Don't run it because of bug in OneWayDescriptorBase many to many matching. TODO: fix this bug. -//OneWayDescriptorQualityTest oneWayDescriptorQuality; + Ptr evals[] = + { + new DetectorQualityEvaluator( "FAST", "quality-detector-fast" ), + new DetectorQualityEvaluator( "GFTT", "quality-detector-gftt" ), + new DetectorQualityEvaluator( "HARRIS", "quality-detector-harris" ), + new DetectorQualityEvaluator( "MSER", "quality-detector-mser" ), + new DetectorQualityEvaluator( "STAR", "quality-detector-star" ), + new DetectorQualityEvaluator( "SIFT", "quality-detector-sift" ), + new DetectorQualityEvaluator( "SURF", "quality-detector-surf" ), -// Don't run them (will validate and save results as "quality-descriptor-sift" and "quality-descriptor-surf" test data). -// TODO: differ result filenames. -//DescriptorQualityTest siftL1DescriptorQuality = DescriptorQualityTest( "SIFT", "quality-descriptor-sift-L1", "BruteForce-L1" ); -//DescriptorQualityTest surfL1DescriptorQuality = DescriptorQualityTest( "SURF", "quality-descriptor-surf-L1", "BruteForce-L1" ); -//DescriptorQualityTest oppSiftL1DescriptorQuality = DescriptorQualityTest( "SIFT", "quality-descriptor-opponent-sift-L1", "BruteForce-L1" ); -//DescriptorQualityTest oppSurfL1DescriptorQuality = DescriptorQualityTest( "SURF", "quality-descriptor-opponent-surf-L1", "BruteForce-L1" ); + new DescriptorQualityEvaluator( "SIFT", "quality-descriptor-sift", "BruteForce" ), + new DescriptorQualityEvaluator( "SURF", "quality-descriptor-surf", "BruteForce" ), + new DescriptorQualityEvaluator( "FERN", "quality-descriptor-fern"), + new CalonderDescriptorQualityEvaluator() + }; + for( size_t i = 0; i < sizeof(evals)/sizeof(evals[0]); i++ ) + { + evals[i]->run(); + cout << endl; + } +} -- 2.7.4