tests/main.cpp

   1 /*
   2  * Copyright (c) 2017-2018 ARM Limited.
   3  *
   4  * SPDX-License-Identifier: MIT
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a copy
   7  * of this software and associated documentation files (the "Software"), to
   8  * deal in the Software without restriction, including without limitation the
   9  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
  10  * sell copies of the Software, and to permit persons to whom the Software is
  11  * furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in all
  14  * copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  19  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  22  * SOFTWARE.
  23  */
  24 #include "support/ToolchainSupport.h"
  25 #include "tests/AssetsLibrary.h"
  26 #include "tests/framework/DatasetModes.h"
  27 #include "tests/framework/Exceptions.h"
  28 #include "tests/framework/Framework.h"
  29 #include "tests/framework/Macros.h"
  30 #include "tests/framework/Profiler.h"
  31 #include "tests/framework/command_line/CommandLineOptions.h"
  32 #include "tests/framework/command_line/CommandLineParser.h"
  33 #include "tests/framework/command_line/CommonOptions.h"
  34 #include "tests/framework/instruments/Instruments.h"
  35 #include "tests/framework/printers/Printers.h"
  36
  37 #ifdef ARM_COMPUTE_CL
  38 #include "arm_compute/runtime/CL/CLScheduler.h"
  39 #endif /* ARM_COMPUTE_CL */
  40 #ifdef ARM_COMPUTE_GC
  41 #include "arm_compute/runtime/GLES_COMPUTE/GCScheduler.h"
  42 #endif /* ARM_COMPUTE_GC */
  43 #include "arm_compute/runtime/Scheduler.h"
  44
  45 #include <fstream>
  46 #include <initializer_list>
  47 #include <iostream>
  48 #include <memory>
  49 #include <random>
  50 #include <utility>
  51
  52 using namespace arm_compute;
  53 using namespace arm_compute::test;
  54
  55 namespace arm_compute
  56 {
  57 namespace test
  58 {
  59 std::unique_ptr<AssetsLibrary> library;
  60 } // namespace test
  61 } // namespace arm_compute
  62
  63 int main(int argc, char **argv)
  64 {
  65 #ifdef ARM_COMPUTE_CL
  66     CLScheduler::get().default_init();
  67 #endif /* ARM_COMPUTE_CL */
  68
  69 #ifdef ARM_COMPUTE_GC
  70     GCScheduler::get().default_init();
  71 #endif /* ARM_COMPUTE_GC */
  72
  73     framework::Framework &framework = framework::Framework::get();
  74
  75     framework::CommandLineParser parser;
  76
  77     std::set<framework::DatasetMode> allowed_modes
  78     {
  79         framework::DatasetMode::PRECOMMIT,
  80         framework::DatasetMode::NIGHTLY,
  81         framework::DatasetMode::ALL
  82     };
  83
  84     framework::CommonOptions options(parser);
  85
  86     auto dataset_mode = parser.add_option<framework::EnumOption<framework::DatasetMode>>("mode", allowed_modes, framework::DatasetMode::PRECOMMIT);
  87     dataset_mode->set_help("For managed datasets select which group to use");
  88     auto filter = parser.add_option<framework::SimpleOption<std::string>>("filter", ".*");
  89     filter->set_help("Regular expression to select test cases");
  90     auto filter_id = parser.add_option<framework::SimpleOption<std::string>>("filter-id");
  91     filter_id->set_help("List of test ids. ... can be used to define a range.");
  92     auto stop_on_error = parser.add_option<framework::ToggleOption>("stop-on-error");
  93     stop_on_error->set_help("Abort execution after the first failed test (useful for debugging)");
  94     auto seed = parser.add_option<framework::SimpleOption<std::random_device::result_type>>("seed", std::random_device()());
  95     seed->set_help("Global seed for random number generation");
  96     auto list_tests = parser.add_option<framework::ToggleOption>("list-tests", false);
  97     list_tests->set_help("List all test names");
  98     auto test_instruments = parser.add_option<framework::ToggleOption>("test-instruments", false);
  99     test_instruments->set_help("Test if the instruments work on the platform");
 100     auto error_on_missing_assets = parser.add_option<framework::ToggleOption>("error-on-missing-assets", false);
 101     error_on_missing_assets->set_help("Mark a test as failed instead of skipping it when assets are missing");
 102     auto assets = parser.add_positional_option<framework::SimpleOption<std::string>>("assets");
 103     assets->set_help("Path to the assets directory");
 104
 105     try
 106     {
 107         parser.parse(argc, argv);
 108
 109         if(options.help->is_set() && options.help->value())
 110         {
 111             parser.print_help(argv[0]);
 112             return 0;
 113         }
 114
 115         std::vector<std::unique_ptr<framework::Printer>> printers = options.create_printers();
 116
 117         Scheduler::get().set_num_threads(options.threads->value());
 118
 119         if(options.log_level->value() > framework::LogLevel::NONE)
 120         {
 121             for(auto &p : printers)
 122             {
 123                 p->print_global_header();
 124             }
 125         }
 126
 127         if(options.log_level->value() >= framework::LogLevel::CONFIG)
 128         {
 129             for(auto &p : printers)
 130             {
 131                 p->print_entry("Version", build_information());
 132                 p->print_entry("Seed", support::cpp11::to_string(seed->value()));
 133                 p->print_entry("Iterations", support::cpp11::to_string(options.iterations->value()));
 134                 p->print_entry("Threads", support::cpp11::to_string(options.threads->value()));
 135                 {
 136                     using support::cpp11::to_string;
 137                     p->print_entry("Dataset mode", to_string(dataset_mode->value()));
 138                 }
 139             }
 140         }
 141
 142         framework.init(options.instruments->value(), options.iterations->value(), dataset_mode->value(), filter->value(), filter_id->value(), options.log_level->value());
 143         for(auto &p : printers)
 144         {
 145             framework.add_printer(p.get());
 146         }
 147         framework.set_throw_errors(options.throw_errors->value());
 148         framework.set_stop_on_error(stop_on_error->value());
 149         framework.set_error_on_missing_assets(error_on_missing_assets->value());
 150
 151         bool success = true;
 152
 153         if(list_tests->value())
 154         {
 155             for(auto &p : printers)
 156             {
 157                 p->print_list_tests(framework.test_infos());
 158                 p->print_global_footer();
 159             }
 160
 161             return 0;
 162         }
 163
 164         if(test_instruments->value())
 165         {
 166             framework::Profiler profiler = framework.get_profiler();
 167             profiler.start();
 168             profiler.stop();
 169             for(auto &p : printers)
 170             {
 171                 p->print_measurements(profiler.measurements());
 172             }
 173
 174             return 0;
 175         }
 176
 177         library = support::cpp14::make_unique<AssetsLibrary>(assets->value(), seed->value());
 178
 179         if(!parser.validate())
 180         {
 181             return 1;
 182         }
 183
 184         success = framework.run();
 185
 186         if(options.log_level->value() > framework::LogLevel::NONE)
 187         {
 188             for(auto &p : printers)
 189             {
 190                 p->print_global_footer();
 191             }
 192         }
 193
 194 #ifdef ARM_COMPUTE_CL
 195         CLScheduler::get().sync();
 196 #endif /* ARM_COMPUTE_CL */
 197
 198         return (success ? 0 : 1);
 199     }
 200     catch(const std::exception &error)
 201     {
 202         std::cerr << error.what() << "\n";
 203
 204         if(options.throw_errors->value())
 205         {
 206             throw;
 207         }
 208
 209         return 1;
 210     }
 211
 212     return 0;
 213 }