tests/main.cpp

   1 /*
   2  * Copyright (c) 2017-2018 ARM Limited.
   3  *
   4  * SPDX-License-Identifier: MIT
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a copy
   7  * of this software and associated documentation files (the "Software"), to
   8  * deal in the Software without restriction, including without limitation the
   9  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
  10  * sell copies of the Software, and to permit persons to whom the Software is
  11  * furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in all
  14  * copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  19  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  22  * SOFTWARE.
  23  */
  24 #include "support/ToolchainSupport.h"
  25 #include "tests/AssetsLibrary.h"
  26 #include "tests/framework/DatasetModes.h"
  27 #include "tests/framework/Exceptions.h"
  28 #include "tests/framework/Framework.h"
  29 #include "tests/framework/Macros.h"
  30 #include "tests/framework/Profiler.h"
  31 #include "tests/framework/command_line/CommandLineOptions.h"
  32 #include "tests/framework/command_line/CommandLineParser.h"
  33 #include "tests/framework/command_line/CommonOptions.h"
  34 #include "tests/framework/instruments/Instruments.h"
  35 #include "tests/framework/printers/Printers.h"
  36
  37 #ifdef ARM_COMPUTE_CL
  38 #include "arm_compute/runtime/CL/CLScheduler.h"
  39 #endif /* ARM_COMPUTE_CL */
  40 #ifdef ARM_COMPUTE_GC
  41 #include "arm_compute/runtime/GLES_COMPUTE/GCScheduler.h"
  42 #endif /* ARM_COMPUTE_GC */
  43 #include "arm_compute/runtime/Scheduler.h"
  44
  45 #include <fstream>
  46 #include <initializer_list>
  47 #include <iostream>
  48 #include <memory>
  49 #include <random>
  50 #include <utility>
  51
  52 using namespace arm_compute;
  53 using namespace arm_compute::test;
  54
  55 namespace arm_compute
  56 {
  57 namespace test
  58 {
  59 std::unique_ptr<AssetsLibrary> library;
  60 } // namespace test
  61 } // namespace arm_compute
  62
  63 int main(int argc, char **argv)
  64 {
  65 #ifdef ARM_COMPUTE_CL
  66     CLScheduler::get().default_init();
  67 #endif /* ARM_COMPUTE_CL */
  68
  69 #ifdef ARM_COMPUTE_GC
  70     GCScheduler::get().default_init();
  71 #endif /* ARM_COMPUTE_GC */
  72
  73     framework::Framework &framework = framework::Framework::get();
  74
  75     framework::CommandLineParser parser;
  76
  77     std::set<framework::DatasetMode> allowed_modes
  78     {
  79         framework::DatasetMode::PRECOMMIT,
  80         framework::DatasetMode::NIGHTLY,
  81         framework::DatasetMode::ALL
  82     };
  83
  84     framework::CommonOptions options(parser);
  85
  86     auto dataset_mode = parser.add_option<framework::EnumOption<framework::DatasetMode>>("mode", allowed_modes, framework::DatasetMode::PRECOMMIT);
  87     dataset_mode->set_help("For managed datasets select which group to use");
  88     auto filter = parser.add_option<framework::SimpleOption<std::string>>("filter", ".*");
  89     filter->set_help("Regular expression to select test cases");
  90     auto filter_id = parser.add_option<framework::SimpleOption<std::string>>("filter-id");
  91     filter_id->set_help("List of test ids. ... can be used to define a range.");
  92     auto stop_on_error = parser.add_option<framework::ToggleOption>("stop-on-error");
  93     stop_on_error->set_help("Abort execution after the first failed test (useful for debugging)");
  94     auto seed = parser.add_option<framework::SimpleOption<std::random_device::result_type>>("seed", std::random_device()());
  95     seed->set_help("Global seed for random number generation");
  96     auto list_tests = parser.add_option<framework::ToggleOption>("list-tests", false);
  97     list_tests->set_help("List all test names");
  98     auto test_instruments = parser.add_option<framework::ToggleOption>("test-instruments", false);
  99     test_instruments->set_help("Test if the instruments work on the platform");
 100     auto error_on_missing_assets = parser.add_option<framework::ToggleOption>("error-on-missing-assets", false);
 101     error_on_missing_assets->set_help("Mark a test as failed instead of skipping it when assets are missing");
 102     auto assets = parser.add_positional_option<framework::SimpleOption<std::string>>("assets");
 103     assets->set_help("Path to the assets directory");
 104
 105     try
 106     {
 107         parser.parse(argc, argv);
 108
 109         if(options.help->is_set() && options.help->value())
 110         {
 111             parser.print_help(argv[0]);
 112             return 0;
 113         }
 114
 115         std::vector<std::unique_ptr<framework::Printer>> printers = options.create_printers();
 116
 117         Scheduler::get().set_num_threads(options.threads->value());
 118
 119         if(options.log_level->value() > framework::LogLevel::NONE)
 120         {
 121             for(auto &p : printers)
 122             {
 123                 p->print_global_header();
 124             }
 125         }
 126
 127         if(options.log_level->value() >= framework::LogLevel::CONFIG)
 128         {
 129             for(auto &p : printers)
 130             {
 131                 p->print_entry("Version", build_information());
 132                 p->print_entry("Seed", support::cpp11::to_string(seed->value()));
 133 #ifdef ARM_COMPUTE_CL
 134                 if(opencl_is_available())
 135                 {
 136                     p->print_entry("CL_DEVICE_VERSION", CLKernelLibrary::get().get_device_version());
 137                 }
 138                 else
 139                 {
 140                     p->print_entry("CL_DEVICE_VERSION", "Unavailable");
 141                 }
 142 #endif /* ARM_COMPUTE_CL */
 143                 p->print_entry("Iterations", support::cpp11::to_string(options.iterations->value()));
 144                 p->print_entry("Threads", support::cpp11::to_string(options.threads->value()));
 145                 {
 146                     using support::cpp11::to_string;
 147                     p->print_entry("Dataset mode", to_string(dataset_mode->value()));
 148                 }
 149             }
 150         }
 151
 152         framework.init(options.instruments->value(), options.iterations->value(), dataset_mode->value(), filter->value(), filter_id->value(), options.log_level->value());
 153         for(auto &p : printers)
 154         {
 155             framework.add_printer(p.get());
 156         }
 157         framework.set_throw_errors(options.throw_errors->value());
 158         framework.set_stop_on_error(stop_on_error->value());
 159         framework.set_error_on_missing_assets(error_on_missing_assets->value());
 160
 161         bool success = true;
 162
 163         if(list_tests->value())
 164         {
 165             for(auto &p : printers)
 166             {
 167                 p->print_list_tests(framework.test_infos());
 168                 p->print_global_footer();
 169             }
 170
 171             return 0;
 172         }
 173
 174         if(test_instruments->value())
 175         {
 176             framework::Profiler profiler = framework.get_profiler();
 177             profiler.start();
 178             profiler.stop();
 179             for(auto &p : printers)
 180             {
 181                 p->print_measurements(profiler.measurements());
 182             }
 183
 184             return 0;
 185         }
 186
 187         library = support::cpp14::make_unique<AssetsLibrary>(assets->value(), seed->value());
 188
 189         if(!parser.validate())
 190         {
 191             return 1;
 192         }
 193
 194         success = framework.run();
 195
 196         if(options.log_level->value() > framework::LogLevel::NONE)
 197         {
 198             for(auto &p : printers)
 199             {
 200                 p->print_global_footer();
 201             }
 202         }
 203
 204 #ifdef ARM_COMPUTE_CL
 205         CLScheduler::get().sync();
 206 #endif /* ARM_COMPUTE_CL */
 207
 208         return (success ? 0 : 1);
 209     }
 210     catch(const std::exception &error)
 211     {
 212         std::cerr << error.what() << "\n";
 213
 214         if(options.throw_errors->value())
 215         {
 216             throw;
 217         }
 218
 219         return 1;
 220     }
 221
 222     return 0;
 223 }