From 0075b86074e19c7386190909914d884694458bdd Mon Sep 17 00:00:00 2001 From: Dmitriy Nikiforov Date: Thu, 24 Aug 2017 21:17:37 +0300 Subject: [PATCH] Add 'run-all' command to execute multiple targets on multiple devices Change-Id: Id2db9249756dbd19e5cc44d66037e74dcc0f93de --- infra/commands/run-all.sh | 289 ++++++++++++++++++++++++++++++++++++++++++++++ infra/tizen_fuzz.sh | 2 +- infra/utils.sh | 14 +++ 3 files changed, 304 insertions(+), 1 deletion(-) create mode 100755 infra/commands/run-all.sh diff --git a/infra/commands/run-all.sh b/infra/commands/run-all.sh new file mode 100755 index 0000000..5b3eaea --- /dev/null +++ b/infra/commands/run-all.sh @@ -0,0 +1,289 @@ +#!/bin/bash -e + +######################################################## +# HELP MESSAGE +######################################################## + +function print_help { + cat << EOF +Usage: ${EXEC} run-all [OPTS] [PROJECT] + +Runs all fuzz targets of all built projects or the specified project on a number +of 'workers' (emulator devices). All workers should be of the same architecture. + +If '--serials' option is not specified, then using all devices connected to sdb. + +Options: + -h, --help Prints this message. + -c, --corpus CORPUS_DIR Push the seed corpus to device. Works only with + '--publish' option. + -p, --publish Push built targets to device before execution. All + missing dependencies will be installed by default. + -s, --serials SERIAL[:SERIAL[...]] + List of serial numbers of workers. Order of serial + numbers does not define at which worker which job + will be executed. + -t, --timeout TIMEOUT Jobs rotation timeout in minutes. Default value is + 0 - no rotation is needed. When rotation is disabled + all jobs will run until crash. +EOF +} + +######################################################## +# UTILITY FUNCTIONS +######################################################## + +# Prints all not running workers. +function get_available_workers() { + for serial in "${SERIALS[@]}"; do + if ! contains_element "${serial}" "${!PIDS_BY_WORKERS[@]}"; then + echo "${serial}" + fi + done +} + +# Starts one pending job on the specified worker. +# Args: +# 1) serial number of the worker +function start_worker() { + if (( ${#PENDING[@]} == 0 )); then + return + fi + + local worker="${1}" + local job="${PENDING[0]}" + local target_name + PENDING=("${PENDING[@]:1}") + + # publish target if needed + if (( PUBLISH )); then + target_name=$(echo "${job}" | cut -d ' ' -f 1) + echo "* Publishing target '${target_name}' to ${worker} ..." + echo "" | "${COMMANDS_DIR}"/publish.sh --serial "${worker}" --with-asan --install-deps \ + "${EXTRA_PUBLISH_ARGS[@]}" "${target_name}" >/dev/null + fi + + # start fuzzing + echo "* Starting job on ${worker}: ${job} ..." + echo "" | "${COMMANDS_DIR}"/run.sh --serial "${worker}" ${job} >/dev/null 2>&1 & + + # save pid and job + local pid=$! + PIDS_BY_WORKERS["${worker}"]=${pid} + JOBS_BY_WORKERS["${worker}"]="${job}" +} + +# Prints all workers which processes were finished. +function get_finished_workers() { + local finished=() + for worker in "${!PIDS_BY_WORKERS[@]}"; do + local pid="${PIDS_BY_WORKERS[${worker}]}" + if [[ ! -d "/proc/${pid}" ]]; then + echo "${worker}" + fi + done +} + +# Generates reports with coverage for all specified workers. +# Args: +# 1-..) list of serial numbers of workers +function generate_reports() { + for worker in "${@}"; do + local job="${JOBS_BY_WORKERS[${worker}]}" + echo "* Generating report for job on ${worker}: ${job} ..." + echo "" | "${COMMANDS_DIR}"/run.sh --serial "${worker}" --dump-coverage ${job} >/dev/null + echo "" | "${COMMANDS_DIR}"/report.sh --serial "${worker}" --install-deps --coverage ${job} >/dev/null + echo + done +} + +# Frees all specified workers and according jobs. +# 1-..) list of serial numbers of workers +function free_workers_with_jobs() { + for worker in "${@}"; do + echo "* Finished job on ${worker}: ${JOBS_BY_WORKERS[${worker}]}" + unset PIDS_BY_WORKERS["${worker}"] + unset JOBS_BY_WORKERS["${worker}"] + done +} + +# Frees all specified workers and puts according jobs back to pending list. +# 1-..) list of serial numbers of workers +function free_workers() { + for worker in "${@}"; do + local job="${JOBS_BY_WORKERS[${worker}]}" + echo "* Finished job on ${worker} (forced): ${job}" + unset PIDS_BY_WORKERS["${worker}"] + unset JOBS_BY_WORKERS["${worker}"] + PENDING+=("${job}") + done +} + +# Stops specified number of workers, generates according reports and frees workers. +# 1) number of workers to be stopped +function stop_workers() { + local workers=("${!PIDS_BY_WORKERS[@]}") + local stopped=() + for (( I=0; I < ${1} && ${#workers[@]} > 0; ++I )); do + local worker="${workers[0]}" + local job="${JOBS_BY_WORKERS[${worker}]}" + local pid="${PIDS_BY_WORKERS[${worker}]}" + local target_bin + target_bin=$(echo "${job}" | cut -d ' ' -f 2) + + echo "* Stopping job on ${worker}: ${job} ..." + SDB_CMD="sdb -s ${worker}" sdb_shell pkill -f "${target_bin}" >/dev/null + wait ${pid} 2>/dev/null || true + + workers=("${workers[@]:1}") + stopped+=("${worker}") + done + + generate_reports "${stopped[@]}" + free_workers "${stopped[@]}" +} + +######################################################## +# GLOBAL VARIABLES +####################################################### + +SERIALS=() +PENDING=() +TIMEOUT=0 # minutes +SLEEP_TIMEOUT=10 # seconds +PUBLISH=0 +EXTRA_PUBLISH_ARGS=() + +declare -A PIDS_BY_WORKERS=() +declare -A JOBS_BY_WORKERS=() + +######################################################## +# GLOBAL OPTIONS PARSING +######################################################## + +while [[ ${1} = -* ]]; do + case ${1} in + '-h'|'--help') + print_help + exit 0 + ;; + '-c'|'--corpus') + EXTRA_PUBLISH_ARGS=("${1}" "${2}") + shift 2 + ;; + '-p'|'--publish') + PUBLISH=1 + shift + ;; + '-s'|'--serials') + SERIALS=($(echo "${2}" | tr ":" " ")) + shift 2 + ;; + '-t'|'--timeout') + TIMEOUT="${2}" + shift 2 + ;; + *) + echo "Error: Unknown option ${1}." + print_help + exit 1 + ;; + esac +done + +TARGET="${1}" + +######################################################## +# RUNNING TARGETS +######################################################## + +# export environment for subcommands +export_all + +# if no serials are specify, assume that all currently working emulators are to be used +if [[ ${#SERIALS[@]} -eq 0 ]]; then + SERIALS=($(sdb devices | sed -En "s/^([a-zA-Z0-9_-.]+)\s+(device)\s+([a-zA-Z0-9_-.<>]+)$/\1/p")) +fi + +# get the architecture +echo "Checking architecture..." +# this script assumes that all workers are of the same architecture +arch=$(SDB_CMD="sdb -s ${SERIALS[0]}" get_device_arch) +echo "Using architecture: ${arch}" + +# find all targets to be executed +[[ -n ${TARGET} ]] && pattern="*/${TARGET}/${arch}/target" || pattern="*/${arch}/target" +targets=($(find "${BUILD_ARTIFACTS_DIR}" -path "${pattern}" 2>/dev/null)) +if (( ${#targets[@]} == 0 )); then + echo "Error: no targets found." + exit 1 +fi + +# print found targets +echo -e "\nTargets to execute:" +cnt=1 +for target in "${targets[@]}"; do + target_name=$(echo "${target}" | sed -E "s|${BUILD_ARTIFACTS_DIR}/(.*)/${arch}/target|\1|") + printf "[%03d/%03d] %s\n" ${cnt} ${#targets[@]} "${target_name}" + for fuzz_test in "${target}"/*; do + test_name=$(basename "${fuzz_test}") + printf " - %s\n" "${test_name}" + PENDING+=("${target_name} ${test_name}") + done + (( ++cnt )) +done + +# print settings +echo +echo "Total jobs: ${#PENDING[@]}" +echo "Total workers: ${#SERIALS[@]}" +echo -n "Rotation: "; (( TIMEOUT )) && echo "${TIMEOUT} min" || echo "no" +echo -n "Publish: "; (( PUBLISH )) && echo "yes" || echo "no" +echo + +# set trap to properly stop workers before exiting +trap "echo '** Terminating...'; stop_workers \${#SERIALS[@]}; exit" 1 2 3 15 + +# run the jobs +working_time=0 +while (( ${#PENDING[@]} > 0 )) || (( ${#PIDS_BY_WORKERS[@]} > 0 )); do + # run all available workers + if (( ${#PENDING[@]} > 0 )); then + workers=($(get_available_workers)) + echo "** Starting ${#workers[@]} workers:" + for worker in "${workers[@]}"; do + start_worker "${worker}" + echo + done + fi + + # execution cycle + while true; do + sleep ${SLEEP_TIMEOUT} + (( working_time += SLEEP_TIMEOUT )) + + # some fuzzer crashed + finished=($(get_finished_workers)) + if (( ${#finished[@]} > 0 )); then + echo "** Some jobs are finished:" + working_time=0 + generate_reports "${finished[@]}" + free_workers_with_jobs "${finished[@]}" + echo + break + fi + + # jobs rotation + if (( TIMEOUT > 0 )) \ + && (( working_time >= (TIMEOUT * 60) )) \ + && (( ${#PENDING[@]} > 0 )); then + echo "** Rotating jobs:" + working_time=0 + stop_workers ${#PENDING[@]} + echo + break + fi + done +done + +echo "** All jobs are finished!" diff --git a/infra/tizen_fuzz.sh b/infra/tizen_fuzz.sh index c7ccbb3..19185c9 100755 --- a/infra/tizen_fuzz.sh +++ b/infra/tizen_fuzz.sh @@ -91,7 +91,7 @@ FUZZING_DIR="/fuzzing" export_all case ${1} in - 'build'|'build-all'|'list'|'publish'|'report'|'run') + 'build'|'build-all'|'list'|'publish'|'report'|'run'|'run-all') "${COMMANDS_DIR}/${1}.sh" ${@:2} ;; *) diff --git a/infra/utils.sh b/infra/utils.sh index d651672..f974d06 100644 --- a/infra/utils.sh +++ b/infra/utils.sh @@ -196,6 +196,19 @@ function prepare_device { sdb_shell mount -o rw,remount / } +# Checks if the element is present in the array. +# Args: +# 1) element to search for +# 2-..) array to search in +function contains_element () { + for e in "${@:2}"; do + if [[ "$e" = "$1" ]]; then + return 0 + fi + done + return 1 +} + # Exports all current environment function export_all { export $( (set -o posix ; set) | awk -F "=" 'BEGIN{ORS=" "}1 $1~/[a-zA-Z]/ {print $1}' ) @@ -213,5 +226,6 @@ function export_all { export -f check_deps export -f check_and_install_deps export -f prepare_device + export -f contains_element export -f export_all } -- 2.7.4