infra/scripts/docker_collect_nnpkg_resources.sh

   1 #!/bin/bash
   2
   3 function join_by
   4 {
   5   local IFS="$1"; shift; echo "$*"
   6 }
   7
   8 [[ "${BASH_SOURCE[0]}" != "${0}" ]] && echo "Please don't source ${BASH_SOURCE[0]}, execute it" && return
   9
  10 CURRENT_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
  11 ROOT_PATH="$CURRENT_PATH/../../"
  12
  13 CONFIG_OPTIONS=""
  14 # mount volume (or directory) for externals
  15 if [ -n "$EXTERNAL_VOLUME" ]; then
  16   DOCKER_OPTS+=" -v $EXTERNAL_VOLUME:/externals"
  17   CONFIG_OPTIONS+=" -DNNAS_EXTERNALS_DIR=/externals"
  18 else
  19   echo "It will use default external path"
  20 fi
  21
  22 # mount volume (or directory) for overlay
  23 if [ -n "$OVERLAY_VOLUME" ]; then
  24   DOCKER_OPTS+=" -v $OVERLAY_VOLUME:/overlay"
  25   CONFIG_OPTIONS+=" -DNNCC_OVERLAY_DIR=/overlay"
  26 else
  27   echo "It will use default overlay path"
  28 fi
  29
  30 # prepare tensorflow
  31 if [ -n "$TENSORFLOW_PREFIX" ]; then
  32   DOCKER_OPTS+=" -v $TENSORFLOW_PREFIX:/opt/tensorflow"
  33   CONFIG_OPTIONS+=" -DTENSORFLOW_PREFIX=/opt/tensorflow"
  34 fi
  35
  36 # prepare onnx
  37 if [ -n "$ONNXRUNTIME_PREFIX" ]; then
  38   DOCKER_OPTS+=" -v $ONNXRUNTIME_PREFIX:/opt/onnxruntime"
  39   CONFIG_OPTIONS+=" -DONNXRUNTIME_PREFIX=/opt/onnxruntime"
  40 fi
  41
  42 # docker image name
  43 # - for bionic, use DOCKER_IMAGE_NAME="nnfw/one-devtools:bionic"
  44 # - for focal, use DOCKER_IMAGE_NAME="nnfw/one-devtools:focal"
  45 if [[ -z $DOCKER_IMAGE_NAME ]]; then
  46   echo "It will use default docker image name"
  47 fi
  48
  49 # Assume that build is already finished, and ready to test
  50 NNAS_WORKSPACE=${NNAS_WORKSPACE:-build}
  51 export NNCC_WORKSPACE=${NNAS_WORKSPACE}/nncc
  52 export DOCKER_OPTS
  53
  54 if [[ -z "${ARCHIVE_PATH}" ]]; then
  55   ARCHIVE_PATH=${NNAS_WORKSPACE}/archive
  56 fi
  57
  58 set -e
  59
  60 pushd $ROOT_PATH > /dev/null
  61
  62 REQUIRED_UNITS=()
  63 # Common Libraries
  64 REQUIRED_UNITS+=("angkor" "cwrap" "pepper-str" "pepper-strcast" "pp")
  65 REQUIRED_UNITS+=("pepper-csv2vec")
  66 REQUIRED_UNITS+=("oops" "safemain" "foder" "crew" "arser" "vconone")
  67 # Hermes Logging Framework
  68 REQUIRED_UNITS+=("hermes" "hermes-std")
  69 # loco IR and related utilities
  70 REQUIRED_UNITS+=("loco" "locop" "locomotiv" "logo-core" "logo")
  71 # Compute
  72 REQUIRED_UNITS+=("luci-compute")
  73 # Circle compiler library (.circle -> .circle)
  74 REQUIRED_UNITS+=("luci")
  75 # Flatbuffer I/O
  76 REQUIRED_UNITS+=("mio-tflite280" "mio-circle05" "mio-tflite2121" "mio-circle06")
  77 # Tools
  78 REQUIRED_UNITS+=("tflite2circle" "circle2circle" "luci-interpreter")
  79 REQUIRED_UNITS+=("souschef" "tflchef" "circlechef" "circle-verify")
  80 # common-artifacts
  81 REQUIRED_UNITS+=("common-artifacts")
  82
  83 # Reset whitelist to build all
  84 ./nncc docker-run ./nncc configure -DENABLE_STRICT_BUILD=ON -DCMAKE_BUILD_TYPE=release \
  85   -DBUILD_WHITELIST=$(join_by ";" "${REQUIRED_UNITS[@]}") \
  86   $CONFIG_OPTIONS
  87 ./nncc docker-run ./nncc build -j4
  88
  89 mkdir -p ${ARCHIVE_PATH}
  90 TEMP_DIR=$(mktemp -d -t resXXXX)
  91 rm -f ${TEMP_DIR}/*
  92 mkdir -p ${TEMP_DIR}/nnpkg-tcs
  93
  94 # Copy nnpakcage only if it has its test data
  95 for nnpkg in $NNCC_WORKSPACE/compiler/common-artifacts/*; do
  96   if [ -d $nnpkg/metadata/tc ]; then
  97     cp -r $nnpkg ${TEMP_DIR}/nnpkg-tcs
  98   fi
  99 done
 100
 101 tar -zcf ${ARCHIVE_PATH}/nnpkg-test-suite.tar.gz -C ${TEMP_DIR} ./
 102 rm -rf ${TEMP_DIR}
 103
 104 echo "resouce generation end"
 105 popd > /dev/null