From 9e76a75fb2f75f0c7828a095416b576eb71700b4 Mon Sep 17 00:00:00 2001
From: =?utf8?q?=EC=98=A4=ED=98=95=EC=84=9D/On-Device=20Lab=28SR=29/Staff?=
 =?utf8?q?=20Engineer/=EC=82=BC=EC=84=B1=EC=A0=84=EC=9E=90?=
 <hseok82.oh@samsung.com>
Date: Fri, 25 Oct 2019 16:53:28 +0900
Subject: [PATCH] [CI] Infra scripts for nncc package and nnpkg test (#8471)

CI infra scripts
- nncc build and resouce collection scripts
- Update build_nnpkg.sh to install nncc package
  - Update nncc-tc-to-nnpk-tc.sh to handle pb file and use tf2nnpkg

Signed-off-by: Hyeongseok Oh <hseok82.oh@samsung.com>
---
 infra/scripts/build_nnpkg.sh                       | 21 ++++++++-
 infra/scripts/docker_build_nncc.sh                 | 55 ++++++++++++++++++++++
 infra/scripts/docker_collect_nnpkg_resources.sh    | 43 +++++++++++++++++
 .../nncc-tc-to-nnpkg-tc/nncc-tc-to-nnpkg-tc.sh     |  9 +++-
 4 files changed, 125 insertions(+), 3 deletions(-)
 create mode 100755 infra/scripts/docker_build_nncc.sh
 create mode 100755 infra/scripts/docker_collect_nnpkg_resources.sh

diff --git a/infra/scripts/build_nnpkg.sh b/infra/scripts/build_nnpkg.sh
index c64d873..d75c62f 100755
--- a/infra/scripts/build_nnpkg.sh
+++ b/infra/scripts/build_nnpkg.sh
@@ -1,5 +1,8 @@
 #!/bin/bash
 
+# Resource: test-resources.tar.gz
+# nncc package: nncc-package.tar.gz
+
 # Test suite: nnpkg-test-suite.tar.gz
 
 [[ "${BASH_SOURCE[0]}" != "${0}" ]] && echo "Please don't source ${BASH_SOURCE[0]}, execute it" && return
@@ -7,15 +10,29 @@
 CURRENT_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 ROOT_PATH="${CURRENT_PATH}/../../"
 
+NNAS_WORKSPACE=${NNAS_WORKSPACE:-build}
+if [[ -z "${ARCHIVE_PATH}" ]]; then
+  ARCHIVE_PATH=${NNAS_WORKSPACE}/archive
+  echo "Default archive directory including nncc package and resources: ${ARCHIVE_PATH}"
+fi
+
 set -e
 
 pushd ${ROOT_PATH} > /dev/null
 
-for f in `find build/compiler/tf2tflite -name "UNIT*" | cut -d'.' -f1 | sort | uniq`;
+RESOURCE_PATH=${NNAS_WORKSPACE}/tfmodel
+mkdir -p bin/nncc
+mkdir -p ${RESOURCE_PATH}
+tar -zxf ${ARCHIVE_PATH}/nncc-package.tar.gz -C bin/nncc
+tar -zxf ${ARCHIVE_PATH}/test-resources.tar.gz -C ${RESOURCE_PATH}
+
+export PATH=${PATH}:${PWD}/bin/nncc/bin
+
+for f in `find ${RESOURCE_PATH} -name "UNIT*.pb" | cut -d'.' -f1 | sort | uniq`;
 do
   tools/nnpackage_tool/nncc-tc-to-nnpkg-tc/nncc-tc-to-nnpkg-tc.sh -o nnpkg-tcs -i ${f%/*} $(basename $f);
 done
 
-tar -zcf nnpkg-test-suite.tar.gz tools/nnpackage_tool/nnpkg_test nnpkg-tcs
+tar -zcf ${ARCHIVE_PATH}/nnpkg-test-suite.tar.gz nnpkg-tcs
 
 popd > /dev/null
diff --git a/infra/scripts/docker_build_nncc.sh b/infra/scripts/docker_build_nncc.sh
new file mode 100755
index 0000000..df67db4
--- /dev/null
+++ b/infra/scripts/docker_build_nncc.sh
@@ -0,0 +1,55 @@
+#!/bin/bash
+
+[[ "${BASH_SOURCE[0]}" != "${0}" ]] && echo "Please don't source ${BASH_SOURCE[0]}, execute it" && return
+
+CURRENT_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_PATH="$CURRENT_PATH/../../"
+
+CONFIG_OPTIONS=""
+# mount volume (or directory) for externals
+if [ -n "$EXTERNAL_VOLUME" ]; then
+  DOCKER_OPTS+=" -v $EXTERNAL_VOLUME:/externals"
+  CONFIG_OPTIONS+=" -DNNAS_EXTERNALS_DIR=/externals"
+else
+  echo "It will use default external path"
+fi
+
+# prepare tensorflow
+if [ -d $TENSORFLOW_PREFIX ]; then
+  DOCKER_OPTS+=" -v $TENSORFLOW_PREFIX:/opt/tensorflow"
+  CONFIG_OPTIONS+=" -DTENSORFLOW_PREFIX=/opt/tensorflow"
+fi
+
+# prepare onnx
+if [ -d $ONNXRUNTIME_PREFIX ]; then
+  DOCKER_OPTS+=" -v $ONNXRUNTIME_PREFIX:/opt/onnxruntime"
+  CONFIG_OPTIONS+=" -DONNXRUNTIME_PREFIX=/opt/onnxruntime"
+fi
+
+# docker image name
+if [[ -z $DOCKER_IMAGE_NAME ]]; then
+  echo "It will use default docker image name"
+fi
+
+NNAS_WORKSPACE=${NNAS_WORKSPACE:-build}
+export NNCC_WORKSPACE=${NNAS_WORKSPACE}/nncc
+export DOCKER_OPTS
+if [[ -z "${ARCHIVE_PATH}" ]]; then
+  ARCHIVE_PATH=${NNAS_WORKSPACE}/archive
+fi
+
+set -e
+
+pushd $ROOT_PATH > /dev/null
+
+./nncc docker-run ./nncc configure -DENABLE_STRICT_BUILD=ON ${CONFIG_OPTIONS} \
+  -DCMAKE_INSTALL_PREFIX="${PWD}/${NNAS_WORKSPACE}/out"
+./nncc docker-run ./nncc build -j4
+
+export DOCKER_ENV_VARS+=" -e NNAS_BUILD_PREFIX=${NNAS_WORKSPACE}"
+./nncc docker-run ./nnas create-package --skipbuild
+
+mkdir -p ${ARCHIVE_PATH}
+tar -zcf ${ARCHIVE_PATH}/nncc-package.tar.gz -C ${NNAS_WORKSPACE}/out ./
+
+popd > /dev/null
diff --git a/infra/scripts/docker_collect_nnpkg_resources.sh b/infra/scripts/docker_collect_nnpkg_resources.sh
new file mode 100755
index 0000000..5991e7f
--- /dev/null
+++ b/infra/scripts/docker_collect_nnpkg_resources.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+[[ "${BASH_SOURCE[0]}" != "${0}" ]] && echo "Please don't source ${BASH_SOURCE[0]}, execute it" && return
+
+CURRENT_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_PATH="$CURRENT_PATH/../../"
+
+# prepare tensorflow
+if [ -d $TENSORFLOW_PREFIX ]; then
+  DOCKER_OPTS+=" -v $TENSORFLOW_PREFIX:/opt/tensorflow"
+fi
+
+# Assume that build is already finished, and ready to test
+NNAS_WORKSPACE=${NNAS_WORKSPACE:-build}
+export NNCC_WORKSPACE=${NNAS_WORKSPACE}/nncc
+export DOCKER_OPTS
+if [ ! -d $NNCC_WORKSPACE ]; then
+  echo "Cannot find build directory $NNCC_WORKSPACE"
+  exit 1
+fi
+
+if [[ -z "${ARCHIVE_PATH}" ]]; then
+  ARCHIVE_PATH=${NNAS_WORKSPACE}/archive
+fi
+
+set -e
+
+pushd $ROOT_PATH > /dev/null
+
+# tf2tflite_test to generate graphdef, input, golden data
+CMD="cd $NNCC_WORKSPACE && \
+     ctest -R ^tf2tflite_test$"
+./nncc docker-run bash -c "${CMD}"
+
+mkdir -p ${ARCHIVE_PATH}
+TEMP_DIR=$(mktemp -d -t resXXXX)
+rm -f ${TEMP_DIR}/*
+cp $NNCC_WORKSPACE/compiler/tf2tflite/UNIT* ${TEMP_DIR}
+tar -zcf ${ARCHIVE_PATH}/test-resources.tar.gz -C ${TEMP_DIR} --exclude=*.tflite ./
+rm -rf ${TEMP_DIR}
+
+echo "resouce generation end"
+popd > /dev/null
diff --git a/tools/nnpackage_tool/nncc-tc-to-nnpkg-tc/nncc-tc-to-nnpkg-tc.sh b/tools/nnpackage_tool/nncc-tc-to-nnpkg-tc/nncc-tc-to-nnpkg-tc.sh
index 01ce147..cf3e544 100755
--- a/tools/nnpackage_tool/nncc-tc-to-nnpkg-tc/nncc-tc-to-nnpkg-tc.sh
+++ b/tools/nnpackage_tool/nncc-tc-to-nnpkg-tc/nncc-tc-to-nnpkg-tc.sh
@@ -5,6 +5,8 @@ set -eu
 progname=$(basename "${BASH_SOURCE[0]}")
 script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 model2nnpkg=${model2nnpkg:-"$script_dir"/../model2nnpkg/model2nnpkg.sh}
+# Need to install nncc package & set path to tf2nnpkg
+tf2nnpkg=$(which tf2nnpkg)
 
 indir="."
 outdir="."
@@ -54,6 +56,7 @@ fi
 tcname=$1
 
 supported_model_types="
+pb
 circle
 tflite
 "
@@ -69,7 +72,11 @@ if [[ "$model_type" == "" ]]; then
   exit 1
 fi
 
-$model2nnpkg -o "$outdir" "$indir/$tcname"."$model_type"
+if [[ "$model_type" == "pb" ]]; then
+  $tf2nnpkg --info "$indir/$tcname".info --graphdef "$indir/$tcname"."$model_type" -o "$outdir"
+else
+  $model2nnpkg -o "$outdir" "$indir/$tcname"."$model_type"
+fi
 
 extensions="
 expected.h5
-- 
2.7.4