Fixing missing cpp tests for Caffe2 setup.py builds (#16037)

Summary:
These were broken (always skipped in setup.py builds) by https://github.com/pytorch/pytorch/pull/15917
Pull Request resolved: https://github.com/pytorch/pytorch/pull/16037

Differential Revision: D13675549

Pulled By: pjh5

fbshipit-source-id: fed50855dd0b5d0c80fface3d8b2156f18aae4e7

diff --git a/.jenkins/caffe2/bench.sh b/.jenkins/caffe2/bench.sh
index 03720a7..d4064c5 100755 (executable)
--- a/.jenkins/caffe2/bench.sh
+++ b/.jenkins/caffe2/bench.sh
@@ -2,7 +2,8 @@
 
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
-# Anywhere except $ROOT_DIR should work
+# Anywhere except $ROOT_DIR should work. This is so the python import doesn't
+# get confused by any 'caffe2' directory in cwd
 cd "$INSTALL_PREFIX"
 
 if [[ $BUILD_ENVIRONMENT == *-cuda* ]]; then
@@ -13,7 +14,7 @@ else
     num_gpus=0
 fi
 
-caffe2_pypath="$(python -c 'import os; import caffe2; print(os.path.dirname(os.path.realpath(caffe2.__file__)))')"
+caffe2_pypath="$(cd /usr && python -c 'import os; import caffe2; print(os.path.dirname(os.path.realpath(caffe2.__file__)))')"
 cmd="$PYTHON $caffe2_pypath/python/examples/resnet50_trainer.py --train_data null --batch_size 64 --epoch_size 6400 --num_epochs 2"
 if (( $num_gpus == 0 )); then
     cmd="$cmd --use_cpu"

diff --git a/.jenkins/caffe2/build.sh b/.jenkins/caffe2/build.sh
index b5efd64..228ec90 100755 (executable)
--- a/.jenkins/caffe2/build.sh
+++ b/.jenkins/caffe2/build.sh
@@ -234,7 +234,7 @@ if [[ "$BUILD_ENVIRONMENT" == *cmake* ]]; then
   # This is to save test binaries for testing
   mv "$INSTALL_PREFIX/test/" "$INSTALL_PREFIX/cpp_test/"
 
-  ls $INSTALL_PREFIX
+  ls -lah $INSTALL_PREFIX
 
 else
   # Python build. Uses setup.py to install into site-packages
@@ -249,6 +249,9 @@ else
   build_args+=("USE_FBGEMM=OFF")
   build_args+=("USE_MKLDNN=OFF")
   build_args+=("USE_DISTRIBUTED=ON")
+  for build_arg in "${build_args[@]}"; do
+    export $build_arg
+  done
 
   # sccache will be stuck if  all cores are used for compiling
   # see https://github.com/pytorch/pytorch/pull/7361
@@ -256,21 +259,8 @@ else
     export MAX_JOBS=`expr $(nproc) - 1`
   fi
 
-  for build_arg in "${build_args[@]}"; do
-    export $build_arg
-  done
   $PYTHON setup.py install --user
 
-  # This is to save test binaries for testing. Copying caffe2/test to
-  # INSTALL_PREFIX, which is /usr/local/caffe2/, enables these setup.py builds
-  # to share cpp-tests test-code with the cmake-only build above. In test.sh
-  # the cpp tests are run in install_prefix
-  cp -r torch/lib/tmp_install $INSTALL_PREFIX
-  mkdir -p "$INSTALL_PREFIX/cpp_test/"
-  cp -r caffe2/test/* "$INSTALL_PREFIX/cpp_test/"
-
-  ls $INSTALL_PREFIX
-
   report_compile_cache_stats
 fi
 

diff --git a/.jenkins/caffe2/common.sh b/.jenkins/caffe2/common.sh
index 7704c11..eeb9da0 100644 (file)
--- a/.jenkins/caffe2/common.sh
+++ b/.jenkins/caffe2/common.sh
@@ -16,3 +16,7 @@ fi
 # builds. In +python builds the cpp tests are copied to /usr/local/caffe2 so
 # that the test code in .jenkins/test.sh is the same
 INSTALL_PREFIX="/usr/local/caffe2"
+
+mkdir -p "$gtest_reports_dir" || true
+mkdir -p "$pytest_reports_dir" || true
+mkdir -p "$INSTALL_PREFIX" || true

diff --git a/.jenkins/caffe2/test.sh b/.jenkins/caffe2/test.sh
index 66207c5..321eb2c 100755 (executable)
--- a/.jenkins/caffe2/test.sh
+++ b/.jenkins/caffe2/test.sh
@@ -8,18 +8,26 @@ if [[ "${BUILD_ENVIRONMENT}" == *-android* ]]; then
   exit 0
 fi
 
-rm -rf "$TEST_DIR" && mkdir -p "$TEST_DIR"
-
-cd "${WORKSPACE}"
+# Find where cpp tests and Caffe2 itself are installed
+if [[ "$BUILD_ENVIRONMENT" == *cmake* ]]; then
+  # For cmake only build we install everything into /usr/local
+  cpp_test_dir="$INSTALL_PREFIX/cpp_test"
+  ld_library_path="$INSTALL_PREFIX/lib"
+else
+  # For Python builds we install into python
+  # cd to /usr first so the python import doesn't get confused by any 'caffe2'
+  # directory in cwd
+  python_installation="$(dirname $(dirname $(cd /usr && python -c 'import os; import caffe2; print(os.path.realpath(caffe2.__file__))')))"
+  caffe2_pypath="$python_installation/caffe2"
+  cpp_test_dir="$python_installation/caffe2/cpp_test"
+  ld_library_path="$python_installation/torch/lib"
+fi
 
-#############
+################################################################################
 # C++ tests #
-#############
-
+################################################################################
 echo "Running C++ tests.."
-export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${INSTALL_PREFIX}/lib"
-mkdir -p "$gtest_reports_dir"
-for test in $(find "${INSTALL_PREFIX}/cpp_test" -executable -type f); do
+for test in $(find "$cpp_test_dir" -executable -type f); do
   case "$test" in
     # skip tests we know are hanging or bad
     */mkl_utils_test|*/aten/integer_divider_test)
@@ -29,7 +37,7 @@ for test in $(find "${INSTALL_PREFIX}/cpp_test" -executable -type f); do
       if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then
         continue
       else
-        "$test"
+        LD_LIBRARY_PATH="$ld_library_path" "$test"
       fi
       ;;
     *)
@@ -43,7 +51,8 @@ for test in $(find "${INSTALL_PREFIX}/cpp_test" -executable -type f); do
       # output than it is to have XML output for Jenkins.
       # Note: in the future, if we want to use xml test reporter once we switch
       # to all gtest, one can simply do:
-      "$test" --gtest_output=xml:"$gtest_reports_dir/$(basename $test).xml"
+      LD_LIBRARY_PATH="$ld_library_path" \
+          "$test" --gtest_output=xml:"$gtest_reports_dir/$(basename $test).xml"
       ;;
   esac
 done
@@ -55,10 +64,6 @@ if [[ "$BUILD_ENVIRONMENT" == *cmake* ]]; then
   exit 0
 fi
 
-# Anywhere except $ROOT_DIR should work
-cd "$INSTALL_PREFIX"
-caffe2_pypath="$(python -c 'import os; import caffe2; print(os.path.dirname(os.path.realpath(caffe2.__file__)))')"
-
 if [[ "$BUILD_ENVIRONMENT" == *ubuntu14.04* ]]; then
   # Hotfix, use hypothesis 3.44.6 on Ubuntu 14.04
   # See comments on
@@ -73,8 +78,6 @@ else
   pip install --user --no-cache-dir hypothesis==3.59.0
 fi
 
-mkdir -p "$pytest_reports_dir"
-
 # Collect additional tests to run (outside caffe2/python)
 EXTRA_TESTS=()
 
@@ -119,7 +122,6 @@ pip install --user pytest-sugar
 #####################
 # torchvision tests #
 #####################
-
 if [[ "$BUILD_ENVIRONMENT" == *onnx* ]]; then
   pip install --user torchvision
   "$ROOT_DIR/scripts/onnx/test.sh"