From 7c3cd0842a41aac47069dcf14567b88c32ea7b28 Mon Sep 17 00:00:00 2001 From: Austin Anderson Date: Mon, 21 May 2018 17:45:22 -0700 Subject: [PATCH] Split generated_examples test into multiple test targets PiperOrigin-RevId: 197490872 --- tensorflow/contrib/lite/build_def.bzl | 110 +++++++++++++--- tensorflow/contrib/lite/testing/BUILD | 142 +++++++-------------- .../lite/testing/generated_examples_zip_test.cc | 79 ++---------- .../tools/ci_build/ci_parameterized_build.sh | 1 - .../tools/ci_build/linux/cpu/run_py3_contrib.sh | 2 +- 5 files changed, 153 insertions(+), 181 deletions(-) diff --git a/tensorflow/contrib/lite/build_def.bzl b/tensorflow/contrib/lite/build_def.bzl index 8521677..9bfc0a0 100644 --- a/tensorflow/contrib/lite/build_def.bzl +++ b/tensorflow/contrib/lite/build_def.bzl @@ -1,4 +1,8 @@ """Generate Flatbuffer binary from json.""" +load( + "//tensorflow:tensorflow.bzl", + "tf_cc_test", +) def tflite_copts(): """Defines compile time flags.""" @@ -185,32 +189,102 @@ def json_to_tflite(name, src, out): tools = [flatc], ) -def gen_zipped_test_files(name, files): +# This is the master list of generated examples that will be made into tests. A +# function called make_XXX_tests() must also appear in generate_examples.py. +# Disable a test by commenting it out. If you do, add a link to a bug or issue. +def generated_test_models(): + return [ + "add", + "arg_max", + "avg_pool", + "batch_to_space_nd", + "concat", + "constant", + "control_dep", + "conv", + "depthwiseconv", + "div", + "exp", + "floor", + "fully_connected", + "fused_batch_norm", + "gather", + "global_batch_norm", + "greater", + "greater_equal", + "l2_pool", + "l2norm", + "less", + "less_equal", + "local_response_norm", + "log_softmax", + "max_pool", + "maximum", + "mean", + "minimum", + "mul", + "neg", + "pad", + "padv2", + # "prelu", + "relu", + "relu1", + "relu6", + "reshape", + "resize_bilinear", + "sigmoid", + "sin", + "slice", + "softmax", + "space_to_batch_nd", + "space_to_depth", + "split", + "squeeze", + "strided_slice", + "strided_slice_1d_exhaustive", + "sub", + "topk", + "transpose", + "transpose_conv", + "where", + ] + +def gen_zip_test(name, test_name, **kwargs): + """Generate a zipped-example test and its dependent zip files. + + Args: + name: Resulting cc_test target name + test_name: Test targets this model. Comes from the list above. + **kwargs: tf_cc_test kwargs. + """ + gen_zipped_test_file( + name = "zip_%s" % test_name, + file = "%s.zip" % test_name, + ) + tf_cc_test(name, **kwargs) + +def gen_zipped_test_file(name, file): """Generate a zip file of tests by using :generate_examples. Args: - name: Name of output. We will produce "`name`_files" as a target. - files: A list of zip file basenames. + name: Name of output. We will produce "`file`.files" as a target. + file: The name of one of the generated_examples targets, e.g. "transpose" """ toco = "//tensorflow/contrib/lite/toco:toco" - out_files = [] - for f in files: - out_file = name + "/" + f - out_files.append(out_file) - native.genrule( - name = name + "_" + f + ".files", - cmd = ("$(locations :generate_examples) --toco $(locations %s) " % toco - + " --zip_to_output " + f + " $(@D)"), - outs = [out_file], - tools = [ - ":generate_examples", - toco, - ], - ) + native.genrule( + name = file + ".files", + cmd = ("$(locations :generate_examples) --toco $(locations %s) " % toco + + " --zip_to_output " + file + " $(@D)"), + outs = [file], + tools = [ + ":generate_examples", + toco, + ], + ) native.filegroup( name = name, - srcs = out_files, + srcs = [file], ) def gen_selected_ops(name, model): diff --git a/tensorflow/contrib/lite/testing/BUILD b/tensorflow/contrib/lite/testing/BUILD index 480685f..74fc32a 100644 --- a/tensorflow/contrib/lite/testing/BUILD +++ b/tensorflow/contrib/lite/testing/BUILD @@ -6,7 +6,8 @@ licenses(["notice"]) # Apache 2.0 load( "//tensorflow/contrib/lite:build_def.bzl", - "gen_zipped_test_files", + "gen_zip_test", + "generated_test_models", ) load("//tensorflow/contrib/lite:special_rules.bzl", "tflite_portable_test_suite") load( @@ -14,61 +15,52 @@ load( "tf_cc_test", ) -gen_zipped_test_files( - name = "optest", - files = [ - "add.zip", - "arg_max.zip", - "avg_pool.zip", - "batch_to_space_nd.zip", - "concat.zip", - "constant.zip", - "control_dep.zip", - "conv.zip", - "depthwiseconv.zip", - "div.zip", - "exp.zip", - "floor.zip", - "fully_connected.zip", - "fused_batch_norm.zip", - "gather.zip", - "global_batch_norm.zip", - "greater.zip", - "greater_equal.zip", - "l2_pool.zip", - "l2norm.zip", - "less.zip", - "less_equal.zip", - "local_response_norm.zip", - "log_softmax.zip", - "max_pool.zip", - "maximum.zip", - "mean.zip", - "minimum.zip", - "mul.zip", - "neg.zip", - "pad.zip", - "padv2.zip", - "relu.zip", - "relu1.zip", - "relu6.zip", - "reshape.zip", - "resize_bilinear.zip", - "sigmoid.zip", - "sin.zip", - "slice.zip", - "softmax.zip", - "space_to_batch_nd.zip", - "space_to_depth.zip", - "split.zip", - "squeeze.zip", - "strided_slice.zip", - "strided_slice_1d_exhaustive.zip", - "sub.zip", - "topk.zip", - "transpose.zip", - "transpose_conv.zip", - "where.zip", +[gen_zip_test( + name = "zip_test_%s" % test_name, + size = "large", + srcs = ["generated_examples_zip_test.cc"], + args = [ + "--zip_file_path=$(location :zip_%s)" % test_name, + # TODO(angerson) We may be able to add an external unzip binary instead + # of relying on an existing one for OSS builds. + "--unzip_binary_path=/usr/bin/unzip", + ], + data = [ + ":zip_%s" % test_name, + ], + shard_count = 20, + tags = [ + "gen_zip_test", + "no_oss", + "tflite_not_portable", + ], + test_name = test_name, + deps = [ + ":parse_testdata_lib", + ":tflite_driver", + ":util", + "@com_google_googletest//:gtest", + "@com_googlesource_code_re2//:re2", + "//tensorflow/contrib/lite:builtin_op_data", + "//tensorflow/contrib/lite:framework", + "//tensorflow/contrib/lite/kernels:builtin_ops", + ] + select({ + "//conditions:default": [ + "//tensorflow/core:framework_internal", + "//tensorflow/core:lib", + "//tensorflow/core:test", + ], + "//tensorflow:android": [ + "//tensorflow/core:android_tensorflow_lib", + "//tensorflow/core:android_tensorflow_test_lib", + ], + }), +) for test_name in generated_test_models()] + +test_suite( + name = "generated_zip_tests", + tags = [ + "gen_zip_test", ], ) @@ -353,42 +345,4 @@ cc_binary( ], ) -tf_cc_test( - name = "generated_examples_zip_test", - size = "large", - srcs = ["generated_examples_zip_test.cc"], - args = [ - "--zip_files_dir=tensorflow/contrib/lite/testing/optest", - # TODO(angerson) We may be able to add an external unzip binary instead - # of relying on an existing one for OSS builds. - "--unzip_binary_path=/usr/bin/unzip", - ], - data = [":optest"], - shard_count = 20, - tags = [ - "no_oss", - "tflite_not_portable", - ], - deps = [ - ":parse_testdata_lib", - ":tflite_driver", - ":util", - "@com_google_googletest//:gtest", - "@com_googlesource_code_re2//:re2", - "//tensorflow/contrib/lite:builtin_op_data", - "//tensorflow/contrib/lite:framework", - "//tensorflow/contrib/lite/kernels:builtin_ops", - ] + select({ - "//conditions:default": [ - "//tensorflow/core:framework_internal", - "//tensorflow/core:lib", - "//tensorflow/core:test", - ], - "//tensorflow:android": [ - "//tensorflow/core:android_tensorflow_lib", - "//tensorflow/core:android_tensorflow_test_lib", - ], - }), -) - tflite_portable_test_suite() diff --git a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc index 581535d..2f069ff 100644 --- a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc +++ b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc @@ -35,7 +35,7 @@ namespace { bool FLAGS_ignore_known_bugs = true; // TODO(b/71769302) zip_files_dir should have a more accurate default, if // possible -string* FLAGS_zip_files_dir = new string("./"); +string* FLAGS_zip_file_path = new string("./"); string* FLAGS_unzip_binary_path = new string("/usr/bin/unzip"); } // namespace @@ -137,7 +137,10 @@ class ZipEnvironment : public ::testing::Environment { *out_dir = dir; return tensorflow::Status::OK(); } else { - return tensorflow::Status(tensorflow::error::UNKNOWN, "unzip failed"); + return tensorflow::Status(tensorflow::error::UNKNOWN, + "unzip failed. " + "stdout:\n" + + out + "\nstderr:\n" + err); } } @@ -191,8 +194,7 @@ tensorflow::Status ReadManifest(const string& original_file, const string& dir, } // Get a list of tests from a zip file `zip_file_name`. -std::vector UnarchiveZipAndFindTestNames(const string& zip_file_name) { - string zip_file = *FLAGS_zip_files_dir + "/" + zip_file_name; +std::vector UnarchiveZipAndFindTestNames(const string& zip_file) { string decompress_tmp_dir; TF_CHECK_OK(zip_environment()->UnZip(zip_file, &decompress_tmp_dir)); std::vector stuff; @@ -251,67 +253,10 @@ struct ZipPathParamName { } }; -// Instantiate a test. This assumes `zip_base`.zip is a declared data file -// of this test. -#define INSTANTIATE_TESTS(zip_base) \ - INSTANTIATE_TEST_CASE_P( \ - zip_base, OpsTest, \ - ::testing::ValuesIn(UnarchiveZipAndFindTestNames(#zip_base ".zip")), \ - ZipPathParamName()); - -INSTANTIATE_TESTS(add) -INSTANTIATE_TESTS(arg_max) -INSTANTIATE_TESTS(avg_pool) -INSTANTIATE_TESTS(batch_to_space_nd) -INSTANTIATE_TESTS(concat) -INSTANTIATE_TESTS(constant) -INSTANTIATE_TESTS(control_dep) -INSTANTIATE_TESTS(conv) -INSTANTIATE_TESTS(depthwiseconv) -INSTANTIATE_TESTS(div) -INSTANTIATE_TESTS(exp) -INSTANTIATE_TESTS(floor) -INSTANTIATE_TESTS(fully_connected) -INSTANTIATE_TESTS(fused_batch_norm) -INSTANTIATE_TESTS(gather) -INSTANTIATE_TESTS(global_batch_norm) -INSTANTIATE_TESTS(greater) -INSTANTIATE_TESTS(greater_equal) -INSTANTIATE_TESTS(l2_pool) -INSTANTIATE_TESTS(l2norm) -INSTANTIATE_TESTS(less) -INSTANTIATE_TESTS(less_equal) -INSTANTIATE_TESTS(local_response_norm) -INSTANTIATE_TESTS(log_softmax) -INSTANTIATE_TESTS(max_pool) -INSTANTIATE_TESTS(maximum) -INSTANTIATE_TESTS(mean) -INSTANTIATE_TESTS(minimum) -INSTANTIATE_TESTS(mul) -INSTANTIATE_TESTS(neg) -INSTANTIATE_TESTS(pad) -INSTANTIATE_TESTS(padv2) -// INSTANTIATE_TESTS(prelu) -INSTANTIATE_TESTS(relu) -INSTANTIATE_TESTS(relu1) -INSTANTIATE_TESTS(relu6) -INSTANTIATE_TESTS(reshape) -INSTANTIATE_TESTS(resize_bilinear) -INSTANTIATE_TESTS(sigmoid) -INSTANTIATE_TESTS(sin) -INSTANTIATE_TESTS(slice) -INSTANTIATE_TESTS(softmax) -INSTANTIATE_TESTS(space_to_batch_nd) -INSTANTIATE_TESTS(space_to_depth) -INSTANTIATE_TESTS(split) -INSTANTIATE_TESTS(squeeze) -INSTANTIATE_TESTS(strided_slice) -INSTANTIATE_TESTS(strided_slice_1d_exhaustive) -INSTANTIATE_TESTS(sub) -INSTANTIATE_TESTS(topk) -INSTANTIATE_TESTS(transpose) -INSTANTIATE_TESTS(transpose_conv) -INSTANTIATE_TESTS(where) +INSTANTIATE_TEST_CASE_P( + tests, OpsTest, + ::testing::ValuesIn(UnarchiveZipAndFindTestNames(*FLAGS_zip_file_path)), + ZipPathParamName()); } // namespace testing } // namespace tflite @@ -324,8 +269,8 @@ int main(int argc, char** argv) { "ignore_known_bugs", &tflite::testing::FLAGS_ignore_known_bugs, "If a particular model is affected by a known bug, the " "corresponding test should expect the outputs to not match."), - tensorflow::Flag("zip_files_dir", tflite::testing::FLAGS_zip_files_dir, - "Required: Location of the test zips."), + tensorflow::Flag("zip_file_path", tflite::testing::FLAGS_zip_file_path, + "Required: Location of the test zip file."), tensorflow::Flag("unzip_binary_path", tflite::testing::FLAGS_unzip_binary_path, "Required: Location of a suitable unzip binary.")}; diff --git a/tensorflow/tools/ci_build/ci_parameterized_build.sh b/tensorflow/tools/ci_build/ci_parameterized_build.sh index 797e0a6..e621f85 100755 --- a/tensorflow/tools/ci_build/ci_parameterized_build.sh +++ b/tensorflow/tools/ci_build/ci_parameterized_build.sh @@ -168,7 +168,6 @@ else BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:embedding_lookup_test" BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:embedding_lookup_sparse_test" BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:fully_connected_test" - # BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/testing:generated_examples_zip_test" BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:hashtable_lookup_test" BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:local_response_norm_test" BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:lsh_projection_test" diff --git a/tensorflow/tools/ci_build/linux/cpu/run_py3_contrib.sh b/tensorflow/tools/ci_build/linux/cpu/run_py3_contrib.sh index 6d017c8..c798081 100755 --- a/tensorflow/tools/ci_build/linux/cpu/run_py3_contrib.sh +++ b/tensorflow/tools/ci_build/linux/cpu/run_py3_contrib.sh @@ -52,7 +52,7 @@ bazel test --test_tag_filters=-no_oss,-oss_serial,-gpu,-benchmark-test -k \ //tensorflow/contrib/lite/kernels:embedding_lookup_test \ //tensorflow/contrib/lite/kernels:embedding_lookup_sparse_test \ //tensorflow/contrib/lite/kernels:fully_connected_test \ - //tensorflow/contrib/lite/testing:generated_examples_zip_test \ + //tensorflow/contrib/lite/testing:generated_zip_tests \ //tensorflow/contrib/lite/kernels:hashtable_lookup_test \ //tensorflow/contrib/lite/kernels:local_response_norm_test \ //tensorflow/contrib/lite/kernels:lsh_projection_test \ -- 2.7.4