From 991279dc7dd168fc7747bef5c2de1b91208872af Mon Sep 17 00:00:00 2001 From: Karl Ostmo Date: Mon, 15 Apr 2019 11:38:44 -0700 Subject: [PATCH] produce diagram for caffe2 build matrix (#18517) Summary: This PR splits the configuration tree data from the logic used to construct the tree, for both `pytorch` and `caffe2` build configs. Caffe2 configs are also now illustrated in a diagram. Pull Request resolved: https://github.com/pytorch/pytorch/pull/18517 Differential Revision: D14936170 Pulled By: kostmo fbshipit-source-id: 7b40a88512627377c5ea0f24765dabfef76ca279 --- .circleci/cimodel/data/binary_build_definitions.py | 29 ++-- .circleci/cimodel/data/caffe2_build_data.py | 111 +++++++++++++++ .circleci/cimodel/data/caffe2_build_definitions.py | 94 +++++-------- .circleci/cimodel/data/pytorch_build_data.py | 147 ++++++++++++++++++++ .../cimodel/data/pytorch_build_definitions.py | 149 +-------------------- .circleci/cimodel/lib/conf_tree.py | 7 + 6 files changed, 315 insertions(+), 222 deletions(-) create mode 100644 .circleci/cimodel/data/caffe2_build_data.py create mode 100644 .circleci/cimodel/data/pytorch_build_data.py diff --git a/.circleci/cimodel/data/binary_build_definitions.py b/.circleci/cimodel/data/binary_build_definitions.py index 161d6a9..55fed2d 100644 --- a/.circleci/cimodel/data/binary_build_definitions.py +++ b/.circleci/cimodel/data/binary_build_definitions.py @@ -108,10 +108,14 @@ def gen_build_env_list(smoke): return newlist -def add_build_entries(jobs_dict, phase, smoke): +def predicate_exclude_nonlinux_and_libtorch(config): + return config.os == "linux" and (config.smoke or config.pydistro != "libtorch") + + +def add_build_entries(jobs_dict, phase, smoke, filter_predicate=lambda x: True): configs = gen_build_env_list(smoke) - for conf_options in configs: + for conf_options in filter(filter_predicate, configs): jobs_dict[conf_options.gen_build_name(phase)] = conf_options.gen_yaml_tree(phase) @@ -119,6 +123,10 @@ def add_binary_build_specs(jobs_dict): add_build_entries(jobs_dict, "build", False) +def add_binary_build_tests(jobs_dict): + add_build_entries(jobs_dict, "test", False, predicate_exclude_nonlinux_and_libtorch) + + def add_binary_build_uploads(jobs_dict): add_build_entries(jobs_dict, "upload", False) @@ -160,19 +168,6 @@ def get_nightly_uploads(): return mylist -def predicate_exclude_nonlinux_and_libtorch(config): - return config.os == "linux" and (config.smoke or config.pydistro != "libtorch") - - -def add_binary_build_tests(jobs_dict): - - configs = gen_build_env_list(False) - filtered_configs = filter(predicate_exclude_nonlinux_and_libtorch, configs) - - for conf_options in filtered_configs: - jobs_dict[conf_options.gen_build_name("test")] = conf_options.gen_yaml_tree("test") - - def gen_schedule_tree(cron_timing): return [{ "schedule": { @@ -195,13 +190,11 @@ def add_jobs_and_render(jobs_dict, toplevel_key, smoke, cron_schedule): build_name = build_config.gen_build_name("build") jobs_list.append(build_name) - d = OrderedDict( + jobs_dict[toplevel_key] = OrderedDict( triggers=gen_schedule_tree(cron_schedule), jobs=jobs_list, ) - jobs_dict[toplevel_key] = d - graph = visualization.generate_graph(get_root(smoke, toplevel_key)) graph.draw(toplevel_key + "-config-dimensions.png", prog="twopi") diff --git a/.circleci/cimodel/data/caffe2_build_data.py b/.circleci/cimodel/data/caffe2_build_data.py new file mode 100644 index 0000000..446f61a --- /dev/null +++ b/.circleci/cimodel/data/caffe2_build_data.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 + +from cimodel.lib.conf_tree import ConfigNode, X +from cimodel.lib.conf_tree import Ver +import cimodel.data.dimensions as dimensions + + +CONFIG_TREE_DATA = [ + (Ver("ubuntu", "14.04"), [ + (Ver("gcc", "4.8"), [X("py2")]), + (Ver("gcc", "4.9"), [X("py2")]), + ]), + (Ver("ubuntu", "16.04"), [ + (Ver("cuda", "8.0"), [X("py2")]), + (Ver("cuda", "9.0"), [ + # TODO make explicit that this is a "secret TensorRT build" + # (see https://github.com/pytorch/pytorch/pull/17323#discussion_r259446749) + X("py2"), + X("cmake"), + ]), + (Ver("cuda", "9.1"), [X("py2")]), + (Ver("mkl"), [X("py2")]), + (Ver("gcc", "5"), [X("onnx_py2")]), + (Ver("clang", "3.8"), [X("py2")]), + (Ver("clang", "3.9"), [X("py2")]), + (Ver("clang", "7"), [X("py2")]), + (Ver("android"), [X("py2")]), + ]), + (Ver("centos", "7"), [ + (Ver("cuda", "9.0"), [X("py2")]), + ]), + (Ver("macos", "10.13"), [ + # TODO ios and system aren't related. system qualifies where the python comes + # from (use the system python instead of homebrew or anaconda) + (Ver("ios"), [X("py2")]), + (Ver("system"), [X("py2")]), + ]), +] + + +class TreeConfigNode(ConfigNode): + def __init__(self, parent, node_name, subtree): + super(TreeConfigNode, self).__init__(parent, self.modify_label(node_name)) + self.subtree = subtree + self.init2(node_name) + + # noinspection PyMethodMayBeStatic + def modify_label(self, label): + return str(label) + + def init2(self, node_name): + pass + + def get_children(self): + return [self.child_constructor()(self, k, v) for (k, v) in self.subtree] + + def is_build_only(self): + return str(self.find_prop("compiler_version")) in [ + "gcc4.9", + "clang3.8", + "clang3.9", + "clang7", + "android", + ] or self.find_prop("distro_version").name == "macos" + + +class TopLevelNode(TreeConfigNode): + def __init__(self, node_name, subtree): + super(TopLevelNode, self).__init__(None, node_name, subtree) + + # noinspection PyMethodMayBeStatic + def child_constructor(self): + return DistroConfigNode + + +class DistroConfigNode(TreeConfigNode): + def init2(self, node_name): + self.props["distro_version"] = node_name + + # noinspection PyMethodMayBeStatic + def child_constructor(self): + return CompilerConfigNode + + +class CompilerConfigNode(TreeConfigNode): + def init2(self, node_name): + self.props["compiler_version"] = node_name + + # noinspection PyMethodMayBeStatic + def child_constructor(self): + return LanguageConfigNode + + +class LanguageConfigNode(TreeConfigNode): + def init2(self, node_name): + self.props["language_version"] = node_name + self.props["build_only"] = self.is_build_only() + + def get_children(self): + + children = [] + for phase in dimensions.PHASES: + if phase == "build" or not self.props["build_only"]: + children.append(PhaseConfigNode(self, phase, [])) + + return children + + +class PhaseConfigNode(TreeConfigNode): + def init2(self, node_name): + self.props["phase_name"] = node_name diff --git a/.circleci/cimodel/data/caffe2_build_definitions.py b/.circleci/cimodel/data/caffe2_build_definitions.py index 7c048da..5898cdb 100644 --- a/.circleci/cimodel/data/caffe2_build_definitions.py +++ b/.circleci/cimodel/data/caffe2_build_definitions.py @@ -2,9 +2,10 @@ from collections import OrderedDict -import cimodel.data.dimensions as dimensions +import cimodel.lib.conf_tree as conf_tree import cimodel.lib.miniutils as miniutils -from cimodel.lib.conf_tree import Ver +import cimodel.lib.visualization as visualization +from cimodel.data.caffe2_build_data import CONFIG_TREE_DATA, TopLevelNode DOCKER_IMAGE_PATH_BASE = "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/" @@ -12,55 +13,14 @@ DOCKER_IMAGE_PATH_BASE = "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/" DOCKER_IMAGE_VERSION = 266 -CONFIG_HIERARCHY = [ - (Ver("ubuntu", "14.04"), [ - (Ver("gcc", "4.8"), ["py2"]), - (Ver("gcc", "4.9"), ["py2"]), - ]), - (Ver("ubuntu", "16.04"), [ - (Ver("cuda", "8.0"), ["py2"]), - (Ver("cuda", "9.0"), [ - # TODO make explicit that this is a "secret TensorRT build" - # (see https://github.com/pytorch/pytorch/pull/17323#discussion_r259446749) - "py2", - "cmake", - ]), - (Ver("cuda", "9.1"), ["py2"]), - (Ver("mkl"), ["py2"]), - (Ver("gcc", "5"), ["onnx_py2"]), - (Ver("clang", "3.8"), ["py2"]), - (Ver("clang", "3.9"), ["py2"]), - (Ver("clang", "7"), ["py2"]), - (Ver("android"), ["py2"]), - ]), - (Ver("centos", "7"), [ - (Ver("cuda", "9.0"), ["py2"]), - ]), - (Ver("macos", "10.13"), [ - # TODO ios and system aren't related. system qualifies where the python comes - # from (use the system python instead of homebrew or anaconda) - (Ver("ios"), ["py2"]), - (Ver("system"), ["py2"]), - ]), -] - - class Conf(object): - def __init__(self, language, distro, compiler, phase): + def __init__(self, language, distro, compiler, phase, build_only): self.language = language self.distro = distro self.compiler = compiler self.phase = phase - - def is_build_only(self): - return str(self.compiler) in [ - "gcc4.9", - "clang3.8", - "clang3.9", - "clang7", - "android", - ] or self.get_platform() == "macos" + self.build_only = build_only # TODO: Eventually we can probably just remove the cudnn7 everywhere. def get_cudnn_insertion(self): @@ -88,7 +48,10 @@ class Conf(object): return self.construct_phase_name(self.phase) def get_platform(self): - return "macos" if self.distro.name == "macos" else "linux" + platform = self.distro.name + if self.distro.name != "macos": + platform = "linux" + return platform def gen_docker_image(self): @@ -135,7 +98,7 @@ class Conf(object): else: tuples.append(("DOCKER_IMAGE", self.gen_docker_image())) - if self.is_build_only(): + if self.build_only: tuples.append(("BUILD_ONLY", miniutils.quote("1"))) d = OrderedDict({"environment": OrderedDict(tuples)}) @@ -149,31 +112,44 @@ class Conf(object): return d -def gen_build_list(): - x = [] - for distro, d1 in CONFIG_HIERARCHY: - for compiler_name, build_languages in d1: - for language in build_languages: - for phase in dimensions.PHASES: +def get_root(): + return TopLevelNode("Caffe2 Builds", CONFIG_TREE_DATA) - c = Conf(language, distro, compiler_name, phase) - if phase == "build" or not c.is_build_only(): - x.append(c) +def instantiate_configs(): - return x + config_list = [] + + root = get_root() + found_configs = conf_tree.dfs(root) + for fc in found_configs: + + c = Conf( + fc.find_prop("language_version"), + fc.find_prop("distro_version"), + fc.find_prop("compiler_version"), + fc.find_prop("phase_name"), + fc.find_prop("build_only"), + ) + + config_list.append(c) + + return config_list def add_caffe2_builds(jobs_dict): - configs = gen_build_list() + configs = instantiate_configs() for conf_options in configs: jobs_dict[conf_options.get_name()] = conf_options.gen_yaml_tree() + graph = visualization.generate_graph(get_root()) + graph.draw("caffe2-config-dimensions.png", prog="twopi") + def get_caffe2_workflows(): - configs = gen_build_list() + configs = instantiate_configs() # TODO Why don't we build this config? # See https://github.com/pytorch/pytorch/pull/17323#discussion_r259450540 diff --git a/.circleci/cimodel/data/pytorch_build_data.py b/.circleci/cimodel/data/pytorch_build_data.py new file mode 100644 index 0000000..09476a9 --- /dev/null +++ b/.circleci/cimodel/data/pytorch_build_data.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 + +from cimodel.lib.conf_tree import ConfigNode, X + + +CONFIG_TREE_DATA = [ + ("trusty", [ + (None, [ + X("2.7.9"), + X("2.7"), + X("3.5"), + X("nightly"), + ]), + ("gcc", [ + ("4.8", [X("3.6")]), + ("5.4", [("3.6", [X(False), X(True)])]), + ("7", [X("3.6")]), + ]), + ]), + ("xenial", [ + ("clang", [ + ("5", [X("3.6")]), + ]), + ("cuda", [ + ("8", [X("3.6")]), + ("9", [ + # Note there are magic strings here + # https://github.com/pytorch/pytorch/blob/master/.jenkins/pytorch/build.sh#L21 + # and + # https://github.com/pytorch/pytorch/blob/master/.jenkins/pytorch/build.sh#L143 + # and + # https://github.com/pytorch/pytorch/blob/master/.jenkins/pytorch/build.sh#L153 + # (from https://github.com/pytorch/pytorch/pull/17323#discussion_r259453144) + X("2.7"), + X("3.6"), + ]), + ("9.2", [X("3.6")]), + ("10", [X("3.6")]), + ]), + ("android", [ + ("r19c", [X("3.6")]), + ]), + ]), +] + + +def get_major_pyver(dotted_version): + parts = dotted_version.split(".") + return "py" + parts[0] + + +class TreeConfigNode(ConfigNode): + def __init__(self, parent, node_name, subtree): + super(TreeConfigNode, self).__init__(parent, self.modify_label(node_name)) + self.subtree = subtree + self.init2(node_name) + + def modify_label(self, label): + return label + + def init2(self, node_name): + pass + + def get_children(self): + return [self.child_constructor()(self, k, v) for (k, v) in self.subtree] + + +class TopLevelNode(TreeConfigNode): + def __init__(self, node_name, subtree): + super(TopLevelNode, self).__init__(None, node_name, subtree) + + # noinspection PyMethodMayBeStatic + def child_constructor(self): + return DistroConfigNode + + +class DistroConfigNode(TreeConfigNode): + def init2(self, node_name): + self.props["distro_name"] = node_name + + def child_constructor(self): + distro = self.find_prop("distro_name") + + next_nodes = { + "trusty": TrustyCompilerConfigNode, + "xenial": XenialCompilerConfigNode, + } + return next_nodes[distro] + + +class TrustyCompilerConfigNode(TreeConfigNode): + + def modify_label(self, label): + return label or "" + + def init2(self, node_name): + self.props["compiler_name"] = node_name + + def child_constructor(self): + return TrustyCompilerVersionConfigNode if self.props["compiler_name"] else PyVerConfigNode + + +class TrustyCompilerVersionConfigNode(TreeConfigNode): + + def init2(self, node_name): + self.props["compiler_version"] = node_name + + # noinspection PyMethodMayBeStatic + def child_constructor(self): + return PyVerConfigNode + + +class PyVerConfigNode(TreeConfigNode): + def init2(self, node_name): + self.props["pyver"] = node_name + self.props["abbreviated_pyver"] = get_major_pyver(node_name) + + # noinspection PyMethodMayBeStatic + def child_constructor(self): + return XlaConfigNode + + +class XlaConfigNode(TreeConfigNode): + def modify_label(self, label): + return "XLA=" + str(label) + + def init2(self, node_name): + self.props["is_xla"] = node_name + + +class XenialCompilerConfigNode(TreeConfigNode): + + def init2(self, node_name): + self.props["compiler_name"] = node_name + + # noinspection PyMethodMayBeStatic + def child_constructor(self): + return XenialCompilerVersionConfigNode + + +class XenialCompilerVersionConfigNode(TreeConfigNode): + def init2(self, node_name): + self.props["compiler_version"] = node_name + + # noinspection PyMethodMayBeStatic + def child_constructor(self): + return PyVerConfigNode diff --git a/.circleci/cimodel/data/pytorch_build_definitions.py b/.circleci/cimodel/data/pytorch_build_definitions.py index a8e85d8..99cdf68 100644 --- a/.circleci/cimodel/data/pytorch_build_definitions.py +++ b/.circleci/cimodel/data/pytorch_build_definitions.py @@ -2,11 +2,11 @@ from collections import OrderedDict +from cimodel.data.pytorch_build_data import TopLevelNode, CONFIG_TREE_DATA import cimodel.data.dimensions as dimensions import cimodel.lib.conf_tree as conf_tree import cimodel.lib.miniutils as miniutils import cimodel.lib.visualization as visualization -from cimodel.lib.conf_tree import ConfigNode DOCKER_IMAGE_PATH_BASE = "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/" @@ -112,9 +112,9 @@ class Conf(object): # caffe2 test job dependent on a pytorch build job. This way we could quickly dedup the repeated # build of pytorch in the caffe2 build job, and just run the caffe2 tests off of a completed # pytorch build job (from https://github.com/pytorch/pytorch/pull/17323#discussion_r259452641) - if phase == "test": - dependency_build = self.parent_build or self - val["requires"] = [dependency_build.gen_build_name("build")] + + dependency_build = self.parent_build or self + val["requires"] = [dependency_build.gen_build_name("build")] return {self.gen_build_name(phase): val} else: @@ -167,59 +167,6 @@ def gen_dependent_configs(xenial_parent_config): return configs -def X(val): - """ - Compact way to write a leaf node - """ - return val, [] - - -CONFIG_TREE_DATA = [ - ("trusty", [ - (None, [ - X("2.7.9"), - X("2.7"), - X("3.5"), - X("nightly"), - ]), - ("gcc", [ - ("4.8", [X("3.6")]), - ("5.4", [("3.6", [X(False), X(True)])]), - ("7", [X("3.6")]), - ]), - ]), - ("xenial", [ - ("clang", [ - ("5", [X("3.6")]), - ]), - ("cuda", [ - ("8", [X("3.6")]), - ("9", [ - # Note there are magic strings here - # https://github.com/pytorch/pytorch/blob/master/.jenkins/pytorch/build.sh#L21 - # and - # https://github.com/pytorch/pytorch/blob/master/.jenkins/pytorch/build.sh#L143 - # and - # https://github.com/pytorch/pytorch/blob/master/.jenkins/pytorch/build.sh#L153 - # (from https://github.com/pytorch/pytorch/pull/17323#discussion_r259453144) - X("2.7"), - X("3.6"), - ]), - ("9.2", [X("3.6")]), - ("10", [X("3.6")]), - ]), - ("android", [ - ("r19c", [X("3.6")]), - ]), - ]), -] - - -def get_major_pyver(dotted_version): - parts = dotted_version.split(".") - return "py" + parts[0] - - def get_root(): return TopLevelNode("PyTorch Builds", CONFIG_TREE_DATA) @@ -230,94 +177,6 @@ def gen_tree(): return configs_list -class TreeConfigNode(ConfigNode): - def __init__(self, parent, node_name, subtree): - super(TreeConfigNode, self).__init__(parent, self.modify_label(node_name)) - self.subtree = subtree - self.init2(node_name) - - def modify_label(self, label): - return label - - def init2(self, node_name): - pass - - def get_children(self): - return [self.child_constructor()(self, k, v) for (k, v) in self.subtree] - - -class TopLevelNode(TreeConfigNode): - def __init__(self, node_name, subtree): - super(TopLevelNode, self).__init__(None, node_name, subtree) - - def child_constructor(self): - return DistroConfigNode - - -class DistroConfigNode(TreeConfigNode): - def init2(self, node_name): - self.props["distro_name"] = node_name - - def child_constructor(self): - distro = self.find_prop("distro_name") - return TrustyCompilerConfigNode if distro == "trusty" else XenialCompilerConfigNode - - -class TrustyCompilerConfigNode(TreeConfigNode): - - def modify_label(self, label): - return label or "" - - def init2(self, node_name): - self.props["compiler_name"] = node_name - - def child_constructor(self): - return TrustyCompilerVersionConfigNode if self.props["compiler_name"] else PyVerConfigNode - - -class TrustyCompilerVersionConfigNode(TreeConfigNode): - - def init2(self, node_name): - self.props["compiler_version"] = node_name - - def child_constructor(self): - return PyVerConfigNode - - -class PyVerConfigNode(TreeConfigNode): - def init2(self, node_name): - self.props["pyver"] = node_name - self.props["abbreviated_pyver"] = get_major_pyver(node_name) - - def child_constructor(self): - return XlaConfigNode - - -class XlaConfigNode(TreeConfigNode): - def modify_label(self, label): - return "XLA=" + str(label) - - def init2(self, node_name): - self.props["is_xla"] = node_name - - -class XenialCompilerConfigNode(TreeConfigNode): - - def init2(self, node_name): - self.props["compiler_name"] = node_name - - def child_constructor(self): - return XenialCompilerVersionConfigNode - - -class XenialCompilerVersionConfigNode(TreeConfigNode): - def init2(self, node_name): - self.props["compiler_version"] = node_name - - def child_constructor(self): - return PyVerConfigNode - - def instantiate_configs(): config_list = [] diff --git a/.circleci/cimodel/lib/conf_tree.py b/.circleci/cimodel/lib/conf_tree.py index 8a1caef..26aa05b 100644 --- a/.circleci/cimodel/lib/conf_tree.py +++ b/.circleci/cimodel/lib/conf_tree.py @@ -1,6 +1,13 @@ #!/usr/bin/env python3 +def X(val): + """ + Compact way to write a leaf node + """ + return val, [] + + class Ver(object): """ Represents a product with a version number -- 2.7.4