return newlist
-def add_build_entries(jobs_dict, phase, smoke):
+def predicate_exclude_nonlinux_and_libtorch(config):
+ return config.os == "linux" and (config.smoke or config.pydistro != "libtorch")
+
+
+def add_build_entries(jobs_dict, phase, smoke, filter_predicate=lambda x: True):
configs = gen_build_env_list(smoke)
- for conf_options in configs:
+ for conf_options in filter(filter_predicate, configs):
jobs_dict[conf_options.gen_build_name(phase)] = conf_options.gen_yaml_tree(phase)
add_build_entries(jobs_dict, "build", False)
+def add_binary_build_tests(jobs_dict):
+ add_build_entries(jobs_dict, "test", False, predicate_exclude_nonlinux_and_libtorch)
+
+
def add_binary_build_uploads(jobs_dict):
add_build_entries(jobs_dict, "upload", False)
return mylist
-def predicate_exclude_nonlinux_and_libtorch(config):
- return config.os == "linux" and (config.smoke or config.pydistro != "libtorch")
-
-
-def add_binary_build_tests(jobs_dict):
-
- configs = gen_build_env_list(False)
- filtered_configs = filter(predicate_exclude_nonlinux_and_libtorch, configs)
-
- for conf_options in filtered_configs:
- jobs_dict[conf_options.gen_build_name("test")] = conf_options.gen_yaml_tree("test")
-
-
def gen_schedule_tree(cron_timing):
return [{
"schedule": {
build_name = build_config.gen_build_name("build")
jobs_list.append(build_name)
- d = OrderedDict(
+ jobs_dict[toplevel_key] = OrderedDict(
triggers=gen_schedule_tree(cron_schedule),
jobs=jobs_list,
)
- jobs_dict[toplevel_key] = d
-
graph = visualization.generate_graph(get_root(smoke, toplevel_key))
graph.draw(toplevel_key + "-config-dimensions.png", prog="twopi")
--- /dev/null
+#!/usr/bin/env python3
+
+from cimodel.lib.conf_tree import ConfigNode, X
+from cimodel.lib.conf_tree import Ver
+import cimodel.data.dimensions as dimensions
+
+
+CONFIG_TREE_DATA = [
+ (Ver("ubuntu", "14.04"), [
+ (Ver("gcc", "4.8"), [X("py2")]),
+ (Ver("gcc", "4.9"), [X("py2")]),
+ ]),
+ (Ver("ubuntu", "16.04"), [
+ (Ver("cuda", "8.0"), [X("py2")]),
+ (Ver("cuda", "9.0"), [
+ # TODO make explicit that this is a "secret TensorRT build"
+ # (see https://github.com/pytorch/pytorch/pull/17323#discussion_r259446749)
+ X("py2"),
+ X("cmake"),
+ ]),
+ (Ver("cuda", "9.1"), [X("py2")]),
+ (Ver("mkl"), [X("py2")]),
+ (Ver("gcc", "5"), [X("onnx_py2")]),
+ (Ver("clang", "3.8"), [X("py2")]),
+ (Ver("clang", "3.9"), [X("py2")]),
+ (Ver("clang", "7"), [X("py2")]),
+ (Ver("android"), [X("py2")]),
+ ]),
+ (Ver("centos", "7"), [
+ (Ver("cuda", "9.0"), [X("py2")]),
+ ]),
+ (Ver("macos", "10.13"), [
+ # TODO ios and system aren't related. system qualifies where the python comes
+ # from (use the system python instead of homebrew or anaconda)
+ (Ver("ios"), [X("py2")]),
+ (Ver("system"), [X("py2")]),
+ ]),
+]
+
+
+class TreeConfigNode(ConfigNode):
+ def __init__(self, parent, node_name, subtree):
+ super(TreeConfigNode, self).__init__(parent, self.modify_label(node_name))
+ self.subtree = subtree
+ self.init2(node_name)
+
+ # noinspection PyMethodMayBeStatic
+ def modify_label(self, label):
+ return str(label)
+
+ def init2(self, node_name):
+ pass
+
+ def get_children(self):
+ return [self.child_constructor()(self, k, v) for (k, v) in self.subtree]
+
+ def is_build_only(self):
+ return str(self.find_prop("compiler_version")) in [
+ "gcc4.9",
+ "clang3.8",
+ "clang3.9",
+ "clang7",
+ "android",
+ ] or self.find_prop("distro_version").name == "macos"
+
+
+class TopLevelNode(TreeConfigNode):
+ def __init__(self, node_name, subtree):
+ super(TopLevelNode, self).__init__(None, node_name, subtree)
+
+ # noinspection PyMethodMayBeStatic
+ def child_constructor(self):
+ return DistroConfigNode
+
+
+class DistroConfigNode(TreeConfigNode):
+ def init2(self, node_name):
+ self.props["distro_version"] = node_name
+
+ # noinspection PyMethodMayBeStatic
+ def child_constructor(self):
+ return CompilerConfigNode
+
+
+class CompilerConfigNode(TreeConfigNode):
+ def init2(self, node_name):
+ self.props["compiler_version"] = node_name
+
+ # noinspection PyMethodMayBeStatic
+ def child_constructor(self):
+ return LanguageConfigNode
+
+
+class LanguageConfigNode(TreeConfigNode):
+ def init2(self, node_name):
+ self.props["language_version"] = node_name
+ self.props["build_only"] = self.is_build_only()
+
+ def get_children(self):
+
+ children = []
+ for phase in dimensions.PHASES:
+ if phase == "build" or not self.props["build_only"]:
+ children.append(PhaseConfigNode(self, phase, []))
+
+ return children
+
+
+class PhaseConfigNode(TreeConfigNode):
+ def init2(self, node_name):
+ self.props["phase_name"] = node_name
from collections import OrderedDict
-import cimodel.data.dimensions as dimensions
+import cimodel.lib.conf_tree as conf_tree
import cimodel.lib.miniutils as miniutils
-from cimodel.lib.conf_tree import Ver
+import cimodel.lib.visualization as visualization
+from cimodel.data.caffe2_build_data import CONFIG_TREE_DATA, TopLevelNode
DOCKER_IMAGE_PATH_BASE = "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/"
DOCKER_IMAGE_VERSION = 266
-CONFIG_HIERARCHY = [
- (Ver("ubuntu", "14.04"), [
- (Ver("gcc", "4.8"), ["py2"]),
- (Ver("gcc", "4.9"), ["py2"]),
- ]),
- (Ver("ubuntu", "16.04"), [
- (Ver("cuda", "8.0"), ["py2"]),
- (Ver("cuda", "9.0"), [
- # TODO make explicit that this is a "secret TensorRT build"
- # (see https://github.com/pytorch/pytorch/pull/17323#discussion_r259446749)
- "py2",
- "cmake",
- ]),
- (Ver("cuda", "9.1"), ["py2"]),
- (Ver("mkl"), ["py2"]),
- (Ver("gcc", "5"), ["onnx_py2"]),
- (Ver("clang", "3.8"), ["py2"]),
- (Ver("clang", "3.9"), ["py2"]),
- (Ver("clang", "7"), ["py2"]),
- (Ver("android"), ["py2"]),
- ]),
- (Ver("centos", "7"), [
- (Ver("cuda", "9.0"), ["py2"]),
- ]),
- (Ver("macos", "10.13"), [
- # TODO ios and system aren't related. system qualifies where the python comes
- # from (use the system python instead of homebrew or anaconda)
- (Ver("ios"), ["py2"]),
- (Ver("system"), ["py2"]),
- ]),
-]
-
-
class Conf(object):
- def __init__(self, language, distro, compiler, phase):
+ def __init__(self, language, distro, compiler, phase, build_only):
self.language = language
self.distro = distro
self.compiler = compiler
self.phase = phase
-
- def is_build_only(self):
- return str(self.compiler) in [
- "gcc4.9",
- "clang3.8",
- "clang3.9",
- "clang7",
- "android",
- ] or self.get_platform() == "macos"
+ self.build_only = build_only
# TODO: Eventually we can probably just remove the cudnn7 everywhere.
def get_cudnn_insertion(self):
return self.construct_phase_name(self.phase)
def get_platform(self):
- return "macos" if self.distro.name == "macos" else "linux"
+ platform = self.distro.name
+ if self.distro.name != "macos":
+ platform = "linux"
+ return platform
def gen_docker_image(self):
else:
tuples.append(("DOCKER_IMAGE", self.gen_docker_image()))
- if self.is_build_only():
+ if self.build_only:
tuples.append(("BUILD_ONLY", miniutils.quote("1")))
d = OrderedDict({"environment": OrderedDict(tuples)})
return d
-def gen_build_list():
- x = []
- for distro, d1 in CONFIG_HIERARCHY:
- for compiler_name, build_languages in d1:
- for language in build_languages:
- for phase in dimensions.PHASES:
+def get_root():
+ return TopLevelNode("Caffe2 Builds", CONFIG_TREE_DATA)
- c = Conf(language, distro, compiler_name, phase)
- if phase == "build" or not c.is_build_only():
- x.append(c)
+def instantiate_configs():
- return x
+ config_list = []
+
+ root = get_root()
+ found_configs = conf_tree.dfs(root)
+ for fc in found_configs:
+
+ c = Conf(
+ fc.find_prop("language_version"),
+ fc.find_prop("distro_version"),
+ fc.find_prop("compiler_version"),
+ fc.find_prop("phase_name"),
+ fc.find_prop("build_only"),
+ )
+
+ config_list.append(c)
+
+ return config_list
def add_caffe2_builds(jobs_dict):
- configs = gen_build_list()
+ configs = instantiate_configs()
for conf_options in configs:
jobs_dict[conf_options.get_name()] = conf_options.gen_yaml_tree()
+ graph = visualization.generate_graph(get_root())
+ graph.draw("caffe2-config-dimensions.png", prog="twopi")
+
def get_caffe2_workflows():
- configs = gen_build_list()
+ configs = instantiate_configs()
# TODO Why don't we build this config?
# See https://github.com/pytorch/pytorch/pull/17323#discussion_r259450540
--- /dev/null
+#!/usr/bin/env python3
+
+from cimodel.lib.conf_tree import ConfigNode, X
+
+
+CONFIG_TREE_DATA = [
+ ("trusty", [
+ (None, [
+ X("2.7.9"),
+ X("2.7"),
+ X("3.5"),
+ X("nightly"),
+ ]),
+ ("gcc", [
+ ("4.8", [X("3.6")]),
+ ("5.4", [("3.6", [X(False), X(True)])]),
+ ("7", [X("3.6")]),
+ ]),
+ ]),
+ ("xenial", [
+ ("clang", [
+ ("5", [X("3.6")]),
+ ]),
+ ("cuda", [
+ ("8", [X("3.6")]),
+ ("9", [
+ # Note there are magic strings here
+ # https://github.com/pytorch/pytorch/blob/master/.jenkins/pytorch/build.sh#L21
+ # and
+ # https://github.com/pytorch/pytorch/blob/master/.jenkins/pytorch/build.sh#L143
+ # and
+ # https://github.com/pytorch/pytorch/blob/master/.jenkins/pytorch/build.sh#L153
+ # (from https://github.com/pytorch/pytorch/pull/17323#discussion_r259453144)
+ X("2.7"),
+ X("3.6"),
+ ]),
+ ("9.2", [X("3.6")]),
+ ("10", [X("3.6")]),
+ ]),
+ ("android", [
+ ("r19c", [X("3.6")]),
+ ]),
+ ]),
+]
+
+
+def get_major_pyver(dotted_version):
+ parts = dotted_version.split(".")
+ return "py" + parts[0]
+
+
+class TreeConfigNode(ConfigNode):
+ def __init__(self, parent, node_name, subtree):
+ super(TreeConfigNode, self).__init__(parent, self.modify_label(node_name))
+ self.subtree = subtree
+ self.init2(node_name)
+
+ def modify_label(self, label):
+ return label
+
+ def init2(self, node_name):
+ pass
+
+ def get_children(self):
+ return [self.child_constructor()(self, k, v) for (k, v) in self.subtree]
+
+
+class TopLevelNode(TreeConfigNode):
+ def __init__(self, node_name, subtree):
+ super(TopLevelNode, self).__init__(None, node_name, subtree)
+
+ # noinspection PyMethodMayBeStatic
+ def child_constructor(self):
+ return DistroConfigNode
+
+
+class DistroConfigNode(TreeConfigNode):
+ def init2(self, node_name):
+ self.props["distro_name"] = node_name
+
+ def child_constructor(self):
+ distro = self.find_prop("distro_name")
+
+ next_nodes = {
+ "trusty": TrustyCompilerConfigNode,
+ "xenial": XenialCompilerConfigNode,
+ }
+ return next_nodes[distro]
+
+
+class TrustyCompilerConfigNode(TreeConfigNode):
+
+ def modify_label(self, label):
+ return label or "<unspecified>"
+
+ def init2(self, node_name):
+ self.props["compiler_name"] = node_name
+
+ def child_constructor(self):
+ return TrustyCompilerVersionConfigNode if self.props["compiler_name"] else PyVerConfigNode
+
+
+class TrustyCompilerVersionConfigNode(TreeConfigNode):
+
+ def init2(self, node_name):
+ self.props["compiler_version"] = node_name
+
+ # noinspection PyMethodMayBeStatic
+ def child_constructor(self):
+ return PyVerConfigNode
+
+
+class PyVerConfigNode(TreeConfigNode):
+ def init2(self, node_name):
+ self.props["pyver"] = node_name
+ self.props["abbreviated_pyver"] = get_major_pyver(node_name)
+
+ # noinspection PyMethodMayBeStatic
+ def child_constructor(self):
+ return XlaConfigNode
+
+
+class XlaConfigNode(TreeConfigNode):
+ def modify_label(self, label):
+ return "XLA=" + str(label)
+
+ def init2(self, node_name):
+ self.props["is_xla"] = node_name
+
+
+class XenialCompilerConfigNode(TreeConfigNode):
+
+ def init2(self, node_name):
+ self.props["compiler_name"] = node_name
+
+ # noinspection PyMethodMayBeStatic
+ def child_constructor(self):
+ return XenialCompilerVersionConfigNode
+
+
+class XenialCompilerVersionConfigNode(TreeConfigNode):
+ def init2(self, node_name):
+ self.props["compiler_version"] = node_name
+
+ # noinspection PyMethodMayBeStatic
+ def child_constructor(self):
+ return PyVerConfigNode
from collections import OrderedDict
+from cimodel.data.pytorch_build_data import TopLevelNode, CONFIG_TREE_DATA
import cimodel.data.dimensions as dimensions
import cimodel.lib.conf_tree as conf_tree
import cimodel.lib.miniutils as miniutils
import cimodel.lib.visualization as visualization
-from cimodel.lib.conf_tree import ConfigNode
DOCKER_IMAGE_PATH_BASE = "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/"
# caffe2 test job dependent on a pytorch build job. This way we could quickly dedup the repeated
# build of pytorch in the caffe2 build job, and just run the caffe2 tests off of a completed
# pytorch build job (from https://github.com/pytorch/pytorch/pull/17323#discussion_r259452641)
- if phase == "test":
- dependency_build = self.parent_build or self
- val["requires"] = [dependency_build.gen_build_name("build")]
+
+ dependency_build = self.parent_build or self
+ val["requires"] = [dependency_build.gen_build_name("build")]
return {self.gen_build_name(phase): val}
else:
return configs
-def X(val):
- """
- Compact way to write a leaf node
- """
- return val, []
-
-
-CONFIG_TREE_DATA = [
- ("trusty", [
- (None, [
- X("2.7.9"),
- X("2.7"),
- X("3.5"),
- X("nightly"),
- ]),
- ("gcc", [
- ("4.8", [X("3.6")]),
- ("5.4", [("3.6", [X(False), X(True)])]),
- ("7", [X("3.6")]),
- ]),
- ]),
- ("xenial", [
- ("clang", [
- ("5", [X("3.6")]),
- ]),
- ("cuda", [
- ("8", [X("3.6")]),
- ("9", [
- # Note there are magic strings here
- # https://github.com/pytorch/pytorch/blob/master/.jenkins/pytorch/build.sh#L21
- # and
- # https://github.com/pytorch/pytorch/blob/master/.jenkins/pytorch/build.sh#L143
- # and
- # https://github.com/pytorch/pytorch/blob/master/.jenkins/pytorch/build.sh#L153
- # (from https://github.com/pytorch/pytorch/pull/17323#discussion_r259453144)
- X("2.7"),
- X("3.6"),
- ]),
- ("9.2", [X("3.6")]),
- ("10", [X("3.6")]),
- ]),
- ("android", [
- ("r19c", [X("3.6")]),
- ]),
- ]),
-]
-
-
-def get_major_pyver(dotted_version):
- parts = dotted_version.split(".")
- return "py" + parts[0]
-
-
def get_root():
return TopLevelNode("PyTorch Builds", CONFIG_TREE_DATA)
return configs_list
-class TreeConfigNode(ConfigNode):
- def __init__(self, parent, node_name, subtree):
- super(TreeConfigNode, self).__init__(parent, self.modify_label(node_name))
- self.subtree = subtree
- self.init2(node_name)
-
- def modify_label(self, label):
- return label
-
- def init2(self, node_name):
- pass
-
- def get_children(self):
- return [self.child_constructor()(self, k, v) for (k, v) in self.subtree]
-
-
-class TopLevelNode(TreeConfigNode):
- def __init__(self, node_name, subtree):
- super(TopLevelNode, self).__init__(None, node_name, subtree)
-
- def child_constructor(self):
- return DistroConfigNode
-
-
-class DistroConfigNode(TreeConfigNode):
- def init2(self, node_name):
- self.props["distro_name"] = node_name
-
- def child_constructor(self):
- distro = self.find_prop("distro_name")
- return TrustyCompilerConfigNode if distro == "trusty" else XenialCompilerConfigNode
-
-
-class TrustyCompilerConfigNode(TreeConfigNode):
-
- def modify_label(self, label):
- return label or "<unspecified>"
-
- def init2(self, node_name):
- self.props["compiler_name"] = node_name
-
- def child_constructor(self):
- return TrustyCompilerVersionConfigNode if self.props["compiler_name"] else PyVerConfigNode
-
-
-class TrustyCompilerVersionConfigNode(TreeConfigNode):
-
- def init2(self, node_name):
- self.props["compiler_version"] = node_name
-
- def child_constructor(self):
- return PyVerConfigNode
-
-
-class PyVerConfigNode(TreeConfigNode):
- def init2(self, node_name):
- self.props["pyver"] = node_name
- self.props["abbreviated_pyver"] = get_major_pyver(node_name)
-
- def child_constructor(self):
- return XlaConfigNode
-
-
-class XlaConfigNode(TreeConfigNode):
- def modify_label(self, label):
- return "XLA=" + str(label)
-
- def init2(self, node_name):
- self.props["is_xla"] = node_name
-
-
-class XenialCompilerConfigNode(TreeConfigNode):
-
- def init2(self, node_name):
- self.props["compiler_name"] = node_name
-
- def child_constructor(self):
- return XenialCompilerVersionConfigNode
-
-
-class XenialCompilerVersionConfigNode(TreeConfigNode):
- def init2(self, node_name):
- self.props["compiler_version"] = node_name
-
- def child_constructor(self):
- return PyVerConfigNode
-
-
def instantiate_configs():
config_list = []
#!/usr/bin/env python3
+def X(val):
+ """
+ Compact way to write a leaf node
+ """
+ return val, []
+
+
class Ver(object):
"""
Represents a product with a version number
projects / platform / upstream / pytorch.git / commitdiff