From e1c272797b1078220b3333386450096062acd27f Mon Sep 17 00:00:00 2001 From: Karl Ostmo Date: Mon, 25 Mar 2019 18:01:39 -0700 Subject: [PATCH] Don't require pygraphviz for regenerate.sh (#17485) Summary: closes #17336 Do not overwrite config.yml if script throws an error Pull Request resolved: https://github.com/pytorch/pytorch/pull/17485 Differential Revision: D14604388 Pulled By: kostmo fbshipit-source-id: 5024545e3a8711abdbc0800911c766929dbca196 --- .circleci/README.md | 4 +- .circleci/cimodel/data/__init__.py | 0 .../binary_build_data.py} | 26 +-- .../cimodel/{ => data}/binary_build_definitions.py | 42 ++--- .../cimodel/{ => data}/caffe2_build_definitions.py | 33 ++-- .circleci/cimodel/data/dimensions.py | 18 ++ .../{ => data}/pytorch_build_definitions.py | 185 ++++++++++----------- .circleci/cimodel/dimensions.py | 4 - .circleci/cimodel/lib/__init__.py | 0 .circleci/cimodel/{ => lib}/conf_tree.py | 1 + .circleci/cimodel/{ => lib}/miniutils.py | 6 +- .circleci/cimodel/{ => lib}/miniyaml.py | 0 .circleci/cimodel/{ => lib}/visualization.py | 25 ++- .circleci/config.yml | 62 ++++--- .circleci/generate_config_yml.py | 71 +++++--- .circleci/regenerate.sh | 4 +- .../verbatim-sources/binary-build-specs-header.yml | 3 - .../verbatim-sources/binary-build-tests-header.yml | 4 - .../binary-build-uploads-header.yml | 2 - .circleci/verbatim-sources/header-section.yml | 4 +- .circleci/verbatim-sources/job-specs-header.yml | 5 - .../verbatim-sources/smoke-test-specs-header.yml | 4 - .../workflows-binary-smoke-header.yml | 3 - .../workflows-nightly-tests-header.yml | 1 - .travis.yml | 5 - 25 files changed, 252 insertions(+), 260 deletions(-) create mode 100644 .circleci/cimodel/data/__init__.py rename .circleci/cimodel/{make_build_configs.py => data/binary_build_data.py} (91%) rename .circleci/cimodel/{ => data}/binary_build_definitions.py (82%) rename .circleci/cimodel/{ => data}/caffe2_build_definitions.py (86%) create mode 100644 .circleci/cimodel/data/dimensions.py rename .circleci/cimodel/{ => data}/pytorch_build_definitions.py (69%) delete mode 100644 .circleci/cimodel/dimensions.py create mode 100644 .circleci/cimodel/lib/__init__.py rename .circleci/cimodel/{ => lib}/conf_tree.py (98%) rename .circleci/cimodel/{ => lib}/miniutils.py (57%) rename .circleci/cimodel/{ => lib}/miniyaml.py (100%) rename .circleci/cimodel/{ => lib}/visualization.py (79%) delete mode 100644 .circleci/verbatim-sources/binary-build-specs-header.yml delete mode 100644 .circleci/verbatim-sources/binary-build-tests-header.yml delete mode 100644 .circleci/verbatim-sources/binary-build-uploads-header.yml delete mode 100644 .circleci/verbatim-sources/job-specs-header.yml delete mode 100644 .circleci/verbatim-sources/smoke-test-specs-header.yml delete mode 100644 .circleci/verbatim-sources/workflows-binary-smoke-header.yml delete mode 100644 .circleci/verbatim-sources/workflows-nightly-tests-header.yml diff --git a/.circleci/README.md b/.circleci/README.md index 74ac300..7a70cb9 100644 --- a/.circleci/README.md +++ b/.circleci/README.md @@ -23,7 +23,9 @@ The documentation, in the form of diagrams, is automatically generated and canno Furthermore, consistency is enforced within the YAML config itself, by using a single source of data to generate multiple parts of the file. -See https://github.com/pytorch/pytorch/issues/17038 +* Facilitates one-off culling/enabling of CI configs for testing PRs on special targets + +Also see https://github.com/pytorch/pytorch/issues/17038 Future direction diff --git a/.circleci/cimodel/data/__init__.py b/.circleci/cimodel/data/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/.circleci/cimodel/make_build_configs.py b/.circleci/cimodel/data/binary_build_data.py similarity index 91% rename from .circleci/cimodel/make_build_configs.py rename to .circleci/cimodel/data/binary_build_data.py index 8782d77..1e5c563 100644 --- a/.circleci/cimodel/make_build_configs.py +++ b/.circleci/cimodel/data/binary_build_data.py @@ -14,7 +14,8 @@ to produce a visualization of config dimensions. from collections import OrderedDict -from cimodel.conf_tree import ConfigNode +from cimodel.lib.conf_tree import ConfigNode +import cimodel.data.dimensions as dimensions LINKING_DIMENSIONS = [ @@ -32,23 +33,8 @@ def get_processor_arch_name(cuda_version): return "cpu" if not cuda_version else "cu" + cuda_version -CUDA_VERSIONS = [ - None, # cpu build - "80", - "90", - "100", -] - -STANDARD_PYTHON_VERSIONS = [ - "2.7", - "3.5", - "3.6", - "3.7", -] - - CONFIG_TREE_DATA = OrderedDict( - linux=(CUDA_VERSIONS, OrderedDict( + linux=(dimensions.CUDA_VERSIONS, OrderedDict( manywheel=[ "2.7m", "2.7mu", @@ -56,14 +42,14 @@ CONFIG_TREE_DATA = OrderedDict( "3.6m", "3.7m", ], - conda=STANDARD_PYTHON_VERSIONS, + conda=dimensions.STANDARD_PYTHON_VERSIONS, libtorch=[ "2.7m", ] )), macos=([None], OrderedDict( - wheel=STANDARD_PYTHON_VERSIONS, - conda=STANDARD_PYTHON_VERSIONS, + wheel=dimensions.STANDARD_PYTHON_VERSIONS, + conda=dimensions.STANDARD_PYTHON_VERSIONS, libtorch=[ "2.7", ], diff --git a/.circleci/cimodel/binary_build_definitions.py b/.circleci/cimodel/data/binary_build_definitions.py similarity index 82% rename from .circleci/cimodel/binary_build_definitions.py rename to .circleci/cimodel/data/binary_build_definitions.py index 547041a..161d6a9 100644 --- a/.circleci/cimodel/binary_build_definitions.py +++ b/.circleci/cimodel/data/binary_build_definitions.py @@ -2,10 +2,10 @@ from collections import OrderedDict -import cimodel.conf_tree as conf_tree -import cimodel.miniutils as miniutils -import cimodel.make_build_configs as make_build_configs -import cimodel.visualization as visualization +import cimodel.data.binary_build_data as binary_build_data +import cimodel.lib.conf_tree as conf_tree +import cimodel.lib.miniutils as miniutils +import cimodel.lib.visualization as visualization class Conf(object): @@ -19,7 +19,7 @@ class Conf(object): self.libtorch_variant = libtorch_variant def gen_build_env_parms(self): - return [self.pydistro] + self.parms + [make_build_configs.get_processor_arch_name(self.cuda_version)] + return [self.pydistro] + self.parms + [binary_build_data.get_processor_arch_name(self.cuda_version)] def gen_docker_image(self): @@ -50,40 +50,28 @@ class Conf(object): def gen_yaml_tree(self, build_or_test): - env_dict = OrderedDict({ - "BUILD_ENVIRONMENT": miniutils.quote(" ".join(self.gen_build_env_parms())), - }) + env_tuples = [("BUILD_ENVIRONMENT", miniutils.quote(" ".join(self.gen_build_env_parms())))] if self.libtorch_variant: - env_dict["LIBTORCH_VARIANT"] = miniutils.quote(self.libtorch_variant) + env_tuples.append(("LIBTORCH_VARIANT", miniutils.quote(self.libtorch_variant))) - os_word_substitution = { - "macos": "mac", - } - - os_name = miniutils.override(self.os, os_word_substitution) - - d = { - "environment": env_dict, - "<<": "*" + "_".join([self.get_name_prefix(), os_name, build_or_test]), - } + os_name = miniutils.override(self.os, {"macos": "mac"}) + d = {"<<": "*" + "_".join([self.get_name_prefix(), os_name, build_or_test])} if build_or_test == "test": - tuples = [] if not (self.smoke and self.os == "macos"): - tuples.append(("DOCKER_IMAGE", self.gen_docker_image())) + env_tuples.append(("DOCKER_IMAGE", self.gen_docker_image())) if self.cuda_version: - tuples.append(("USE_CUDA_DOCKER_RUNTIME", miniutils.quote("1"))) - - for (k, v) in tuples: - env_dict[k] = v + env_tuples.append(("USE_CUDA_DOCKER_RUNTIME", miniutils.quote("1"))) else: if self.os == "linux" and build_or_test != "upload": d["docker"] = [{"image": self.gen_docker_image()}] + d["environment"] = OrderedDict(env_tuples) + if build_or_test == "test": if self.cuda_version: d["resource_class"] = "gpu.medium" @@ -93,9 +81,9 @@ class Conf(object): def get_root(smoke, name): - return make_build_configs.TopLevelNode( + return binary_build_data.TopLevelNode( name, - make_build_configs.CONFIG_TREE_DATA, + binary_build_data.CONFIG_TREE_DATA, smoke, ) diff --git a/.circleci/cimodel/caffe2_build_definitions.py b/.circleci/cimodel/data/caffe2_build_definitions.py similarity index 86% rename from .circleci/cimodel/caffe2_build_definitions.py rename to .circleci/cimodel/data/caffe2_build_definitions.py index 6ee76b0..30647c5 100644 --- a/.circleci/cimodel/caffe2_build_definitions.py +++ b/.circleci/cimodel/data/caffe2_build_definitions.py @@ -2,9 +2,9 @@ from collections import OrderedDict -import cimodel.dimensions as dimensions -import cimodel.miniutils as miniutils -from cimodel.conf_tree import Ver +import cimodel.data.dimensions as dimensions +import cimodel.lib.miniutils as miniutils +from cimodel.lib.conf_tree import Ver DOCKER_IMAGE_PATH_BASE = "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/" @@ -126,33 +126,22 @@ class Conf(object): tuples.append(("BUILD_IOS", miniutils.quote("1"))) if self.phase == "test": - use_cuda_docker = self.compiler.name == "cuda" - if use_cuda_docker: + # TODO cuda should not be considered a compiler + if self.compiler.name == "cuda": tuples.append(("USE_CUDA_DOCKER_RUNTIME", miniutils.quote("1"))) - if not self.distro.name == "macos": - tuples.append(("DOCKER_IMAGE", self.gen_docker_image())) - - if self.is_build_only(): - if not self.distro.name == "macos": - tuples.append(("BUILD_ONLY", miniutils.quote("1"))) - - # TODO: not sure we need the distinction between system and homebrew anymore. Our python handling in cmake - # and setuptools is more robust now than when we first had these. if self.distro.name == "macos": - tuples.append(("PYTHON_INSTALLATION", miniutils.quote("system"))) tuples.append(("PYTHON_VERSION", miniutils.quote("2"))) - env_dict = OrderedDict(tuples) + else: + tuples.append(("DOCKER_IMAGE", self.gen_docker_image())) + if self.is_build_only(): + tuples.append(("BUILD_ONLY", miniutils.quote("1"))) - d = OrderedDict([ - ("environment", env_dict), - ]) + d = OrderedDict({"environment": OrderedDict(tuples)}) if self.phase == "test": - is_large = self.compiler.name != "cuda" - - resource_class = "large" if is_large else "gpu.medium" + resource_class = "large" if self.compiler.name != "cuda" else "gpu.medium" d["resource_class"] = resource_class d["<<"] = "*" + "_".join(["caffe2", self.get_platform(), self.phase, "defaults"]) diff --git a/.circleci/cimodel/data/dimensions.py b/.circleci/cimodel/data/dimensions.py new file mode 100644 index 0000000..edf297d --- /dev/null +++ b/.circleci/cimodel/data/dimensions.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 + + +PHASES = ["build", "test"] + +CUDA_VERSIONS = [ + None, # cpu build + "80", + "90", + "100", +] + +STANDARD_PYTHON_VERSIONS = [ + "2.7", + "3.5", + "3.6", + "3.7", +] diff --git a/.circleci/cimodel/pytorch_build_definitions.py b/.circleci/cimodel/data/pytorch_build_definitions.py similarity index 69% rename from .circleci/cimodel/pytorch_build_definitions.py rename to .circleci/cimodel/data/pytorch_build_definitions.py index 4e94b5b..48bfb67 100644 --- a/.circleci/cimodel/pytorch_build_definitions.py +++ b/.circleci/cimodel/data/pytorch_build_definitions.py @@ -2,11 +2,11 @@ from collections import OrderedDict -import cimodel.conf_tree as conf_tree -import cimodel.dimensions as dimensions -import cimodel.miniutils as miniutils -import cimodel.visualization as visualization -from cimodel.conf_tree import ConfigNode +import cimodel.data.dimensions as dimensions +import cimodel.lib.conf_tree as conf_tree +import cimodel.lib.miniutils as miniutils +import cimodel.lib.visualization as visualization +from cimodel.lib.conf_tree import ConfigNode DOCKER_IMAGE_PATH_BASE = "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/" @@ -129,11 +129,7 @@ class HiddenConf(object): def gen_workflow_yaml_item(self, phase): - val = OrderedDict() - dependency_build = self.parent_build - val["requires"] = [dependency_build.gen_build_name("build")] - - return {self.gen_build_name(phase): val} + return {self.gen_build_name(phase): {"requires": [self.parent_build.gen_build_name("build")]}} def gen_build_name(self, _): return self.name @@ -153,10 +149,10 @@ def gen_dependent_configs(xenial_parent_config): for parms, gpu in extra_parms: c = Conf( - "xenial", + xenial_parent_config.distro, ["py3"] + parms, pyver="3.6", - cuda_version="8", + cuda_version=xenial_parent_config.cuda_version, restrict_phases=["test"], gpu_resource=gpu, parent_build=xenial_parent_config, @@ -170,25 +166,33 @@ def gen_dependent_configs(xenial_parent_config): return configs -# TODO make the schema consistent between "trusty" and "xenial" +def X(val): + """ + Compact way to write a leaf node + """ + return val, [] + + CONFIG_TREE_DATA = [ ("trusty", [ - ("2.7.9", []), - ("2.7", []), - ("3.5", []), - ("3.6", [ - ("gcc4.8", []), - ("gcc5.4", [False, True]), - ("gcc7", []), + (None, [ + X("2.7.9"), + X("2.7"), + X("3.5"), + X("nightly"), + ]), + ("gcc", [ + ("4.8", [X("3.6")]), + ("5.4", [("3.6", [X(False), X(True)])]), + ("7", [X("3.6")]), ]), - ("nightly", []), ]), ("xenial", [ ("clang", [ - ("5", [("3.6", [])]), + ("5", [X("3.6")]), ]), ("cuda", [ - ("8", [("3.6", [])]), + ("8", [X("3.6")]), ("9", [ # Note there are magic strings here # https://github.com/pytorch/pytorch/blob/master/.jenkins/pytorch/build.sh#L21 @@ -197,11 +201,11 @@ CONFIG_TREE_DATA = [ # and # https://github.com/pytorch/pytorch/blob/master/.jenkins/pytorch/build.sh#L153 # (from https://github.com/pytorch/pytorch/pull/17323#discussion_r259453144) - ("2.7", []), - ("3.6", []), + X("2.7"), + X("3.6"), ]), - ("9.2", [("3.6", [])]), - ("10", [("3.6", [])]), + ("9.2", [X("3.6")]), + ("10", [X("3.6")]), ]), ]), ] @@ -222,99 +226,92 @@ def gen_tree(): return configs_list -class TopLevelNode(ConfigNode): - def __init__(self, node_name, config_tree_data): - super(TopLevelNode, self).__init__(None, node_name) - - self.config_tree_data = config_tree_data - - def get_children(self): - return [DistroConfigNode(self, d, p) for (d, p) in self.config_tree_data] - +class TreeConfigNode(ConfigNode): + def __init__(self, parent, node_name, subtree): + super(TreeConfigNode, self).__init__(parent, self.modify_label(node_name)) + self.subtree = subtree + self.init2(node_name) -class DistroConfigNode(ConfigNode): - def __init__(self, parent, distro_name, subtree): - super(DistroConfigNode, self).__init__(parent, distro_name) + def modify_label(self, label): + return label - self.subtree = subtree - self.props["distro_name"] = distro_name + def init2(self, node_name): + pass def get_children(self): + return [self.child_constructor()(self, k, v) for (k, v) in self.subtree] - if self.find_prop("distro_name") == "trusty": - return [PyVerConfigNode(self, k, v) for k, v in self.subtree] - else: - return [XenialCompilerConfigNode(self, v, subtree) for (v, subtree) in self.subtree] +class TopLevelNode(TreeConfigNode): + def __init__(self, node_name, subtree): + super(TopLevelNode, self).__init__(None, node_name, subtree) -class PyVerConfigNode(ConfigNode): - def __init__(self, parent, pyver, subtree): - super(PyVerConfigNode, self).__init__(parent, pyver) + def child_constructor(self): + return DistroConfigNode - self.subtree = subtree - self.props["pyver"] = pyver - - self.props["abbreviated_pyver"] = get_major_pyver(pyver) - def get_children(self): - return [CompilerConfigNode(self, v, xla_options) for (v, xla_options) in self.subtree] +class DistroConfigNode(TreeConfigNode): + def init2(self, node_name): + self.props["distro_name"] = node_name + def child_constructor(self): + distro = self.find_prop("distro_name") + return TrustyCompilerConfigNode if distro == "trusty" else XenialCompilerConfigNode -class CompilerConfigNode(ConfigNode): - def __init__(self, parent, compiler_name, subtree): - super(CompilerConfigNode, self).__init__(parent, compiler_name) - self.props["compiler_name"] = compiler_name +class TrustyCompilerConfigNode(TreeConfigNode): - self.subtree = subtree + def modify_label(self, label): + return label or "" - def get_children(self): - return [XlaConfigNode(self, v) for v in self.subtree] + def init2(self, node_name): + self.props["compiler_name"] = node_name + def child_constructor(self): + return TrustyCompilerVersionConfigNode if self.props["compiler_name"] else PyVerConfigNode -class XenialCompilerConfigNode(ConfigNode): - def __init__(self, parent, compiler_name, subtree): - super(XenialCompilerConfigNode, self).__init__(parent, compiler_name) - self.props["compiler_name"] = compiler_name +class TrustyCompilerVersionConfigNode(TreeConfigNode): - self.subtree = subtree + def init2(self, node_name): + self.props["compiler_version"] = node_name - def get_children(self): - return [XenialCompilerVersionConfigNode(self, k, v) for (k, v) in self.subtree] + def child_constructor(self): + return PyVerConfigNode -class XenialCompilerVersionConfigNode(ConfigNode): - def __init__(self, parent, compiler_version, subtree): - super(XenialCompilerVersionConfigNode, self).__init__(parent, compiler_version) +class PyVerConfigNode(TreeConfigNode): + def init2(self, node_name): + self.props["pyver"] = node_name + self.props["abbreviated_pyver"] = get_major_pyver(node_name) - self.subtree = subtree + def child_constructor(self): + return XlaConfigNode - self.props["compiler_version"] = compiler_version - def get_children(self): - return [XenialPythonVersionConfigNode(self, v) for (v, _) in self.subtree] +class XlaConfigNode(TreeConfigNode): + def modify_label(self, label): + return "XLA=" + str(label) + def init2(self, node_name): + self.props["is_xla"] = node_name -class XenialPythonVersionConfigNode(ConfigNode): - def __init__(self, parent, python_version): - super(XenialPythonVersionConfigNode, self).__init__(parent, python_version) - self.props["pyver"] = python_version - self.props["abbreviated_pyver"] = get_major_pyver(python_version) +class XenialCompilerConfigNode(TreeConfigNode): - def get_children(self): - return [] + def init2(self, node_name): + self.props["compiler_name"] = node_name + def child_constructor(self): + return XenialCompilerVersionConfigNode -class XlaConfigNode(ConfigNode): - def __init__(self, parent, xla_enabled): - super(XlaConfigNode, self).__init__(parent, "XLA=" + str(xla_enabled)) - self.props["is_xla"] = xla_enabled +class XenialCompilerVersionConfigNode(TreeConfigNode): + def init2(self, node_name): + self.props["compiler_version"] = node_name - def get_children(self): - return [] + def child_constructor(self): + return PyVerConfigNode def instantiate_configs(): @@ -330,18 +327,17 @@ def instantiate_configs(): python_version = None if distro_name == "xenial": python_version = fc.find_prop("pyver") - - if distro_name == "xenial": parms_list = [fc.find_prop("abbreviated_pyver")] else: parms_list = ["py" + fc.find_prop("pyver")] + compiler_name = fc.find_prop("compiler_name") + cuda_version = None - if fc.find_prop("compiler_name") == "cuda": + if compiler_name == "cuda": cuda_version = fc.find_prop("compiler_version") - compiler_name = fc.find_prop("compiler_name") - if compiler_name and compiler_name != "cuda": + elif compiler_name: gcc_version = compiler_name + (fc.find_prop("compiler_version") or "") parms_list.append(gcc_version) @@ -381,7 +377,6 @@ def add_build_env_defs(jobs_dict): mydict = OrderedDict() config_list = instantiate_configs() - for c in config_list: for phase in dimensions.PHASES: @@ -427,7 +422,7 @@ def get_workflow_list(): x.append(conf_options.gen_workflow_yaml_item(phase)) # TODO convert to recursion - for conf in conf_options.dependent_tests: + for conf in conf_options.get_dependents(): x.append(conf.gen_workflow_yaml_item("test")) return x diff --git a/.circleci/cimodel/dimensions.py b/.circleci/cimodel/dimensions.py deleted file mode 100644 index ae7840c..0000000 --- a/.circleci/cimodel/dimensions.py +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env python3 - - -PHASES = ["build", "test"] diff --git a/.circleci/cimodel/lib/__init__.py b/.circleci/cimodel/lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/.circleci/cimodel/conf_tree.py b/.circleci/cimodel/lib/conf_tree.py similarity index 98% rename from .circleci/cimodel/conf_tree.py rename to .circleci/cimodel/lib/conf_tree.py index 9548ec7..8a1caef 100644 --- a/.circleci/cimodel/conf_tree.py +++ b/.circleci/cimodel/lib/conf_tree.py @@ -22,6 +22,7 @@ class ConfigNode(object): def get_label(self): return self.node_name + # noinspection PyMethodMayBeStatic def get_children(self): return [] diff --git a/.circleci/cimodel/miniutils.py b/.circleci/cimodel/lib/miniutils.py similarity index 57% rename from .circleci/cimodel/miniutils.py rename to .circleci/cimodel/lib/miniutils.py index dd21cd4..b10fc44 100644 --- a/.circleci/cimodel/miniutils.py +++ b/.circleci/cimodel/lib/miniutils.py @@ -2,7 +2,11 @@ def quote(s): - return '"' + s + '"' + return sandwich('"', s) + + +def sandwich(bread, jam): + return bread + jam + bread def override(word, substitutions): diff --git a/.circleci/cimodel/miniyaml.py b/.circleci/cimodel/lib/miniyaml.py similarity index 100% rename from .circleci/cimodel/miniyaml.py rename to .circleci/cimodel/lib/miniyaml.py diff --git a/.circleci/cimodel/visualization.py b/.circleci/cimodel/lib/visualization.py similarity index 79% rename from .circleci/cimodel/visualization.py rename to .circleci/cimodel/lib/visualization.py index 2f472c2..c842bf5 100644 --- a/.circleci/cimodel/visualization.py +++ b/.circleci/cimodel/lib/visualization.py @@ -6,7 +6,7 @@ This module encapsulates dependencies on pygraphviz import colorsys -import cimodel.conf_tree as conf_tree +import cimodel.lib.conf_tree as conf_tree def rgb2hex(rgb_tuple): @@ -16,6 +16,25 @@ def rgb2hex(rgb_tuple): return "#" + "".join(map(to_hex, list(rgb_tuple))) +def handle_missing_graphviz(f): + """ + If the user has not installed pygraphviz, this causes + calls to the draw() method of the returned object to do nothing. + """ + try: + import pygraphviz + return f + + except ModuleNotFoundError: + + class FakeGraph: + def draw(self, *args, **kwargs): + pass + + return lambda _: FakeGraph() + + +@handle_missing_graphviz def generate_graph(toplevel_config_node): """ Traverses the graph once first just to find the max depth @@ -27,9 +46,9 @@ def generate_graph(toplevel_config_node): for config in config_list: max_depth = max(max_depth, config.get_depth()) - from pygraphviz import AGraph - # color the nodes using the max depth + + from pygraphviz import AGraph dot = AGraph() def node_discovery_callback(node, sibling_index, sibling_count): diff --git a/.circleci/config.yml b/.circleci/config.yml index 748ed4e..f9c23b4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,8 +5,8 @@ # https://github.com/pytorch/ossci-job-dsl/blob/master/src/main/groovy/ossci/pytorch/DockerVersion.groovy and # https://github.com/pytorch/ossci-job-dsl/blob/master/src/main/groovy/ossci/caffe2/DockerVersion.groovy, # and then update DOCKER_IMAGE_VERSION at the top of the following files: -# * cimodel/pytorch_build_definitions.py -# * cimodel/caffe2_build_definitions.py +# * cimodel/data/pytorch_build_definitions.py +# * cimodel/data/caffe2_build_definitions.py docker_config_defaults: &docker_config_defaults user: jenkins @@ -1103,10 +1103,8 @@ smoke_mac_test: &smoke_mac_test ############################################################################## -############################################################################## # Job specifications job specs ############################################################################## -############################################################################## version: 2 jobs: pytorch_linux_trusty_py2_7_9_build: @@ -1148,6 +1146,19 @@ jobs: resource_class: large <<: *pytorch_linux_test_defaults + pytorch_linux_trusty_pynightly_build: + environment: + BUILD_ENVIRONMENT: pytorch-linux-trusty-pynightly-build + DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-trusty-pynightly:291" + <<: *pytorch_linux_build_defaults + + pytorch_linux_trusty_pynightly_test: + environment: + BUILD_ENVIRONMENT: pytorch-linux-trusty-pynightly-test + DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-trusty-pynightly:291" + resource_class: large + <<: *pytorch_linux_test_defaults + pytorch_linux_trusty_py3_6_gcc4_8_build: environment: BUILD_ENVIRONMENT: pytorch-linux-trusty-py3.6-gcc4.8-build @@ -1200,19 +1211,6 @@ jobs: resource_class: large <<: *pytorch_linux_test_defaults - pytorch_linux_trusty_pynightly_build: - environment: - BUILD_ENVIRONMENT: pytorch-linux-trusty-pynightly-build - DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-trusty-pynightly:291" - <<: *pytorch_linux_build_defaults - - pytorch_linux_trusty_pynightly_test: - environment: - BUILD_ENVIRONMENT: pytorch-linux-trusty-pynightly-test - DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-trusty-pynightly:291" - resource_class: large - <<: *pytorch_linux_test_defaults - pytorch_linux_xenial_py3_clang5_asan_build: environment: BUILD_ENVIRONMENT: pytorch-linux-xenial-py3-clang5-asan-build @@ -1677,14 +1675,12 @@ jobs: environment: BUILD_ENVIRONMENT: caffe2-py2-ios-macos10.13-build BUILD_IOS: "1" - PYTHON_INSTALLATION: "system" PYTHON_VERSION: "2" <<: *caffe2_macos_build_defaults caffe2_py2_system_macos10_13_build: environment: BUILD_ENVIRONMENT: caffe2-py2-system-macos10.13-build - PYTHON_INSTALLATION: "system" PYTHON_VERSION: "2" <<: *caffe2_macos_build_defaults @@ -2040,10 +2036,11 @@ jobs: BUILD_ENVIRONMENT: "libtorch 2.7 cpu" <<: *binary_mac_build - # Binary build tests - # These are the smoke tests run right after the build, before the upload. If - # these fail, the upload doesn't happen - ############################################################################# +############################################################################## +# Binary build tests +# These are the smoke tests run right after the build, before the upload. +# If these fail, the upload doesn't happen. +############################################################################## binary_linux_manywheel_2.7m_cpu_test: environment: BUILD_ENVIRONMENT: "manywheel 2.7m cpu" @@ -2339,8 +2336,9 @@ jobs: # resource_class: gpu.medium # <<: *binary_linux_test - # Binary build uploads - ############################################################################# +############################################################################## +# Binary build uploads +############################################################################## binary_linux_manywheel_2.7m_cpu_upload: environment: BUILD_ENVIRONMENT: "manywheel 2.7m cpu" @@ -2586,7 +2584,6 @@ jobs: BUILD_ENVIRONMENT: "libtorch 2.7 cpu" <<: *binary_mac_upload - ############################################################################## # Smoke test specs individual job specifications ############################################################################## @@ -3065,6 +3062,10 @@ workflows: - pytorch_linux_trusty_py3_5_test: requires: - pytorch_linux_trusty_py3_5_build + - pytorch_linux_trusty_pynightly_build + - pytorch_linux_trusty_pynightly_test: + requires: + - pytorch_linux_trusty_pynightly_build - pytorch_linux_trusty_py3_6_gcc4_8_build - pytorch_linux_trusty_py3_6_gcc4_8_test: requires: @@ -3081,10 +3082,6 @@ workflows: - pytorch_linux_trusty_py3_6_gcc7_test: requires: - pytorch_linux_trusty_py3_6_gcc7_build - - pytorch_linux_trusty_pynightly_build - - pytorch_linux_trusty_pynightly_test: - requires: - - pytorch_linux_trusty_pynightly_build - pytorch_linux_xenial_py3_clang5_asan_build - pytorch_linux_xenial_py3_clang5_asan_test: requires: @@ -3270,7 +3267,6 @@ workflows: - smoke_macos_conda_3.7_cpu - smoke_macos_libtorch_2.7_cpu - ############################################################################## # Daily binary build trigger ############################################################################## @@ -3333,7 +3329,9 @@ workflows: - binary_macos_conda_3.7_cpu_build - binary_macos_libtorch_2.7_cpu_build - # Nightly tests +############################################################################## +# Nightly tests +############################################################################## - binary_linux_manywheel_2.7m_cpu_test: requires: - binary_linux_manywheel_2.7m_cpu_build diff --git a/.circleci/generate_config_yml.py b/.circleci/generate_config_yml.py index 06bce6f..f239c36 100755 --- a/.circleci/generate_config_yml.py +++ b/.circleci/generate_config_yml.py @@ -3,18 +3,18 @@ """ This script is the source of truth for config.yml. Please see README.md in this directory for details. - -In this module, """ import os import sys -from collections import OrderedDict +import shutil +from collections import namedtuple, OrderedDict -import cimodel.pytorch_build_definitions as pytorch_build_definitions -import cimodel.binary_build_definitions as binary_build_definitions -import cimodel.caffe2_build_definitions as caffe2_build_definitions -import cimodel.miniyaml as miniyaml +import cimodel.data.pytorch_build_definitions as pytorch_build_definitions +import cimodel.data.binary_build_definitions as binary_build_definitions +import cimodel.data.caffe2_build_definitions as caffe2_build_definitions +import cimodel.lib.miniutils as miniutils +import cimodel.lib.miniyaml as miniyaml class File(object): @@ -26,16 +26,17 @@ class File(object): def write(self, output_filehandle): with open(os.path.join("verbatim-sources", self.filename)) as fh: - output_filehandle.write(fh.read()) + shutil.copyfileobj(fh, output_filehandle) + + +class FunctionGen(namedtuple('FunctionGen', 'function depth')): + __slots__ = () -class Treegen(object): +class Treegen(FunctionGen): """ Insert the content of a YAML tree into config.yml """ - def __init__(self, function, depth): - self.function = function - self.depth = depth def write(self, output_filehandle): build_dict = OrderedDict() @@ -43,18 +44,33 @@ class Treegen(object): miniyaml.render(output_filehandle, build_dict, self.depth) -class Listgen(object): +class Listgen(FunctionGen): """ Insert the content of a YAML list into config.yml """ - def __init__(self, function, depth): - self.function = function - self.depth = depth - def write(self, output_filehandle): miniyaml.render(output_filehandle, self.function(), self.depth) +def horizontal_rule(): + return "".join("#" * 78) + + +class Header(object): + + def __init__(self, title, summary=None): + self.title = title + self.summary_lines = summary or [] + + def write(self, output_filehandle): + text_lines = [self.title] + self.summary_lines + comment_lines = ["# " + x for x in text_lines] + lines = miniutils.sandwich([horizontal_rule()], comment_lines) + + for line in filter(None, lines): + output_filehandle.write(line + "\n") + + # Order of this list matters to the generated config.yml. YAML_SOURCES = [ File("header-section.yml"), @@ -64,30 +80,35 @@ YAML_SOURCES = [ File("linux-binary-build-defaults.yml"), File("macos-binary-build-defaults.yml"), File("nightly-build-smoke-tests-defaults.yml"), - File("job-specs-header.yml"), + Header("Job specifications job specs"), Treegen(pytorch_build_definitions.add_build_env_defs, 0), File("job-specs-custom.yml"), Treegen(caffe2_build_definitions.add_caffe2_builds, 1), File("job-specs-html-update.yml"), - File("binary-build-specs-header.yml"), + Header("Binary build specs individual job specifications"), Treegen(binary_build_definitions.add_binary_build_specs, 1), - File("binary-build-tests-header.yml"), + Header( + "Binary build tests", [ + "These are the smoke tests run right after the build, before the upload.", + "If these fail, the upload doesn't happen." + ] + ), Treegen(binary_build_definitions.add_binary_build_tests, 1), File("binary-build-tests.yml"), - File("binary-build-uploads-header.yml"), + Header("Binary build uploads"), Treegen(binary_build_definitions.add_binary_build_uploads, 1), - File("smoke-test-specs-header.yml"), + Header("Smoke test specs individual job specifications"), Treegen(binary_build_definitions.add_smoke_test_specs, 1), File("workflows.yml"), Listgen(pytorch_build_definitions.get_workflow_list, 3), File("workflows-pytorch-macos-builds.yml"), Listgen(caffe2_build_definitions.get_caffe2_workflows, 3), File("workflows-binary-builds-smoke-subset.yml"), - File("workflows-binary-smoke-header.yml"), + Header("Daily smoke test trigger"), Treegen(binary_build_definitions.add_binary_smoke_test_jobs, 1), - File("workflows-binary-build-header.yml"), + Header("Daily binary build trigger"), Treegen(binary_build_definitions.add_binary_build_jobs, 1), - File("workflows-nightly-tests-header.yml"), + Header("Nightly tests"), Listgen(binary_build_definitions.get_nightly_tests, 3), File("workflows-nightly-uploads-header.yml"), Listgen(binary_build_definitions.get_nightly_uploads, 3), diff --git a/.circleci/regenerate.sh b/.circleci/regenerate.sh index 5afaadf..43eff74 100755 --- a/.circleci/regenerate.sh +++ b/.circleci/regenerate.sh @@ -3,4 +3,6 @@ # Allows this script to be invoked from any directory: cd $(dirname "$0") -./generate_config_yml.py > config.yml +NEW_FILE=$(mktemp) +./generate_config_yml.py > $NEW_FILE +cp $NEW_FILE config.yml diff --git a/.circleci/verbatim-sources/binary-build-specs-header.yml b/.circleci/verbatim-sources/binary-build-specs-header.yml deleted file mode 100644 index d6cbd36..0000000 --- a/.circleci/verbatim-sources/binary-build-specs-header.yml +++ /dev/null @@ -1,3 +0,0 @@ -############################################################################## -# Binary build specs individual job specifications -############################################################################## diff --git a/.circleci/verbatim-sources/binary-build-tests-header.yml b/.circleci/verbatim-sources/binary-build-tests-header.yml deleted file mode 100644 index 379f3b5..0000000 --- a/.circleci/verbatim-sources/binary-build-tests-header.yml +++ /dev/null @@ -1,4 +0,0 @@ - # Binary build tests - # These are the smoke tests run right after the build, before the upload. If - # these fail, the upload doesn't happen - ############################################################################# diff --git a/.circleci/verbatim-sources/binary-build-uploads-header.yml b/.circleci/verbatim-sources/binary-build-uploads-header.yml deleted file mode 100644 index d316695..0000000 --- a/.circleci/verbatim-sources/binary-build-uploads-header.yml +++ /dev/null @@ -1,2 +0,0 @@ - # Binary build uploads - ############################################################################# diff --git a/.circleci/verbatim-sources/header-section.yml b/.circleci/verbatim-sources/header-section.yml index a178109..6febfb9 100644 --- a/.circleci/verbatim-sources/header-section.yml +++ b/.circleci/verbatim-sources/header-section.yml @@ -5,8 +5,8 @@ # https://github.com/pytorch/ossci-job-dsl/blob/master/src/main/groovy/ossci/pytorch/DockerVersion.groovy and # https://github.com/pytorch/ossci-job-dsl/blob/master/src/main/groovy/ossci/caffe2/DockerVersion.groovy, # and then update DOCKER_IMAGE_VERSION at the top of the following files: -# * cimodel/pytorch_build_definitions.py -# * cimodel/caffe2_build_definitions.py +# * cimodel/data/pytorch_build_definitions.py +# * cimodel/data/caffe2_build_definitions.py docker_config_defaults: &docker_config_defaults user: jenkins diff --git a/.circleci/verbatim-sources/job-specs-header.yml b/.circleci/verbatim-sources/job-specs-header.yml deleted file mode 100644 index 2d5618f..0000000 --- a/.circleci/verbatim-sources/job-specs-header.yml +++ /dev/null @@ -1,5 +0,0 @@ -############################################################################## -############################################################################## -# Job specifications job specs -############################################################################## -############################################################################## diff --git a/.circleci/verbatim-sources/smoke-test-specs-header.yml b/.circleci/verbatim-sources/smoke-test-specs-header.yml deleted file mode 100644 index 8817d33..0000000 --- a/.circleci/verbatim-sources/smoke-test-specs-header.yml +++ /dev/null @@ -1,4 +0,0 @@ - -############################################################################## -# Smoke test specs individual job specifications -############################################################################## diff --git a/.circleci/verbatim-sources/workflows-binary-smoke-header.yml b/.circleci/verbatim-sources/workflows-binary-smoke-header.yml deleted file mode 100644 index a3c7752..0000000 --- a/.circleci/verbatim-sources/workflows-binary-smoke-header.yml +++ /dev/null @@ -1,3 +0,0 @@ -############################################################################## -# Daily smoke test trigger -############################################################################## diff --git a/.circleci/verbatim-sources/workflows-nightly-tests-header.yml b/.circleci/verbatim-sources/workflows-nightly-tests-header.yml deleted file mode 100644 index 285493f..0000000 --- a/.circleci/verbatim-sources/workflows-nightly-tests-header.yml +++ /dev/null @@ -1 +0,0 @@ - # Nightly tests diff --git a/.travis.yml b/.travis.yml index 5c181bd..a46fdb6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,11 +15,6 @@ matrix: - name: "Ensure consistent CircleCI YAML" python: "3.6" dist: xenial - install: - - sudo add-apt-repository universe - - sudo apt update - - sudo apt install graphviz - - pip3 install pygraphviz script: cd .circleci && ./ensure-consistency.py - name: "Python 2.7 Lint" python: "2.7" -- 2.7.4