Furthermore, consistency is enforced within the YAML config itself, by using a single source of data to generate
multiple parts of the file.
-See https://github.com/pytorch/pytorch/issues/17038
+* Facilitates one-off culling/enabling of CI configs for testing PRs on special targets
+
+Also see https://github.com/pytorch/pytorch/issues/17038
Future direction
from collections import OrderedDict
-from cimodel.conf_tree import ConfigNode
+from cimodel.lib.conf_tree import ConfigNode
+import cimodel.data.dimensions as dimensions
LINKING_DIMENSIONS = [
return "cpu" if not cuda_version else "cu" + cuda_version
-CUDA_VERSIONS = [
- None, # cpu build
- "80",
- "90",
- "100",
-]
-
-STANDARD_PYTHON_VERSIONS = [
- "2.7",
- "3.5",
- "3.6",
- "3.7",
-]
-
-
CONFIG_TREE_DATA = OrderedDict(
- linux=(CUDA_VERSIONS, OrderedDict(
+ linux=(dimensions.CUDA_VERSIONS, OrderedDict(
manywheel=[
"2.7m",
"2.7mu",
"3.6m",
"3.7m",
],
- conda=STANDARD_PYTHON_VERSIONS,
+ conda=dimensions.STANDARD_PYTHON_VERSIONS,
libtorch=[
"2.7m",
]
)),
macos=([None], OrderedDict(
- wheel=STANDARD_PYTHON_VERSIONS,
- conda=STANDARD_PYTHON_VERSIONS,
+ wheel=dimensions.STANDARD_PYTHON_VERSIONS,
+ conda=dimensions.STANDARD_PYTHON_VERSIONS,
libtorch=[
"2.7",
],
from collections import OrderedDict
-import cimodel.conf_tree as conf_tree
-import cimodel.miniutils as miniutils
-import cimodel.make_build_configs as make_build_configs
-import cimodel.visualization as visualization
+import cimodel.data.binary_build_data as binary_build_data
+import cimodel.lib.conf_tree as conf_tree
+import cimodel.lib.miniutils as miniutils
+import cimodel.lib.visualization as visualization
class Conf(object):
self.libtorch_variant = libtorch_variant
def gen_build_env_parms(self):
- return [self.pydistro] + self.parms + [make_build_configs.get_processor_arch_name(self.cuda_version)]
+ return [self.pydistro] + self.parms + [binary_build_data.get_processor_arch_name(self.cuda_version)]
def gen_docker_image(self):
def gen_yaml_tree(self, build_or_test):
- env_dict = OrderedDict({
- "BUILD_ENVIRONMENT": miniutils.quote(" ".join(self.gen_build_env_parms())),
- })
+ env_tuples = [("BUILD_ENVIRONMENT", miniutils.quote(" ".join(self.gen_build_env_parms())))]
if self.libtorch_variant:
- env_dict["LIBTORCH_VARIANT"] = miniutils.quote(self.libtorch_variant)
+ env_tuples.append(("LIBTORCH_VARIANT", miniutils.quote(self.libtorch_variant)))
- os_word_substitution = {
- "macos": "mac",
- }
-
- os_name = miniutils.override(self.os, os_word_substitution)
-
- d = {
- "environment": env_dict,
- "<<": "*" + "_".join([self.get_name_prefix(), os_name, build_or_test]),
- }
+ os_name = miniutils.override(self.os, {"macos": "mac"})
+ d = {"<<": "*" + "_".join([self.get_name_prefix(), os_name, build_or_test])}
if build_or_test == "test":
- tuples = []
if not (self.smoke and self.os == "macos"):
- tuples.append(("DOCKER_IMAGE", self.gen_docker_image()))
+ env_tuples.append(("DOCKER_IMAGE", self.gen_docker_image()))
if self.cuda_version:
- tuples.append(("USE_CUDA_DOCKER_RUNTIME", miniutils.quote("1")))
-
- for (k, v) in tuples:
- env_dict[k] = v
+ env_tuples.append(("USE_CUDA_DOCKER_RUNTIME", miniutils.quote("1")))
else:
if self.os == "linux" and build_or_test != "upload":
d["docker"] = [{"image": self.gen_docker_image()}]
+ d["environment"] = OrderedDict(env_tuples)
+
if build_or_test == "test":
if self.cuda_version:
d["resource_class"] = "gpu.medium"
def get_root(smoke, name):
- return make_build_configs.TopLevelNode(
+ return binary_build_data.TopLevelNode(
name,
- make_build_configs.CONFIG_TREE_DATA,
+ binary_build_data.CONFIG_TREE_DATA,
smoke,
)
from collections import OrderedDict
-import cimodel.dimensions as dimensions
-import cimodel.miniutils as miniutils
-from cimodel.conf_tree import Ver
+import cimodel.data.dimensions as dimensions
+import cimodel.lib.miniutils as miniutils
+from cimodel.lib.conf_tree import Ver
DOCKER_IMAGE_PATH_BASE = "308535385114.dkr.ecr.us-east-1.amazonaws.com/caffe2/"
tuples.append(("BUILD_IOS", miniutils.quote("1")))
if self.phase == "test":
- use_cuda_docker = self.compiler.name == "cuda"
- if use_cuda_docker:
+ # TODO cuda should not be considered a compiler
+ if self.compiler.name == "cuda":
tuples.append(("USE_CUDA_DOCKER_RUNTIME", miniutils.quote("1")))
- if not self.distro.name == "macos":
- tuples.append(("DOCKER_IMAGE", self.gen_docker_image()))
-
- if self.is_build_only():
- if not self.distro.name == "macos":
- tuples.append(("BUILD_ONLY", miniutils.quote("1")))
-
- # TODO: not sure we need the distinction between system and homebrew anymore. Our python handling in cmake
- # and setuptools is more robust now than when we first had these.
if self.distro.name == "macos":
- tuples.append(("PYTHON_INSTALLATION", miniutils.quote("system")))
tuples.append(("PYTHON_VERSION", miniutils.quote("2")))
- env_dict = OrderedDict(tuples)
+ else:
+ tuples.append(("DOCKER_IMAGE", self.gen_docker_image()))
+ if self.is_build_only():
+ tuples.append(("BUILD_ONLY", miniutils.quote("1")))
- d = OrderedDict([
- ("environment", env_dict),
- ])
+ d = OrderedDict({"environment": OrderedDict(tuples)})
if self.phase == "test":
- is_large = self.compiler.name != "cuda"
-
- resource_class = "large" if is_large else "gpu.medium"
+ resource_class = "large" if self.compiler.name != "cuda" else "gpu.medium"
d["resource_class"] = resource_class
d["<<"] = "*" + "_".join(["caffe2", self.get_platform(), self.phase, "defaults"])
--- /dev/null
+#!/usr/bin/env python3
+
+
+PHASES = ["build", "test"]
+
+CUDA_VERSIONS = [
+ None, # cpu build
+ "80",
+ "90",
+ "100",
+]
+
+STANDARD_PYTHON_VERSIONS = [
+ "2.7",
+ "3.5",
+ "3.6",
+ "3.7",
+]
from collections import OrderedDict
-import cimodel.conf_tree as conf_tree
-import cimodel.dimensions as dimensions
-import cimodel.miniutils as miniutils
-import cimodel.visualization as visualization
-from cimodel.conf_tree import ConfigNode
+import cimodel.data.dimensions as dimensions
+import cimodel.lib.conf_tree as conf_tree
+import cimodel.lib.miniutils as miniutils
+import cimodel.lib.visualization as visualization
+from cimodel.lib.conf_tree import ConfigNode
DOCKER_IMAGE_PATH_BASE = "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/"
def gen_workflow_yaml_item(self, phase):
- val = OrderedDict()
- dependency_build = self.parent_build
- val["requires"] = [dependency_build.gen_build_name("build")]
-
- return {self.gen_build_name(phase): val}
+ return {self.gen_build_name(phase): {"requires": [self.parent_build.gen_build_name("build")]}}
def gen_build_name(self, _):
return self.name
for parms, gpu in extra_parms:
c = Conf(
- "xenial",
+ xenial_parent_config.distro,
["py3"] + parms,
pyver="3.6",
- cuda_version="8",
+ cuda_version=xenial_parent_config.cuda_version,
restrict_phases=["test"],
gpu_resource=gpu,
parent_build=xenial_parent_config,
return configs
-# TODO make the schema consistent between "trusty" and "xenial"
+def X(val):
+ """
+ Compact way to write a leaf node
+ """
+ return val, []
+
+
CONFIG_TREE_DATA = [
("trusty", [
- ("2.7.9", []),
- ("2.7", []),
- ("3.5", []),
- ("3.6", [
- ("gcc4.8", []),
- ("gcc5.4", [False, True]),
- ("gcc7", []),
+ (None, [
+ X("2.7.9"),
+ X("2.7"),
+ X("3.5"),
+ X("nightly"),
+ ]),
+ ("gcc", [
+ ("4.8", [X("3.6")]),
+ ("5.4", [("3.6", [X(False), X(True)])]),
+ ("7", [X("3.6")]),
]),
- ("nightly", []),
]),
("xenial", [
("clang", [
- ("5", [("3.6", [])]),
+ ("5", [X("3.6")]),
]),
("cuda", [
- ("8", [("3.6", [])]),
+ ("8", [X("3.6")]),
("9", [
# Note there are magic strings here
# https://github.com/pytorch/pytorch/blob/master/.jenkins/pytorch/build.sh#L21
# and
# https://github.com/pytorch/pytorch/blob/master/.jenkins/pytorch/build.sh#L153
# (from https://github.com/pytorch/pytorch/pull/17323#discussion_r259453144)
- ("2.7", []),
- ("3.6", []),
+ X("2.7"),
+ X("3.6"),
]),
- ("9.2", [("3.6", [])]),
- ("10", [("3.6", [])]),
+ ("9.2", [X("3.6")]),
+ ("10", [X("3.6")]),
]),
]),
]
return configs_list
-class TopLevelNode(ConfigNode):
- def __init__(self, node_name, config_tree_data):
- super(TopLevelNode, self).__init__(None, node_name)
-
- self.config_tree_data = config_tree_data
-
- def get_children(self):
- return [DistroConfigNode(self, d, p) for (d, p) in self.config_tree_data]
-
+class TreeConfigNode(ConfigNode):
+ def __init__(self, parent, node_name, subtree):
+ super(TreeConfigNode, self).__init__(parent, self.modify_label(node_name))
+ self.subtree = subtree
+ self.init2(node_name)
-class DistroConfigNode(ConfigNode):
- def __init__(self, parent, distro_name, subtree):
- super(DistroConfigNode, self).__init__(parent, distro_name)
+ def modify_label(self, label):
+ return label
- self.subtree = subtree
- self.props["distro_name"] = distro_name
+ def init2(self, node_name):
+ pass
def get_children(self):
+ return [self.child_constructor()(self, k, v) for (k, v) in self.subtree]
- if self.find_prop("distro_name") == "trusty":
- return [PyVerConfigNode(self, k, v) for k, v in self.subtree]
- else:
- return [XenialCompilerConfigNode(self, v, subtree) for (v, subtree) in self.subtree]
+class TopLevelNode(TreeConfigNode):
+ def __init__(self, node_name, subtree):
+ super(TopLevelNode, self).__init__(None, node_name, subtree)
-class PyVerConfigNode(ConfigNode):
- def __init__(self, parent, pyver, subtree):
- super(PyVerConfigNode, self).__init__(parent, pyver)
+ def child_constructor(self):
+ return DistroConfigNode
- self.subtree = subtree
- self.props["pyver"] = pyver
-
- self.props["abbreviated_pyver"] = get_major_pyver(pyver)
- def get_children(self):
- return [CompilerConfigNode(self, v, xla_options) for (v, xla_options) in self.subtree]
+class DistroConfigNode(TreeConfigNode):
+ def init2(self, node_name):
+ self.props["distro_name"] = node_name
+ def child_constructor(self):
+ distro = self.find_prop("distro_name")
+ return TrustyCompilerConfigNode if distro == "trusty" else XenialCompilerConfigNode
-class CompilerConfigNode(ConfigNode):
- def __init__(self, parent, compiler_name, subtree):
- super(CompilerConfigNode, self).__init__(parent, compiler_name)
- self.props["compiler_name"] = compiler_name
+class TrustyCompilerConfigNode(TreeConfigNode):
- self.subtree = subtree
+ def modify_label(self, label):
+ return label or "<unspecified>"
- def get_children(self):
- return [XlaConfigNode(self, v) for v in self.subtree]
+ def init2(self, node_name):
+ self.props["compiler_name"] = node_name
+ def child_constructor(self):
+ return TrustyCompilerVersionConfigNode if self.props["compiler_name"] else PyVerConfigNode
-class XenialCompilerConfigNode(ConfigNode):
- def __init__(self, parent, compiler_name, subtree):
- super(XenialCompilerConfigNode, self).__init__(parent, compiler_name)
- self.props["compiler_name"] = compiler_name
+class TrustyCompilerVersionConfigNode(TreeConfigNode):
- self.subtree = subtree
+ def init2(self, node_name):
+ self.props["compiler_version"] = node_name
- def get_children(self):
- return [XenialCompilerVersionConfigNode(self, k, v) for (k, v) in self.subtree]
+ def child_constructor(self):
+ return PyVerConfigNode
-class XenialCompilerVersionConfigNode(ConfigNode):
- def __init__(self, parent, compiler_version, subtree):
- super(XenialCompilerVersionConfigNode, self).__init__(parent, compiler_version)
+class PyVerConfigNode(TreeConfigNode):
+ def init2(self, node_name):
+ self.props["pyver"] = node_name
+ self.props["abbreviated_pyver"] = get_major_pyver(node_name)
- self.subtree = subtree
+ def child_constructor(self):
+ return XlaConfigNode
- self.props["compiler_version"] = compiler_version
- def get_children(self):
- return [XenialPythonVersionConfigNode(self, v) for (v, _) in self.subtree]
+class XlaConfigNode(TreeConfigNode):
+ def modify_label(self, label):
+ return "XLA=" + str(label)
+ def init2(self, node_name):
+ self.props["is_xla"] = node_name
-class XenialPythonVersionConfigNode(ConfigNode):
- def __init__(self, parent, python_version):
- super(XenialPythonVersionConfigNode, self).__init__(parent, python_version)
- self.props["pyver"] = python_version
- self.props["abbreviated_pyver"] = get_major_pyver(python_version)
+class XenialCompilerConfigNode(TreeConfigNode):
- def get_children(self):
- return []
+ def init2(self, node_name):
+ self.props["compiler_name"] = node_name
+ def child_constructor(self):
+ return XenialCompilerVersionConfigNode
-class XlaConfigNode(ConfigNode):
- def __init__(self, parent, xla_enabled):
- super(XlaConfigNode, self).__init__(parent, "XLA=" + str(xla_enabled))
- self.props["is_xla"] = xla_enabled
+class XenialCompilerVersionConfigNode(TreeConfigNode):
+ def init2(self, node_name):
+ self.props["compiler_version"] = node_name
- def get_children(self):
- return []
+ def child_constructor(self):
+ return PyVerConfigNode
def instantiate_configs():
python_version = None
if distro_name == "xenial":
python_version = fc.find_prop("pyver")
-
- if distro_name == "xenial":
parms_list = [fc.find_prop("abbreviated_pyver")]
else:
parms_list = ["py" + fc.find_prop("pyver")]
+ compiler_name = fc.find_prop("compiler_name")
+
cuda_version = None
- if fc.find_prop("compiler_name") == "cuda":
+ if compiler_name == "cuda":
cuda_version = fc.find_prop("compiler_version")
- compiler_name = fc.find_prop("compiler_name")
- if compiler_name and compiler_name != "cuda":
+ elif compiler_name:
gcc_version = compiler_name + (fc.find_prop("compiler_version") or "")
parms_list.append(gcc_version)
mydict = OrderedDict()
config_list = instantiate_configs()
-
for c in config_list:
for phase in dimensions.PHASES:
x.append(conf_options.gen_workflow_yaml_item(phase))
# TODO convert to recursion
- for conf in conf_options.dependent_tests:
+ for conf in conf_options.get_dependents():
x.append(conf.gen_workflow_yaml_item("test"))
return x
+++ /dev/null
-#!/usr/bin/env python3
-
-
-PHASES = ["build", "test"]
def get_label(self):
return self.node_name
+ # noinspection PyMethodMayBeStatic
def get_children(self):
return []
def quote(s):
- return '"' + s + '"'
+ return sandwich('"', s)
+
+
+def sandwich(bread, jam):
+ return bread + jam + bread
def override(word, substitutions):
import colorsys
-import cimodel.conf_tree as conf_tree
+import cimodel.lib.conf_tree as conf_tree
def rgb2hex(rgb_tuple):
return "#" + "".join(map(to_hex, list(rgb_tuple)))
+def handle_missing_graphviz(f):
+ """
+ If the user has not installed pygraphviz, this causes
+ calls to the draw() method of the returned object to do nothing.
+ """
+ try:
+ import pygraphviz
+ return f
+
+ except ModuleNotFoundError:
+
+ class FakeGraph:
+ def draw(self, *args, **kwargs):
+ pass
+
+ return lambda _: FakeGraph()
+
+
+@handle_missing_graphviz
def generate_graph(toplevel_config_node):
"""
Traverses the graph once first just to find the max depth
for config in config_list:
max_depth = max(max_depth, config.get_depth())
- from pygraphviz import AGraph
-
# color the nodes using the max depth
+
+ from pygraphviz import AGraph
dot = AGraph()
def node_discovery_callback(node, sibling_index, sibling_count):
# https://github.com/pytorch/ossci-job-dsl/blob/master/src/main/groovy/ossci/pytorch/DockerVersion.groovy and
# https://github.com/pytorch/ossci-job-dsl/blob/master/src/main/groovy/ossci/caffe2/DockerVersion.groovy,
# and then update DOCKER_IMAGE_VERSION at the top of the following files:
-# * cimodel/pytorch_build_definitions.py
-# * cimodel/caffe2_build_definitions.py
+# * cimodel/data/pytorch_build_definitions.py
+# * cimodel/data/caffe2_build_definitions.py
docker_config_defaults: &docker_config_defaults
user: jenkins
##############################################################################
-##############################################################################
# Job specifications job specs
##############################################################################
-##############################################################################
version: 2
jobs:
pytorch_linux_trusty_py2_7_9_build:
resource_class: large
<<: *pytorch_linux_test_defaults
+ pytorch_linux_trusty_pynightly_build:
+ environment:
+ BUILD_ENVIRONMENT: pytorch-linux-trusty-pynightly-build
+ DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-trusty-pynightly:291"
+ <<: *pytorch_linux_build_defaults
+
+ pytorch_linux_trusty_pynightly_test:
+ environment:
+ BUILD_ENVIRONMENT: pytorch-linux-trusty-pynightly-test
+ DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-trusty-pynightly:291"
+ resource_class: large
+ <<: *pytorch_linux_test_defaults
+
pytorch_linux_trusty_py3_6_gcc4_8_build:
environment:
BUILD_ENVIRONMENT: pytorch-linux-trusty-py3.6-gcc4.8-build
resource_class: large
<<: *pytorch_linux_test_defaults
- pytorch_linux_trusty_pynightly_build:
- environment:
- BUILD_ENVIRONMENT: pytorch-linux-trusty-pynightly-build
- DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-trusty-pynightly:291"
- <<: *pytorch_linux_build_defaults
-
- pytorch_linux_trusty_pynightly_test:
- environment:
- BUILD_ENVIRONMENT: pytorch-linux-trusty-pynightly-test
- DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-trusty-pynightly:291"
- resource_class: large
- <<: *pytorch_linux_test_defaults
-
pytorch_linux_xenial_py3_clang5_asan_build:
environment:
BUILD_ENVIRONMENT: pytorch-linux-xenial-py3-clang5-asan-build
environment:
BUILD_ENVIRONMENT: caffe2-py2-ios-macos10.13-build
BUILD_IOS: "1"
- PYTHON_INSTALLATION: "system"
PYTHON_VERSION: "2"
<<: *caffe2_macos_build_defaults
caffe2_py2_system_macos10_13_build:
environment:
BUILD_ENVIRONMENT: caffe2-py2-system-macos10.13-build
- PYTHON_INSTALLATION: "system"
PYTHON_VERSION: "2"
<<: *caffe2_macos_build_defaults
BUILD_ENVIRONMENT: "libtorch 2.7 cpu"
<<: *binary_mac_build
- # Binary build tests
- # These are the smoke tests run right after the build, before the upload. If
- # these fail, the upload doesn't happen
- #############################################################################
+##############################################################################
+# Binary build tests
+# These are the smoke tests run right after the build, before the upload.
+# If these fail, the upload doesn't happen.
+##############################################################################
binary_linux_manywheel_2.7m_cpu_test:
environment:
BUILD_ENVIRONMENT: "manywheel 2.7m cpu"
# resource_class: gpu.medium
# <<: *binary_linux_test
- # Binary build uploads
- #############################################################################
+##############################################################################
+# Binary build uploads
+##############################################################################
binary_linux_manywheel_2.7m_cpu_upload:
environment:
BUILD_ENVIRONMENT: "manywheel 2.7m cpu"
BUILD_ENVIRONMENT: "libtorch 2.7 cpu"
<<: *binary_mac_upload
-
##############################################################################
# Smoke test specs individual job specifications
##############################################################################
- pytorch_linux_trusty_py3_5_test:
requires:
- pytorch_linux_trusty_py3_5_build
+ - pytorch_linux_trusty_pynightly_build
+ - pytorch_linux_trusty_pynightly_test:
+ requires:
+ - pytorch_linux_trusty_pynightly_build
- pytorch_linux_trusty_py3_6_gcc4_8_build
- pytorch_linux_trusty_py3_6_gcc4_8_test:
requires:
- pytorch_linux_trusty_py3_6_gcc7_test:
requires:
- pytorch_linux_trusty_py3_6_gcc7_build
- - pytorch_linux_trusty_pynightly_build
- - pytorch_linux_trusty_pynightly_test:
- requires:
- - pytorch_linux_trusty_pynightly_build
- pytorch_linux_xenial_py3_clang5_asan_build
- pytorch_linux_xenial_py3_clang5_asan_test:
requires:
- smoke_macos_conda_3.7_cpu
- smoke_macos_libtorch_2.7_cpu
-
##############################################################################
# Daily binary build trigger
##############################################################################
- binary_macos_conda_3.7_cpu_build
- binary_macos_libtorch_2.7_cpu_build
- # Nightly tests
+##############################################################################
+# Nightly tests
+##############################################################################
- binary_linux_manywheel_2.7m_cpu_test:
requires:
- binary_linux_manywheel_2.7m_cpu_build
"""
This script is the source of truth for config.yml.
Please see README.md in this directory for details.
-
-In this module,
"""
import os
import sys
-from collections import OrderedDict
+import shutil
+from collections import namedtuple, OrderedDict
-import cimodel.pytorch_build_definitions as pytorch_build_definitions
-import cimodel.binary_build_definitions as binary_build_definitions
-import cimodel.caffe2_build_definitions as caffe2_build_definitions
-import cimodel.miniyaml as miniyaml
+import cimodel.data.pytorch_build_definitions as pytorch_build_definitions
+import cimodel.data.binary_build_definitions as binary_build_definitions
+import cimodel.data.caffe2_build_definitions as caffe2_build_definitions
+import cimodel.lib.miniutils as miniutils
+import cimodel.lib.miniyaml as miniyaml
class File(object):
def write(self, output_filehandle):
with open(os.path.join("verbatim-sources", self.filename)) as fh:
- output_filehandle.write(fh.read())
+ shutil.copyfileobj(fh, output_filehandle)
+
+
+class FunctionGen(namedtuple('FunctionGen', 'function depth')):
+ __slots__ = ()
-class Treegen(object):
+class Treegen(FunctionGen):
"""
Insert the content of a YAML tree into config.yml
"""
- def __init__(self, function, depth):
- self.function = function
- self.depth = depth
def write(self, output_filehandle):
build_dict = OrderedDict()
miniyaml.render(output_filehandle, build_dict, self.depth)
-class Listgen(object):
+class Listgen(FunctionGen):
"""
Insert the content of a YAML list into config.yml
"""
- def __init__(self, function, depth):
- self.function = function
- self.depth = depth
-
def write(self, output_filehandle):
miniyaml.render(output_filehandle, self.function(), self.depth)
+def horizontal_rule():
+ return "".join("#" * 78)
+
+
+class Header(object):
+
+ def __init__(self, title, summary=None):
+ self.title = title
+ self.summary_lines = summary or []
+
+ def write(self, output_filehandle):
+ text_lines = [self.title] + self.summary_lines
+ comment_lines = ["# " + x for x in text_lines]
+ lines = miniutils.sandwich([horizontal_rule()], comment_lines)
+
+ for line in filter(None, lines):
+ output_filehandle.write(line + "\n")
+
+
# Order of this list matters to the generated config.yml.
YAML_SOURCES = [
File("header-section.yml"),
File("linux-binary-build-defaults.yml"),
File("macos-binary-build-defaults.yml"),
File("nightly-build-smoke-tests-defaults.yml"),
- File("job-specs-header.yml"),
+ Header("Job specifications job specs"),
Treegen(pytorch_build_definitions.add_build_env_defs, 0),
File("job-specs-custom.yml"),
Treegen(caffe2_build_definitions.add_caffe2_builds, 1),
File("job-specs-html-update.yml"),
- File("binary-build-specs-header.yml"),
+ Header("Binary build specs individual job specifications"),
Treegen(binary_build_definitions.add_binary_build_specs, 1),
- File("binary-build-tests-header.yml"),
+ Header(
+ "Binary build tests", [
+ "These are the smoke tests run right after the build, before the upload.",
+ "If these fail, the upload doesn't happen."
+ ]
+ ),
Treegen(binary_build_definitions.add_binary_build_tests, 1),
File("binary-build-tests.yml"),
- File("binary-build-uploads-header.yml"),
+ Header("Binary build uploads"),
Treegen(binary_build_definitions.add_binary_build_uploads, 1),
- File("smoke-test-specs-header.yml"),
+ Header("Smoke test specs individual job specifications"),
Treegen(binary_build_definitions.add_smoke_test_specs, 1),
File("workflows.yml"),
Listgen(pytorch_build_definitions.get_workflow_list, 3),
File("workflows-pytorch-macos-builds.yml"),
Listgen(caffe2_build_definitions.get_caffe2_workflows, 3),
File("workflows-binary-builds-smoke-subset.yml"),
- File("workflows-binary-smoke-header.yml"),
+ Header("Daily smoke test trigger"),
Treegen(binary_build_definitions.add_binary_smoke_test_jobs, 1),
- File("workflows-binary-build-header.yml"),
+ Header("Daily binary build trigger"),
Treegen(binary_build_definitions.add_binary_build_jobs, 1),
- File("workflows-nightly-tests-header.yml"),
+ Header("Nightly tests"),
Listgen(binary_build_definitions.get_nightly_tests, 3),
File("workflows-nightly-uploads-header.yml"),
Listgen(binary_build_definitions.get_nightly_uploads, 3),
# Allows this script to be invoked from any directory:
cd $(dirname "$0")
-./generate_config_yml.py > config.yml
+NEW_FILE=$(mktemp)
+./generate_config_yml.py > $NEW_FILE
+cp $NEW_FILE config.yml
+++ /dev/null
-##############################################################################
-# Binary build specs individual job specifications
-##############################################################################
+++ /dev/null
- # Binary build tests
- # These are the smoke tests run right after the build, before the upload. If
- # these fail, the upload doesn't happen
- #############################################################################
+++ /dev/null
- # Binary build uploads
- #############################################################################
# https://github.com/pytorch/ossci-job-dsl/blob/master/src/main/groovy/ossci/pytorch/DockerVersion.groovy and
# https://github.com/pytorch/ossci-job-dsl/blob/master/src/main/groovy/ossci/caffe2/DockerVersion.groovy,
# and then update DOCKER_IMAGE_VERSION at the top of the following files:
-# * cimodel/pytorch_build_definitions.py
-# * cimodel/caffe2_build_definitions.py
+# * cimodel/data/pytorch_build_definitions.py
+# * cimodel/data/caffe2_build_definitions.py
docker_config_defaults: &docker_config_defaults
user: jenkins
+++ /dev/null
-##############################################################################
-##############################################################################
-# Job specifications job specs
-##############################################################################
-##############################################################################
+++ /dev/null
-
-##############################################################################
-# Smoke test specs individual job specifications
-##############################################################################
+++ /dev/null
-##############################################################################
-# Daily smoke test trigger
-##############################################################################
+++ /dev/null
- # Nightly tests
- name: "Ensure consistent CircleCI YAML"
python: "3.6"
dist: xenial
- install:
- - sudo add-apt-repository universe
- - sudo apt update
- - sudo apt install graphviz
- - pip3 install pygraphviz
script: cd .circleci && ./ensure-consistency.py
- name: "Python 2.7 Lint"
python: "2.7"
projects / platform / upstream / pytorch.git / commitdiff