1 # -*- test-case-name: twisted.python.test.test_release -*-
2 # Copyright (c) Twisted Matrix Laboratories.
3 # See LICENSE for details.
6 Twisted's automated release system.
8 This module is only for use within Twisted's release system. If you are anyone
9 else, do not use it. The interface and behaviour will change without notice.
11 Only Linux is supported by this code. It should not be used by any tools
12 which must run on multiple platforms (eg the setup.py script).
16 from datetime import date
20 from tempfile import mkdtemp
23 from subprocess import PIPE, STDOUT, Popen
25 from twisted.python.versions import Version
26 from twisted.python.filepath import FilePath
27 from twisted.python.dist import twisted_subprojects
29 # This import is an example of why you shouldn't use this module unless you're
32 from twisted.lore.scripts import lore
36 # The offset between a year and the corresponding major version number.
42 Execute a vector of arguments.
44 @type args: C{list} of C{str}
45 @param args: A list of arguments, the first of which will be used as the
49 @return: All of the standard output.
51 @raise CommandFailed: when the program exited with a non-0 exit code.
53 process = Popen(args, stdout=PIPE, stderr=STDOUT)
54 stdout = process.stdout.read()
55 exitCode = process.wait()
57 raise CommandFailed(None, -exitCode, stdout)
59 raise CommandFailed(exitCode, None, stdout)
63 class CommandFailed(Exception):
65 Raised when a child process exits unsuccessfully.
67 @type exitStatus: C{int}
68 @ivar exitStatus: The exit status for the child process.
70 @type exitSignal: C{int}
71 @ivar exitSignal: The exit signal for the child process.
74 @ivar output: The bytes read from stdout and stderr of the child process.
76 def __init__(self, exitStatus, exitSignal, output):
77 Exception.__init__(self, exitStatus, exitSignal, output)
78 self.exitStatus = exitStatus
79 self.exitSignal = exitSignal
84 def _changeVersionInFile(old, new, filename):
86 Replace the C{old} version number with the C{new} one in the given
89 replaceInFile(filename, {old.base(): new.base()})
93 def getNextVersion(version, now=None):
95 Calculate the version number for a new release of Twisted based on
96 the previous version number.
98 @param version: The previous version number.
99 @param now: (optional) The current date.
101 # XXX: This has no way of incrementing the patch number. Currently, we
102 # don't need it. See bug 2915. Jonathan Lange, 2007-11-20.
105 major = now.year - VERSION_OFFSET
106 if major != version.major:
109 minor = version.minor + 1
110 return Version(version.package, major, minor, 0)
113 def changeAllProjectVersions(root, versionTemplate, today=None):
115 Change the version of all projects (including core and all subprojects).
117 If the current version of a project is pre-release, then also change the
118 versions in the current NEWS entries for that project.
120 @type root: L{FilePath}
121 @param root: The root of the Twisted source tree.
122 @type versionTemplate: L{Version}
123 @param versionTemplate: The version of all projects. The name will be
124 replaced for each respective project.
126 @param today: A YYYY-MM-DD formatted string. If not provided, defaults to
127 the current day, according to the system clock.
130 today = date.today().strftime('%Y-%m-%d')
131 for project in findTwistedProjects(root):
132 if project.directory.basename() == "twisted":
133 packageName = "twisted"
135 packageName = "twisted." + project.directory.basename()
136 oldVersion = project.getVersion()
137 newVersion = Version(packageName, versionTemplate.major,
138 versionTemplate.minor, versionTemplate.micro,
139 prerelease=versionTemplate.prerelease)
141 if oldVersion.prerelease:
142 builder = NewsBuilder()
143 builder._changeNewsVersion(
144 root.child("NEWS"), builder._getNewsName(project),
145 oldVersion, newVersion, today)
146 builder._changeNewsVersion(
147 project.directory.child("topfiles").child("NEWS"),
148 builder._getNewsName(project), oldVersion, newVersion,
151 # The placement of the top-level README with respect to other files (eg
152 # _version.py) is sufficiently different from the others that we just
153 # have to handle it specially.
154 if packageName == "twisted":
155 _changeVersionInFile(
156 oldVersion, newVersion, root.child('README').path)
158 project.updateVersion(newVersion)
163 class Project(object):
165 A representation of a project that has a version.
167 @ivar directory: A L{twisted.python.filepath.FilePath} pointing to the base
168 directory of a Twisted-style Python package. The package should contain
169 a C{_version.py} file and a C{topfiles} directory that contains a
173 def __init__(self, directory):
174 self.directory = directory
179 self.__class__.__name__, self.directory)
182 def getVersion(self):
184 @return: A L{Version} specifying the version number of the project
185 based on live python modules.
188 execfile(self.directory.child("_version.py").path, namespace)
189 return namespace["version"]
192 def updateVersion(self, version):
194 Replace the existing version numbers in _version.py and README files
195 with the specified version.
197 oldVersion = self.getVersion()
198 replaceProjectVersion(self.directory.child("_version.py").path,
200 _changeVersionInFile(
202 self.directory.child("topfiles").child("README").path)
206 def findTwistedProjects(baseDirectory):
208 Find all Twisted-style projects beneath a base directory.
210 @param baseDirectory: A L{twisted.python.filepath.FilePath} to look inside.
211 @return: A list of L{Project}.
214 for filePath in baseDirectory.walk():
215 if filePath.basename() == 'topfiles':
216 projectDirectory = filePath.parent()
217 projects.append(Project(projectDirectory))
222 def updateTwistedVersionInformation(baseDirectory, now):
224 Update the version information for Twisted and all subprojects to the
225 date-based version number.
227 @param baseDirectory: Where to look for Twisted. If None, the function
228 infers the information from C{twisted.__file__}.
229 @param now: The current date (as L{datetime.date}). If None, it defaults
232 for project in findTwistedProjects(baseDirectory):
233 project.updateVersion(getNextVersion(project.getVersion(), now=now))
236 def generateVersionFileData(version):
238 Generate the data to be placed into a _version.py file.
240 @param version: A version object.
242 if version.prerelease is not None:
243 prerelease = ", prerelease=%r" % (version.prerelease,)
247 # This is an auto-generated file. Do not edit it.
248 from twisted.python import versions
249 version = versions.Version(%r, %s, %s, %s%s)
250 ''' % (version.package, version.major, version.minor, version.micro, prerelease)
254 def replaceProjectVersion(filename, newversion):
256 Write version specification code into the given filename, which
257 sets the version to the given version number.
259 @param filename: A filename which is most likely a "_version.py"
260 under some Twisted project.
261 @param newversion: A version object.
263 # XXX - this should be moved to Project and renamed to writeVersionFile.
265 f = open(filename, 'w')
266 f.write(generateVersionFileData(newversion))
271 def replaceInFile(filename, oldToNew):
273 I replace the text `oldstr' with `newstr' in `filename' using science.
275 os.rename(filename, filename+'.bak')
276 f = open(filename+'.bak')
279 for k,v in oldToNew.items():
281 f = open(filename + '.new', 'w')
284 os.rename(filename+'.new', filename)
285 os.unlink(filename+'.bak')
289 class NoDocumentsFound(Exception):
291 Raised when no input documents are found.
296 class LoreBuilderMixin(object):
298 Base class for builders which invoke lore.
300 def lore(self, arguments):
302 Run lore with the given arguments.
304 @param arguments: A C{list} of C{str} giving command line arguments to
305 lore which should be used.
307 options = lore.Options()
308 options.parseOptions(["--null"] + arguments)
309 lore.runGivenOptions(options)
313 class DocBuilder(LoreBuilderMixin):
315 Generate HTML documentation for projects.
318 def build(self, version, resourceDir, docDir, template, apiBaseURL=None,
321 Build the documentation in C{docDir} with Lore.
323 Input files ending in .xhtml will be considered. Output will written as
326 @param version: the version of the documentation to pass to lore.
327 @type version: C{str}
329 @param resourceDir: The directory which contains the toplevel index and
330 stylesheet file for this section of documentation.
331 @type resourceDir: L{twisted.python.filepath.FilePath}
333 @param docDir: The directory of the documentation.
334 @type docDir: L{twisted.python.filepath.FilePath}
336 @param template: The template used to generate the documentation.
337 @type template: L{twisted.python.filepath.FilePath}
339 @type apiBaseURL: C{str} or C{NoneType}
340 @param apiBaseURL: A format string which will be interpolated with the
341 fully-qualified Python name for each API link. For example, to
342 generate the Twisted 8.0.0 documentation, pass
343 C{"http://twistedmatrix.com/documents/8.0.0/api/%s.html"}.
345 @param deleteInput: If True, the input documents will be deleted after
346 their output is generated.
347 @type deleteInput: C{bool}
349 @raise NoDocumentsFound: When there are no .xhtml files in the given
352 linkrel = self.getLinkrel(resourceDir, docDir)
353 inputFiles = docDir.globChildren("*.xhtml")
354 filenames = [x.path for x in inputFiles]
356 raise NoDocumentsFound("No input documents found in %s" % (docDir,))
357 if apiBaseURL is not None:
358 arguments = ["--config", "baseurl=" + apiBaseURL]
361 arguments.extend(["--config", "template=%s" % (template.path,),
362 "--config", "ext=.html",
363 "--config", "version=%s" % (version,),
364 "--linkrel", linkrel] + filenames)
367 for inputFile in inputFiles:
371 def getLinkrel(self, resourceDir, docDir):
373 Calculate a value appropriate for Lore's --linkrel option.
375 Lore's --linkrel option defines how to 'find' documents that are
376 linked to from TEMPLATE files (NOT document bodies). That is, it's a
377 prefix for links ('a' and 'link') in the template.
379 @param resourceDir: The directory which contains the toplevel index and
380 stylesheet file for this section of documentation.
381 @type resourceDir: L{twisted.python.filepath.FilePath}
383 @param docDir: The directory containing documents that must link to
385 @type docDir: L{twisted.python.filepath.FilePath}
387 if resourceDir != docDir:
388 return '/'.join(filePathDelta(docDir, resourceDir)) + "/"
394 class ManBuilder(LoreBuilderMixin):
396 Generate man pages of the different existing scripts.
399 def build(self, manDir):
401 Generate Lore input files from the man pages in C{manDir}.
403 Input files ending in .1 will be considered. Output will written as
406 @param manDir: The directory of the man pages.
407 @type manDir: L{twisted.python.filepath.FilePath}
409 @raise NoDocumentsFound: When there are no .1 files in the given
412 inputFiles = manDir.globChildren("*.1")
413 filenames = [x.path for x in inputFiles]
415 raise NoDocumentsFound("No manual pages found in %s" % (manDir,))
416 arguments = ["--input", "man",
418 "--config", "ext=-man.xhtml"] + filenames
423 class APIBuilder(object):
425 Generate API documentation from source files using
426 U{pydoctor<http://codespeak.net/~mwh/pydoctor/>}. This requires
427 pydoctor to be installed and usable (which means you won't be able to
428 use it with Python 2.3).
430 def build(self, projectName, projectURL, sourceURL, packagePath,
433 Call pydoctor's entry point with options which will generate HTML
434 documentation for the specified package's API.
436 @type projectName: C{str}
437 @param projectName: The name of the package for which to generate
440 @type projectURL: C{str}
441 @param projectURL: The location (probably an HTTP URL) of the project
444 @type sourceURL: C{str}
445 @param sourceURL: The location (probably an HTTP URL) of the root of
446 the source browser for the project.
448 @type packagePath: L{FilePath}
449 @param packagePath: The path to the top-level of the package named by
452 @type outputPath: L{FilePath}
453 @param outputPath: An existing directory to which the generated API
454 documentation will be written.
456 from pydoctor.driver import main
458 ["--project-name", projectName,
459 "--project-url", projectURL,
460 "--system-class", "pydoctor.twistedmodel.TwistedSystem",
461 "--project-base-dir", packagePath.parent().path,
462 "--html-viewsource-base", sourceURL,
463 "--add-package", packagePath.path,
464 "--html-output", outputPath.path,
465 "--html-write-function-pages", "--quiet", "--make-html"])
469 class BookBuilder(LoreBuilderMixin):
471 Generate the LaTeX and PDF documentation.
473 The book is built by assembling a number of LaTeX documents. Only the
474 overall document which describes how to assemble the documents is stored
475 in LaTeX in the source. The rest of the documentation is generated from
476 Lore input files. These are primarily XHTML files (of the particular
477 Lore subset), but man pages are stored in GROFF format. BookBuilder
478 expects all of its input to be Lore XHTML format, so L{ManBuilder}
479 should be invoked first if the man pages are to be included in the
480 result (this is determined by the book LaTeX definition file).
481 Therefore, a sample usage of BookBuilder may look something like this::
484 man.build(FilePath("doc/core/man"))
487 FilePath('doc/core/howto'),
488 [FilePath('doc/core/howto'), FilePath('doc/core/howto/tutorial'),
489 FilePath('doc/core/man'), FilePath('doc/core/specifications')],
490 FilePath('doc/core/howto/book.tex'), FilePath('/tmp/book.pdf'))
492 def run(self, command):
494 Execute a command in a child process and return the output.
496 @type command: C{str}
497 @param command: The shell command to run.
499 @raise CommandFailed: If the child process exits with an error.
501 return runCommand(command)
504 def buildTeX(self, howtoDir):
506 Build LaTeX files for lore input files in the given directory.
508 Input files ending in .xhtml will be considered. Output will written as
511 @type howtoDir: L{FilePath}
512 @param howtoDir: A directory containing lore input files.
514 @raise ValueError: If C{howtoDir} does not exist.
516 if not howtoDir.exists():
517 raise ValueError("%r does not exist." % (howtoDir.path,))
519 ["--output", "latex",
520 "--config", "section"] +
521 [child.path for child in howtoDir.globChildren("*.xhtml")])
524 def buildPDF(self, bookPath, inputDirectory, outputPath):
526 Build a PDF from the given a LaTeX book document.
528 @type bookPath: L{FilePath}
529 @param bookPath: The location of a LaTeX document defining a book.
531 @type inputDirectory: L{FilePath}
532 @param inputDirectory: The directory which the inputs of the book are
535 @type outputPath: L{FilePath}
536 @param outputPath: The location to which to write the resulting book.
538 if not bookPath.basename().endswith(".tex"):
539 raise ValueError("Book filename must end with .tex")
541 workPath = FilePath(mkdtemp())
543 startDir = os.getcwd()
545 os.chdir(inputDirectory.path)
548 "latex", "-interaction=nonstopmode",
549 "-output-directory=" + workPath.path,
552 # What I tell you three times is true!
553 # The first two invocations of latex on the book file allows it
554 # correctly create page numbers for in-text references. Why this is
555 # the case, I could not tell you. -exarkun
559 bookBaseWithoutExtension = bookPath.basename()[:-4]
560 dviPath = workPath.child(bookBaseWithoutExtension + ".dvi")
561 psPath = workPath.child(bookBaseWithoutExtension + ".ps")
562 pdfPath = workPath.child(bookBaseWithoutExtension + ".pdf")
564 "dvips", "-o", psPath.path, "-t", "letter", "-Ppdf",
566 self.run(["ps2pdf13", psPath.path, pdfPath.path])
567 pdfPath.moveTo(outputPath)
572 workPath.moveTo(bookPath.parent().child(workPath.basename()))
576 def build(self, baseDirectory, inputDirectories, bookPath, outputPath):
578 Build a PDF book from the given TeX book definition and directories
579 containing lore inputs.
581 @type baseDirectory: L{FilePath}
582 @param baseDirectory: The directory which the inputs of the book are
585 @type inputDirectories: C{list} of L{FilePath}
586 @param inputDirectories: The paths which contain lore inputs to be
589 @type bookPath: L{FilePath}
590 @param bookPath: The location of a LaTeX document defining a book.
592 @type outputPath: L{FilePath}
593 @param outputPath: The location to which to write the resulting book.
595 for inputDir in inputDirectories:
596 self.buildTeX(inputDir)
597 self.buildPDF(bookPath, baseDirectory, outputPath)
598 for inputDirectory in inputDirectories:
599 for child in inputDirectory.children():
600 if child.splitext()[1] == ".tex" and child != bookPath:
605 class NewsBuilder(object):
607 Generate the new section of a NEWS file.
609 The C{_FEATURE}, C{_BUGFIX}, C{_DOC}, C{_REMOVAL}, and C{_MISC}
610 attributes of this class are symbolic names for the news entry types
611 which are supported. Conveniently, they each also take on the value of
612 the file name extension which indicates a news entry of that type.
614 @cvar _headings: A C{dict} mapping one of the news entry types to the
615 heading to write out for that type of news entry.
617 @cvar _NO_CHANGES: A C{str} giving the text which appears when there are
618 no significant changes in a release.
620 @cvar _TICKET_HINT: A C{str} giving the text which appears at the top of
621 each news file and which should be kept at the top, not shifted down
622 with all the other content. Put another way, this is the text after
623 which the new news text is inserted.
626 _FEATURE = ".feature"
629 _REMOVAL = ".removal"
633 _FEATURE: "Features",
635 _DOC: "Improved Documentation",
636 _REMOVAL: "Deprecations and Removals",
640 _NO_CHANGES = "No significant changes have been made for this release.\n"
643 'Ticket numbers in this file can be looked up by visiting\n'
644 'http://twistedmatrix.com/trac/ticket/<number>\n'
649 Return today's date as a string in YYYY-MM-DD format.
651 return date.today().strftime('%Y-%m-%d')
654 def _findChanges(self, path, ticketType):
656 Load all the feature ticket summaries.
658 @param path: A L{FilePath} the direct children of which to search
661 @param ticketType: The type of news entries to search for. One of
662 L{NewsBuilder._FEATURE}, L{NewsBuilder._BUGFIX},
663 L{NewsBuilder._REMOVAL}, or L{NewsBuilder._MISC}.
665 @return: A C{list} of two-tuples. The first element is the ticket
666 number as an C{int}. The second element of each tuple is the
667 description of the feature.
670 for child in path.children():
671 base, ext = os.path.splitext(child.basename())
672 if ext == ticketType:
675 ' '.join(child.getContent().splitlines())))
680 def _formatHeader(self, header):
682 Format a header for a NEWS file.
684 A header is a title with '=' signs underlining it.
686 @param header: The header string to format.
688 @return: A C{str} containing C{header}.
690 return header + '\n' + '=' * len(header) + '\n\n'
693 def _writeHeader(self, fileObj, header):
695 Write a version header to the given file.
697 @param fileObj: A file-like object to which to write the header.
698 @param header: The header to write to the file.
701 fileObj.write(self._formatHeader(header))
704 def _writeSection(self, fileObj, header, tickets):
706 Write out one section (features, bug fixes, etc) to the given file.
708 @param fileObj: A file-like object to which to write the news section.
710 @param header: The header for the section to write.
713 @param tickets: A C{list} of ticket information of the sort returned
714 by L{NewsBuilder._findChanges}.
720 for (ticket, description) in tickets:
721 reverse.setdefault(description, []).append(ticket)
722 for description in reverse:
723 reverse[description].sort()
724 reverse = reverse.items()
725 reverse.sort(key=lambda (descr, tickets): tickets[0])
727 fileObj.write(header + '\n' + '-' * len(header) + '\n')
728 for (description, relatedTickets) in reverse:
729 ticketList = ', '.join([
730 '#' + str(ticket) for ticket in relatedTickets])
731 entry = ' - %s (%s)' % (description, ticketList)
732 entry = textwrap.fill(entry, subsequent_indent=' ')
733 fileObj.write(entry + '\n')
737 def _writeMisc(self, fileObj, header, tickets):
739 Write out a miscellaneous-changes section to the given file.
741 @param fileObj: A file-like object to which to write the news section.
743 @param header: The header for the section to write.
746 @param tickets: A C{list} of ticket information of the sort returned
747 by L{NewsBuilder._findChanges}.
752 fileObj.write(header + '\n' + '-' * len(header) + '\n')
753 formattedTickets = []
754 for (ticket, ignored) in tickets:
755 formattedTickets.append('#' + str(ticket))
756 entry = ' - ' + ', '.join(formattedTickets)
757 entry = textwrap.fill(entry, subsequent_indent=' ')
758 fileObj.write(entry + '\n\n')
761 def build(self, path, output, header):
763 Load all of the change information from the given directory and write
764 it out to the given output file.
766 @param path: A directory (probably a I{topfiles} directory) containing
767 change information in the form of <ticket>.<change type> files.
768 @type path: L{FilePath}
770 @param output: The NEWS file to which the results will be prepended.
771 @type output: L{FilePath}
773 @param header: The top-level header to use when writing the news.
777 for part in (self._FEATURE, self._BUGFIX, self._DOC, self._REMOVAL):
778 tickets = self._findChanges(path, part)
780 changes.append((part, tickets))
781 misc = self._findChanges(path, self._MISC)
783 oldNews = output.getContent()
784 newNews = output.sibling('NEWS.new').open('w')
785 if oldNews.startswith(self._TICKET_HINT):
786 newNews.write(self._TICKET_HINT)
787 oldNews = oldNews[len(self._TICKET_HINT):]
789 self._writeHeader(newNews, header)
791 for (part, tickets) in changes:
792 self._writeSection(newNews, self._headings.get(part), tickets)
794 newNews.write(self._NO_CHANGES)
796 self._writeMisc(newNews, self._headings.get(self._MISC), misc)
798 newNews.write(oldNews)
800 output.sibling('NEWS.new').moveTo(output)
803 def _getNewsName(self, project):
805 Return the name of C{project} that should appear in NEWS.
807 @param project: A L{Project}
808 @return: The name of C{project}.
810 name = project.directory.basename().title()
811 if name == 'Twisted':
816 def _iterProjects(self, baseDirectory):
818 Iterate through the Twisted projects in C{baseDirectory}, yielding
819 everything we need to know to build news for them.
821 Yields C{topfiles}, C{news}, C{name}, C{version} for each sub-project
822 in reverse-alphabetical order. C{topfile} is the L{FilePath} for the
823 topfiles directory, C{news} is the L{FilePath} for the NEWS file,
824 C{name} is the nice name of the project (as should appear in the NEWS
825 file), C{version} is the current version string for that project.
827 @param baseDirectory: A L{FilePath} representing the root directory
828 beneath which to find Twisted projects for which to generate
829 news (see L{findTwistedProjects}).
830 @type baseDirectory: L{FilePath}
832 # Get all the subprojects to generate news for
833 projects = findTwistedProjects(baseDirectory)
834 # And order them alphabetically for ease of reading
835 projects.sort(key=lambda proj: proj.directory.path)
836 # And generate them backwards since we write news by prepending to
840 for aggregateNews in [False, True]:
841 for project in projects:
842 topfiles = project.directory.child("topfiles")
844 news = baseDirectory.child("NEWS")
846 news = topfiles.child("NEWS")
847 name = self._getNewsName(project)
848 version = project.getVersion()
849 yield topfiles, news, name, version
852 def buildAll(self, baseDirectory):
854 Find all of the Twisted subprojects beneath C{baseDirectory} and update
855 their news files from the ticket change description files in their
856 I{topfiles} directories and update the news file in C{baseDirectory}
857 with all of the news.
859 @param baseDirectory: A L{FilePath} representing the root directory
860 beneath which to find Twisted projects for which to generate
861 news (see L{findTwistedProjects}).
863 today = self._today()
864 for topfiles, news, name, version in self._iterProjects(baseDirectory):
867 "Twisted %s %s (%s)" % (name, version.base(), today))
870 def _changeNewsVersion(self, news, name, oldVersion, newVersion, today):
872 Change all references to the current version number in a NEWS file to
873 refer to C{newVersion} instead.
875 @param news: The NEWS file to change.
876 @type news: L{FilePath}
877 @param name: The name of the project to change.
879 @param oldVersion: The old version of the project.
880 @type oldVersion: L{Version}
881 @param newVersion: The new version of the project.
882 @type newVersion: L{Version}
883 @param today: A YYYY-MM-DD string representing today's date.
886 newHeader = self._formatHeader(
887 "Twisted %s %s (%s)" % (name, newVersion.base(), today))
888 expectedHeaderRegex = re.compile(
889 r"Twisted %s %s \(\d{4}-\d\d-\d\d\)\n=+\n\n" % (
890 re.escape(name), re.escape(oldVersion.base())))
891 oldNews = news.getContent()
892 match = expectedHeaderRegex.search(oldNews)
894 oldHeader = match.group()
895 replaceInFile(news.path, {oldHeader: newHeader})
898 def main(self, args):
900 Build all news files.
902 @param args: The command line arguments to process. This must contain
903 one string, the path to the base of the Twisted checkout for which
905 @type args: C{list} of C{str}
908 sys.exit("Must specify one argument: the path to the Twisted checkout")
909 self.buildAll(FilePath(args[0]))
913 def filePathDelta(origin, destination):
915 Return a list of strings that represent C{destination} as a path relative
918 It is assumed that both paths represent directories, not files. That is to
919 say, the delta of L{twisted.python.filepath.FilePath} /foo/bar to
920 L{twisted.python.filepath.FilePath} /foo/baz will be C{../baz},
923 @type origin: L{twisted.python.filepath.FilePath}
924 @param origin: The origin of the relative path.
926 @type destination: L{twisted.python.filepath.FilePath}
927 @param destination: The destination of the relative path.
930 path1 = origin.path.split(os.sep)
931 path2 = destination.path.split(os.sep)
932 for elem1, elem2 in zip(path1, path2):
937 path = [".."] * (len(path1) - commonItems)
938 return path + path2[commonItems:]
942 class DistributionBuilder(object):
944 A builder of Twisted distributions.
946 This knows how to build tarballs for Twisted and all of its subprojects.
948 from twisted.python.dist import twisted_subprojects as subprojects
950 def __init__(self, rootDirectory, outputDirectory, templatePath=None,
953 Create a distribution builder.
955 @param rootDirectory: root of a Twisted export which will populate
957 @type rootDirectory: L{FilePath}.
959 @param outputDirectory: The directory in which to create the tarballs.
960 @type outputDirectory: L{FilePath}
962 @param templatePath: Path to the template file that is used for the
964 @type templatePath: L{FilePath}
966 @type apiBaseURL: C{str} or C{NoneType}
967 @param apiBaseURL: A format string which will be interpolated with the
968 fully-qualified Python name for each API link. For example, to
969 generate the Twisted 8.0.0 documentation, pass
970 C{"http://twistedmatrix.com/documents/8.0.0/api/%s.html"}.
972 self.rootDirectory = rootDirectory
973 self.outputDirectory = outputDirectory
974 self.templatePath = templatePath
975 self.apiBaseURL = apiBaseURL
976 self.manBuilder = ManBuilder()
977 self.docBuilder = DocBuilder()
980 def _buildDocInDir(self, path, version, howtoPath):
982 Generate documentation in the given path, building man pages first if
983 necessary and swallowing errors (so that directories without lore
984 documentation in them are ignored).
986 @param path: The path containing documentation to build.
987 @type path: L{FilePath}
988 @param version: The version of the project to include in all generated
990 @type version: C{str}
991 @param howtoPath: The "resource path" as L{DocBuilder} describes it.
992 @type howtoPath: L{FilePath}
994 if self.templatePath is None:
995 self.templatePath = self.rootDirectory.descendant(
996 ["doc", "core", "howto", "template.tpl"])
997 if path.basename() == "man":
998 self.manBuilder.build(path)
1001 self.docBuilder.build(version, howtoPath, path,
1002 self.templatePath, self.apiBaseURL, True)
1003 except NoDocumentsFound:
1007 def buildTwisted(self, version):
1009 Build the main Twisted distribution in C{Twisted-<version>.tar.bz2}.
1011 bin/admin is excluded.
1013 @type version: C{str}
1014 @param version: The version of Twisted to build.
1016 @return: The tarball file.
1017 @rtype: L{FilePath}.
1019 releaseName = "Twisted-%s" % (version,)
1020 buildPath = lambda *args: '/'.join((releaseName,) + args)
1022 outputFile = self.outputDirectory.child(releaseName + ".tar.bz2")
1023 tarball = tarfile.TarFile.open(outputFile.path, 'w:bz2')
1025 docPath = self.rootDirectory.child("doc")
1029 for subProjectDir in docPath.children():
1030 if subProjectDir.isdir():
1031 for child in subProjectDir.walk():
1032 self._buildDocInDir(child, version,
1033 subProjectDir.child("howto"))
1035 for binthing in self.rootDirectory.child("bin").children():
1036 # bin/admin should not be included.
1037 if binthing.basename() != "admin":
1038 tarball.add(binthing.path,
1039 buildPath("bin", binthing.basename()))
1041 for submodule in self.rootDirectory.child("twisted").children():
1042 if submodule.basename() == "plugins":
1043 for plugin in submodule.children():
1044 tarball.add(plugin.path, buildPath("twisted", "plugins",
1047 tarball.add(submodule.path, buildPath("twisted",
1048 submodule.basename()))
1050 for docDir in self.rootDirectory.child("doc").children():
1051 tarball.add(docDir.path, buildPath("doc", docDir.basename()))
1053 for toplevel in self.rootDirectory.children():
1054 if not toplevel.isdir():
1055 tarball.add(toplevel.path, buildPath(toplevel.basename()))
1062 def buildCore(self, version):
1064 Build a core distribution in C{TwistedCore-<version>.tar.bz2}.
1066 This is very similar to L{buildSubProject}, but core tarballs and the
1067 input are laid out slightly differently.
1069 - scripts are in the top level of the C{bin} directory.
1070 - code is included directly from the C{twisted} directory, excluding
1072 - all plugins except the subproject plugins are included.
1074 @type version: C{str}
1075 @param version: The version of Twisted to build.
1077 @return: The tarball file.
1078 @rtype: L{FilePath}.
1080 releaseName = "TwistedCore-%s" % (version,)
1081 outputFile = self.outputDirectory.child(releaseName + ".tar.bz2")
1082 buildPath = lambda *args: '/'.join((releaseName,) + args)
1083 tarball = self._createBasicSubprojectTarball(
1084 "core", version, outputFile)
1086 # Include the bin directory for the subproject.
1087 for path in self.rootDirectory.child("bin").children():
1088 if not path.isdir():
1089 tarball.add(path.path, buildPath("bin", path.basename()))
1091 # Include all files within twisted/ that aren't part of a subproject.
1092 for path in self.rootDirectory.child("twisted").children():
1093 if path.basename() == "plugins":
1094 for plugin in path.children():
1095 for subproject in self.subprojects:
1096 if plugin.basename() == "twisted_%s.py" % (subproject,):
1099 tarball.add(plugin.path,
1100 buildPath("twisted", "plugins",
1102 elif not path.basename() in self.subprojects + ["topfiles"]:
1103 tarball.add(path.path, buildPath("twisted", path.basename()))
1105 tarball.add(self.rootDirectory.child("twisted").child("topfiles").path,
1112 def buildSubProject(self, projectName, version):
1114 Build a subproject distribution in
1115 C{Twisted<Projectname>-<version>.tar.bz2}.
1117 @type projectName: C{str}
1118 @param projectName: The lowercase name of the subproject to build.
1119 @type version: C{str}
1120 @param version: The version of Twisted to build.
1122 @return: The tarball file.
1123 @rtype: L{FilePath}.
1125 releaseName = "Twisted%s-%s" % (projectName.capitalize(), version)
1126 outputFile = self.outputDirectory.child(releaseName + ".tar.bz2")
1127 buildPath = lambda *args: '/'.join((releaseName,) + args)
1128 subProjectDir = self.rootDirectory.child("twisted").child(projectName)
1130 tarball = self._createBasicSubprojectTarball(projectName, version,
1133 tarball.add(subProjectDir.child("topfiles").path, releaseName)
1135 # Include all files in the subproject package except for topfiles.
1136 for child in subProjectDir.children():
1137 name = child.basename()
1138 if name != "topfiles":
1141 buildPath("twisted", projectName, name))
1143 pluginsDir = self.rootDirectory.child("twisted").child("plugins")
1144 # Include the plugin for the subproject.
1145 pluginFileName = "twisted_%s.py" % (projectName,)
1146 pluginFile = pluginsDir.child(pluginFileName)
1147 if pluginFile.exists():
1148 tarball.add(pluginFile.path,
1149 buildPath("twisted", "plugins", pluginFileName))
1151 # Include the bin directory for the subproject.
1152 binPath = self.rootDirectory.child("bin").child(projectName)
1154 tarball.add(binPath.path, buildPath("bin"))
1160 def _createBasicSubprojectTarball(self, projectName, version, outputFile):
1162 Helper method to create and fill a tarball with things common between
1163 subprojects and core.
1165 @param projectName: The subproject's name.
1166 @type projectName: C{str}
1167 @param version: The version of the release.
1168 @type version: C{str}
1169 @param outputFile: The location of the tar file to create.
1170 @type outputFile: L{FilePath}
1172 releaseName = "Twisted%s-%s" % (projectName.capitalize(), version)
1173 buildPath = lambda *args: '/'.join((releaseName,) + args)
1175 tarball = tarfile.TarFile.open(outputFile.path, 'w:bz2')
1177 tarball.add(self.rootDirectory.child("LICENSE").path,
1178 buildPath("LICENSE"))
1180 docPath = self.rootDirectory.child("doc").child(projectName)
1183 for child in docPath.walk():
1184 self._buildDocInDir(child, version, docPath.child("howto"))
1185 tarball.add(docPath.path, buildPath("doc"))
1191 class UncleanWorkingDirectory(Exception):
1193 Raised when the working directory of an SVN checkout is unclean.
1198 class NotWorkingDirectory(Exception):
1200 Raised when a directory does not appear to be an SVN working directory.
1205 def buildAllTarballs(checkout, destination, templatePath=None):
1207 Build complete tarballs (including documentation) for Twisted and all
1210 This should be called after the version numbers have been updated and
1213 @type checkout: L{FilePath}
1214 @param checkout: The SVN working copy from which a pristine source tree
1216 @type destination: L{FilePath}
1217 @param destination: The directory in which tarballs will be placed.
1218 @type templatePath: L{FilePath}
1219 @param templatePath: Location of the template file that is used for the
1220 howto documentation.
1222 @raise UncleanWorkingDirectory: If there are modifications to the
1223 working directory of C{checkout}.
1224 @raise NotWorkingDirectory: If the C{checkout} path is not an SVN checkout.
1226 if not checkout.child(".svn").exists():
1227 raise NotWorkingDirectory(
1228 "%s does not appear to be an SVN working directory."
1230 if runCommand(["svn", "st", checkout.path]).strip():
1231 raise UncleanWorkingDirectory(
1232 "There are local modifications to the SVN checkout in %s."
1235 workPath = FilePath(mkdtemp())
1236 export = workPath.child("export")
1237 runCommand(["svn", "export", checkout.path, export.path])
1238 twistedPath = export.child("twisted")
1239 version = Project(twistedPath).getVersion()
1240 versionString = version.base()
1242 apiBaseURL = "http://twistedmatrix.com/documents/%s/api/%%s.html" % (
1244 if not destination.exists():
1245 destination.createDirectory()
1246 db = DistributionBuilder(export, destination, templatePath=templatePath,
1247 apiBaseURL=apiBaseURL)
1249 db.buildCore(versionString)
1250 for subproject in twisted_subprojects:
1251 if twistedPath.child(subproject).exists():
1252 db.buildSubProject(subproject, versionString)
1254 db.buildTwisted(versionString)
1258 class ChangeVersionsScript(object):
1260 A thing for changing version numbers. See L{main}.
1262 changeAllProjectVersions = staticmethod(changeAllProjectVersions)
1264 def main(self, args):
1266 Given a list of command-line arguments, change all the Twisted versions
1267 in the current directory.
1269 @type args: list of str
1270 @param args: List of command line arguments. This should only
1271 contain the version number.
1274 "Version should be in a form kind of like '1.2.3[pre4]'")
1276 sys.exit("Must specify exactly one argument to change-versions")
1279 major, minor, micro_and_pre = version.split(".")
1281 raise SystemExit(version_format)
1282 if "pre" in micro_and_pre:
1283 micro, pre = micro_and_pre.split("pre")
1285 micro = micro_and_pre
1294 raise SystemExit(version_format)
1295 version_template = Version("Whatever",
1296 major, minor, micro, prerelease=pre)
1297 self.changeAllProjectVersions(FilePath("."), version_template)
1301 class BuildTarballsScript(object):
1303 A thing for building release tarballs. See L{main}.
1305 buildAllTarballs = staticmethod(buildAllTarballs)
1307 def main(self, args):
1309 Build all release tarballs.
1311 @type args: list of C{str}
1312 @param args: The command line arguments to process. This must contain
1313 at least two strings: the checkout directory and the destination
1314 directory. An optional third string can be specified for the website
1315 template file, used for building the howto documentation. If this
1316 string isn't specified, the default template included in twisted
1319 if len(args) < 2 or len(args) > 3:
1320 sys.exit("Must specify at least two arguments: "
1321 "Twisted checkout and destination path. The optional third "
1322 "argument is the website template path.")
1324 self.buildAllTarballs(FilePath(args[0]), FilePath(args[1]))
1325 elif len(args) == 3:
1326 self.buildAllTarballs(FilePath(args[0]), FilePath(args[1]),
1331 class BuildAPIDocsScript(object):
1333 A thing for building API documentation. See L{main}.
1336 def buildAPIDocs(self, projectRoot, output):
1338 Build the API documentation of Twisted, with our project policy.
1340 @param projectRoot: A L{FilePath} representing the root of the Twisted
1342 @param output: A L{FilePath} pointing to the desired output directory.
1344 version = Project(projectRoot.child("twisted")).getVersion()
1345 versionString = version.base()
1346 sourceURL = ("http://twistedmatrix.com/trac/browser/tags/releases/"
1347 "twisted-%s" % (versionString,))
1348 apiBuilder = APIBuilder()
1351 "http://twistedmatrix.com/",
1353 projectRoot.child("twisted"),
1357 def main(self, args):
1359 Build API documentation.
1361 @type args: list of str
1362 @param args: The command line arguments to process. This must contain
1363 two strings: the path to the root of the Twisted checkout, and a
1364 path to an output directory.
1367 sys.exit("Must specify two arguments: "
1368 "Twisted checkout and destination path")
1369 self.buildAPIDocs(FilePath(args[0]), FilePath(args[1]))