wholetrans/docs-matrix-spec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Simplify changelog generation

Browse source 
We don'e need `{{server_server_changelog_r0.1.0}}` (for example), so don't go through the hassle of generating it. Instead, we'll generate the changelog for the requested versions of each API and put that in place. In the future, we may wish to consider bringing back more complicated variables when/if we start generating released versions of the spec on the fly rather than manually.
This commit is contained in:
Travis Ralston 2019-02-06 22:02:21 -07:00
parent 54ee861b5f
commit 76946a8a7c
2 changed files with 92 additions and 98 deletions
Download patch file Download diff file Expand all files Collapse all files

9
scripts/templating/matrix_templates/sections.py
View file

@ -34,15 +34,10 @@ class MatrixSections(Sections):
def render_changelogs(self): def render_changelogs(self):
rendered = {} rendered = {}
changelogs = self.units.get("changelogs") changelogs = self.units.get("changelogs")
for spec, versioned in changelogs.items(): for spec, changelog_text in changelogs.items():
spec_var = "%s_changelog" % spec spec_var = "%s_changelog" % spec
logger.info("Rendering changelog for spec: %s" % spec) logger.info("Rendering changelog for spec: %s" % spec)
for version, changelog in versioned.items(): rendered[spec_var] = changelog_text
version_var = "%s_%s" % (spec_var, version)
logger.info("Rendering changelog for %s" % version_var)
rendered[version_var] = changelog
if version == "preferred":
rendered[spec_var] = changelog
return rendered return rendered
def _render_events(self, filterFn, sortFn): def _render_events(self, filterFn, sortFn):

181
scripts/templating/matrix_templates/units.py
View file

@ -904,9 +904,20 @@ class MatrixUnits(Units):
return schema return schema
def load_changelogs(self, substitutions): def load_changelogs(self, substitutions):
"""Loads the changelog unit for later rendering in a section.
Args:
substitutions: dict of variable name to value. Provided by the gendoc script.
Returns:
A dict of API name ("client_server", for example) to changelog.
"""
changelogs = {} changelogs = {}
preferred_versions = { # The APIs and versions we'll prepare changelogs for. We use the substitutions
# to ensure that we pick up the right version for generated documentation. This
# defaults to "unstable" as a version for incremental generated documentation (CI).
prepare_versions = {
"server_server": substitutions.get("%SERVER_RELEASE_LABEL%", "unstable"), "server_server": substitutions.get("%SERVER_RELEASE_LABEL%", "unstable"),
"client_server": substitutions.get("%CLIENT_RELEASE_LABEL%", "unstable"), "client_server": substitutions.get("%CLIENT_RELEASE_LABEL%", "unstable"),
"identity_service": substitutions.get("%IDENTITY_RELEASE_LABEL%", "unstable"), "identity_service": substitutions.get("%IDENTITY_RELEASE_LABEL%", "unstable"),
@ -914,112 +925,100 @@ class MatrixUnits(Units):
"application_service": substitutions.get("%APPSERVICE_RELEASE_LABEL%", "unstable"), "application_service": substitutions.get("%APPSERVICE_RELEASE_LABEL%", "unstable"),
} }
# Changelog generation is a bit complicated. We rely on towncrier to # Changelogs are split into two places: towncrier for the unstable changelog and
# generate the unstable/current changelog, but otherwise use the RST # the RST file for historical versions. If the prepare_versions dict above has
# edition to record historical changelogs. This is done by prepending # a version other than "unstable" specified for an API, we'll use the historical
# the towncrier output to the RST in memory, then parsing the RST by # changelog and otherwise generate the towncrier log in-memory.
# hand. We parse the entire changelog to create a changelog for each
# version which may be of use in some APIs.
# Map specific headers to specific keys that'll be used eventually for api_name, target_version in prepare_versions.items():
# in variables. Things not listed here will get lowercased and formatted logger.info("Generating changelog for %s at %s" % (api_name, target_version,))
# such that characters not [a-z0-9] will be replaced with an underscore.
keyword_versions = {
"Unreleased Changes": "unstable"
}
# Only generate changelogs for things that have an RST document
for f in os.listdir(CHANGELOG_DIR):
if not f.endswith(".rst"):
continue
path = os.path.join(CHANGELOG_DIR, f)
name = f[:-4] # take off ".rst"
# If there's a directory with the same name, we'll try to generate
# a towncrier changelog and prepend it to the general changelog.
tc_path = os.path.join(CHANGELOG_DIR, name)
tc_lines = []
if os.path.isdir(tc_path):
logger.info("Generating towncrier changelog for: %s" % name)
p = subprocess.Popen(
['towncrier', '--version', 'Unreleased Changes', '--name', name, '--draft'],
cwd=tc_path,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
)
stdout, stderr = p.communicate()
if p.returncode != 0:
# Something broke - dump as much information as we can
logger.error("Towncrier exited with code %s" % p.returncode)
logger.error(stdout.decode('UTF-8'))
logger.error(stderr.decode('UTF-8'))
raw_log = ""
else:
raw_log = stdout.decode('UTF-8')
# This is a bit of a hack, but it does mean that the log at least gets *something*
# to tell us it broke
if not raw_log.startswith("Unreleased Changes"):
logger.error("Towncrier appears to have failed to generate a changelog")
logger.error(raw_log)
raw_log = ""
tc_lines = raw_log.splitlines()
title_part = None
changelog_lines = [] changelog_lines = []
with open(path, "r", encoding="utf-8") as f: if target_version == 'unstable':
lines = f.readlines() # generate towncrier log
tc_path = os.path.join(CHANGELOG_DIR, api_name)
if os.path.isdir(tc_path):
logger.info("Generating towncrier changelog for: %s" % api_name)
p = subprocess.Popen(
['towncrier', '--version', 'unstable', '--name', api_name, '--draft'],
cwd=tc_path,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
)
stdout, stderr = p.communicate()
if p.returncode != 0:
# Something broke - dump as much information as we can
logger.error("Towncrier exited with code %s" % p.returncode)
logger.error(stdout.decode('UTF-8'))
logger.error(stderr.decode('UTF-8'))
raw_log = ""
else:
raw_log = stdout.decode('UTF-8')
# This is a bit of a hack, but it does mean that the log at least gets *something*
# to tell us it broke
if not raw_log.startswith("unstable"):
logger.error("Towncrier appears to have failed to generate a changelog")
logger.error(raw_log)
raw_log = ""
changelog_lines = raw_log.splitlines()
else:
# read in the existing RST changelog
logger.info("Reading changelog RST for %s" % api_name)
rst_path = os.path.join(CHANGELOG_DIR, "%s.rst" % api_name)
with open(rst_path, 'r', encoding="utf-8") as f:
changelog_lines = f.readlines()
# Parse the changelog lines to find the header we're looking for and therefore
# the changelog body.
prev_line = None prev_line = None
for line in (tc_lines + lines): title_part = None
changelog_body_lines = []
have_changelog = False
for line in changelog_lines:
if prev_line is None: if prev_line is None:
prev_line = line prev_line = line
continue continue
if not title_part: if not title_part:
# find the title underline (at least 3 =) # Titles we care about are underlined with at least 3 equal signs
if re.match("^[=]{3,}$", line.strip()): if re.match("^[=]{3,}$", line.strip()):
title_part = prev_line logger.info("Found header %s" % prev_line)
title_part = prev_line.strip()
continue continue
prev_line = line prev_line = line
else: # have title, get body (stop on next title or EOF) else:
# we have a title, start parsing the body
if re.match("^[=]{3,}$", line.strip()): if re.match("^[=]{3,}$", line.strip()):
# we hit another title, so pop the last line of # we hit another title. prev_line will be the new section's header.
# the changelog and record the changelog # do a check to see if the section we just read is the one we want - if
new_title = changelog_lines.pop() # it is, use that changelog and move on. If it isn't, keep reading.
if name not in changelogs: if title_part == target_version:
changelogs[name] = {} changelogs[api_name] = "".join(changelog_body_lines)
if title_part in keyword_versions: have_changelog = True
title_part = keyword_versions[title_part] break
title_part = title_part.strip().replace("^[a-zA-Z0-9]", "_").lower() # not the section we want - start the next section
changelog = "".join(changelog_lines) title_part = changelog_body_lines.pop().strip()
changelogs[name][title_part] = changelog changelog_body_lines = []
# reset for the next version
changelog_lines = []
title_part = new_title.strip()
continue continue
# Don't generate subheadings (we'll keep the title though)
if re.match("^[-]{3,}$", line.strip()): if re.match("^[-]{3,}$", line.strip()):
continue # the last line is a subheading - drop this line because it's the underline
if line.strip().startswith(".. version: "): # and that causes problems with rendering. We'll keep the header text though.
# The changelog is directing us to use a different title
# for the changelog.
title_part = line.strip()[len(".. version: "):]
continue continue
if line.strip().startswith(".. "): if line.strip().startswith(".. "):
continue # skip comments # skip comments
changelog_lines.append(" " + line + '\n') continue
if len(changelog_lines) > 0 and title_part is not None: # if we made it this far, append the line to the changelog body. We indent it so
if name not in changelogs: # that it renders correctly in the section. We also add newlines so that there's
changelogs[name] = {} # intentionally blank lines that make rst2html happy.
if title_part in keyword_versions: changelog_body_lines.append(" " + line + '\n')
title_part = keyword_versions[title_part] # do some quick checks to see if the last read section is our changelog
changelog = "".join(changelog_lines) if not have_changelog:
changelogs[name][title_part.replace("^[a-zA-Z0-9]", "_").lower()] = changelog logger.info("No changelog - testing %s == %s" % (target_version, title_part,))
preferred_changelog = changelogs[name]["unstable"] if title_part == target_version and len(changelog_body_lines) > 0:
if name in preferred_versions: changelogs[api_name] = "".join(changelog_body_lines)
preferred_changelog = changelogs[name][preferred_versions[name]] else:
changelogs[name]["preferred"] = preferred_changelog raise ValueError("No changelog for %s at %s" % (api_name, target_version,))
# return our `dict[api_name] => changelog` as the last step.
return changelogs return changelogs
def load_unstable_warnings(self, substitutions): def load_unstable_warnings(self, substitutions):