Merge pull request #1846 from matrix-org/travis/fix-changelog

Fix changelog generation for non-default versions

meta/releasing_a_spec.md

@@ -18,16 +18,18 @@ The remainder of the process is as follows:
 1. Activate your Python 3 virtual environment.
 1. Having checked out the new release branch, navigate your way over to `./changelogs`.
 1. Follow the release instructions provided in the README.md located there.
-1. Update the changelog section of the specification you're releasing to make a reference
-   to the new version.
 1. Update any version/link references across all specifications.
-1. Ensure the `targets.yml` file lists the version correctly.
+1. Generate the specification using `./scripts/gendoc.py`, specifying all the
-1. Commit the changes and PR them to master.
+   API versions at the time of generation. For example: `./scripts/gendoc.py -c r0.4.0 -s r0.1.0 -i r0.1.0 #etc`
-1. Tag the release with the format `client_server/r0.4.0`.
+1. PR the changes to the matrix-org/matrix.org repository (for historic tracking).
-1. Add the changes to the matrix-org/matrix.org repository (for historic tracking).
    * This is done by making a PR to the `unstyled_docs/spec` folder for the version and
      specification you're releasing.
    * Don't forget to symlink the new release as `latest`.
+   * For the client-server API, don't forget to generate the swagger JSON by using
+     `./scripts/dump-swagger.py -c r0.4.0`. This will also need symlinking to `latest`.
+1. Commit the changes and PR them to master. **Wait for review from the spec core team.**
+   * Link to your matrix-org/matrix.org so both can be reviewed at the same time.
+1. Tag the release with the format `client_server/r0.4.0`.
 1. Perform a release on GitHub to tag the release.
 1. Yell from the mountaintop to the world about the new release.
 

scripts/templating/matrix_templates/sections.py

@@ -34,15 +34,10 @@ class MatrixSections(Sections):
     def render_changelogs(self):
         rendered = {}
         changelogs = self.units.get("changelogs")
-        for spec, versioned in changelogs.items():
+        for spec, changelog_text in changelogs.items():
             spec_var = "%s_changelog" % spec
             logger.info("Rendering changelog for spec: %s" % spec)
-            for version, changelog in versioned.items():
+            rendered[spec_var] = changelog_text
-                version_var = "%s_%s" % (spec_var, version)
-                logger.info("Rendering changelog for %s" % version_var)
-                rendered[version_var] = changelog
-                if version == "unstable":
-                    rendered[spec_var] = changelog
         return rendered
 
     def _render_events(self, filterFn, sortFn):

scripts/templating/matrix_templates/units.py

@@ -903,38 +903,83 @@ class MatrixUnits(Units):
 
         return schema
 
-    def load_changelogs(self):
+    def load_changelogs(self, substitutions):
+        """Loads the changelog unit for later rendering in a section.
+
+        Args:
+            substitutions: dict of variable name to value. Provided by the gendoc script.
+
+        Returns:
+            A dict of API name ("client_server", for example) to changelog.
+        """
         changelogs = {}
 
-        # Changelog generation is a bit complicated. We rely on towncrier to
+        # The APIs and versions we'll prepare changelogs for. We use the substitutions
-        # generate the unstable/current changelog, but otherwise use the RST
+        # to ensure that we pick up the right version for generated documentation. This
-        # edition to record historical changelogs. This is done by prepending
+        # defaults to "unstable" as a version for incremental generated documentation (CI).
-        # the towncrier output to the RST in memory, then parsing the RST by
+        prepare_versions = {
-        # hand. We parse the entire changelog to create a changelog for each
+            "server_server": substitutions.get("%SERVER_RELEASE_LABEL%", "unstable"),
-        # version which may be of use in some APIs.
+            "client_server": substitutions.get("%CLIENT_RELEASE_LABEL%", "unstable"),
-
+            "identity_service": substitutions.get("%IDENTITY_RELEASE_LABEL%", "unstable"),
-        # Map specific headers to specific keys that'll be used eventually
+            "push_gateway": substitutions.get("%PUSH_GATEWAY_RELEASE_LABEL%", "unstable"),
-        # in variables. Things not listed here will get lowercased and formatted
+            "application_service": substitutions.get("%APPSERVICE_RELEASE_LABEL%", "unstable"),
-        # such that characters not [a-z0-9] will be replaced with an underscore.
-        keyword_versions = {
-            "Unreleased Changes": "unstable"
         }
 
-        # Only generate changelogs for things that have an RST document
+        # Changelogs are split into two places: towncrier for the unstable changelog and
-        for f in os.listdir(CHANGELOG_DIR):
+        # the RST file for historical versions. If the prepare_versions dict above has
-            if not f.endswith(".rst"):
+        # a version other than "unstable" specified for an API, we'll use the historical
-                continue
+        # changelog and otherwise generate the towncrier log in-memory.
-            path = os.path.join(CHANGELOG_DIR, f)
-            name = f[:-4]  # take off ".rst"
 
-            # If there's a directory with the same name, we'll try to generate
+        for api_name, target_version in prepare_versions.items():
-            # a towncrier changelog and prepend it to the general changelog.
+            logger.info("Generating changelog for %s at %s" % (api_name, target_version,))
-            tc_path = os.path.join(CHANGELOG_DIR, name)
+            changelog_lines = []
-            tc_lines = []
+            if target_version == 'unstable':
+                # generate towncrier log
+                changelog_lines = self._read_towncrier_changelog(api_name)
+            else:
+                # read in the existing RST changelog
+                changelog_lines = self._read_rst_changelog(api_name)
+
+            # Parse the changelog lines to find the header we're looking for and therefore
+            # the changelog body.
+            prev_line = None
+            title_part = None
+            changelog_body_lines = []
+            for line in changelog_lines:
+                if prev_line is None:
+                    prev_line = line
+                    continue
+                if re.match("^[=]{3,}$", line.strip()):
+                    # the last line was a header - use that as our new title_part
+                    title_part = prev_line.strip()
+                    continue
+                if re.match("^[-]{3,}$", line.strip()):
+                    # the last line is a subheading - drop this line because it's the underline
+                    # and that causes problems with rendering. We'll keep the header text though.
+                    continue
+                if line.strip().startswith(".. "):
+                    # skip comments
+                    continue
+                if title_part == target_version:
+                    # if we made it this far, append the line to the changelog body. We indent it so
+                    # that it renders correctly in the section. We also add newlines so that there's
+                    # intentionally blank lines that make rst2html happy.
+                    changelog_body_lines.append("    " + line + '\n')
+
+            if len(changelog_body_lines) > 0:
+                changelogs[api_name] = "".join(changelog_body_lines)
+            else:
+                raise ValueError("No changelog for %s at %s" % (api_name, target_version,))
+
+        # return our `dict[api_name] => changelog` as the last step.
+        return changelogs
+
+    def _read_towncrier_changelog(self, api_name):
+        tc_path = os.path.join(CHANGELOG_DIR, api_name)
         if os.path.isdir(tc_path):
-                logger.info("Generating towncrier changelog for: %s" % name)
+            logger.info("Generating towncrier changelog for: %s" % api_name)
             p = subprocess.Popen(
-                    ['towncrier', '--version', 'Unreleased Changes', '--name', name, '--draft'],
+                ['towncrier', '--version', 'unstable', '--name', api_name, '--draft'],
                 cwd=tc_path,
                 stderr=subprocess.PIPE,
                 stdout=subprocess.PIPE,
@@ -951,64 +996,18 @@ class MatrixUnits(Units):
 
                 # This is a bit of a hack, but it does mean that the log at least gets *something*
                 # to tell us it broke
-                    if not raw_log.startswith("Unreleased Changes"):
+                if not raw_log.startswith("unstable"):
                     logger.error("Towncrier appears to have failed to generate a changelog")
                     logger.error(raw_log)
                     raw_log = ""
-                tc_lines = raw_log.splitlines()
+            return raw_log.splitlines()
+        return []
 
-            title_part = None
+    def _read_rst_changelog(self, api_name):
-            changelog_lines = []
+        logger.info("Reading changelog RST for %s" % api_name)
-            with open(path, "r", encoding="utf-8") as f:
+        rst_path = os.path.join(CHANGELOG_DIR, "%s.rst" % api_name)
-                lines = f.readlines()
+        with open(rst_path, 'r', encoding="utf-8") as f:
-            prev_line = None
+            return f.readlines()
-            for line in (tc_lines + lines):
-                if prev_line is None:
-                    prev_line = line
-                    continue
-                if not title_part:
-                    # find the title underline (at least 3 =)
-                    if re.match("^[=]{3,}$", line.strip()):
-                        title_part = prev_line
-                        continue
-                    prev_line = line
-                else:  # have title, get body (stop on next title or EOF)
-                    if re.match("^[=]{3,}$", line.strip()):
-                        # we hit another title, so pop the last line of
-                        # the changelog and record the changelog
-                        new_title = changelog_lines.pop()
-                        if name not in changelogs:
-                            changelogs[name] = {}
-                        if title_part in keyword_versions:
-                            title_part = keyword_versions[title_part]
-                        title_part = title_part.strip().replace("^[a-zA-Z0-9]", "_").lower()
-                        changelog = "".join(changelog_lines)
-                        changelogs[name][title_part] = changelog
-
-                        # reset for the next version
-                        changelog_lines = []
-                        title_part = new_title.strip()
-                        continue
-                    # Don't generate subheadings (we'll keep the title though)
-                    if re.match("^[-]{3,}$", line.strip()):
-                        continue
-                    if line.strip().startswith(".. version: "):
-                        # The changelog is directing us to use a different title
-                        # for the changelog.
-                        title_part = line.strip()[len(".. version: "):]
-                        continue
-                    if line.strip().startswith(".. "):
-                        continue  # skip comments
-                    changelog_lines.append("    " + line + '\n')
-            if len(changelog_lines) > 0 and title_part is not None:
-                if name not in changelogs:
-                    changelogs[name] = {}
-                if title_part in keyword_versions:
-                    title_part = keyword_versions[title_part]
-                changelog = "".join(changelog_lines)
-                changelogs[name][title_part.replace("^[a-zA-Z0-9]", "_").lower()] = changelog
-
-        return changelogs
 
     def load_unstable_warnings(self, substitutions):
         warning = """