Support rendering of proposal tables

.gitignore

@@ -1,6 +1,7 @@
 /api/node_modules
 /assets
 /assets.tar.gz
+/data/msc
 /env*
 /node_modules
 /resources

content/proposals.md

@@ -338,7 +338,7 @@ request trackers.
 | Spec PR In Review               | [spec-pr-in-review](https://github.com/matrix-org/matrix-doc/issues?q=label%3Aproposal+label%3Aspec-pr-in-review+)                           | The spec PR has been written, and is currently under review                                                             |
 | Spec PR Merged                  | [merged](https://github.com/matrix-org/matrix-doc/issues?q=label%3Aproposal+label%3Amerged)                                                  | A proposal with a sufficient working implementation and whose Spec PR has been merged!                                  |
 | Postponed                       | [proposal-postponed](https://github.com/matrix-org/matrix-doc/issues?q=label%3Aproposal+label%3Aproposal-postponed+)                         | A proposal that is temporarily blocked or a feature that may not be useful currently but perhaps sometime in the future |
-| Abandoned                       | [proposal-closed](https://github.com/matrix-org/matrix-doc/issues?q=label%3Aproposal+label%3Aabandoned)                                      | A proposal where the author/shepherd is not responsive                                                                  |
+| Abandoned                       | [abandoned](https://github.com/matrix-org/matrix-doc/issues?q=label%3Aproposal+label%3Aabandoned)                                      | A proposal where the author/shepherd is not responsive                                                                  |
 | Obsolete                        | [obsolete](https://github.com/matrix-org/matrix-doc/issues?q=label%3Aproposal+label%3Aobsolete+)                                             | A proposal which has been made obsolete by another proposal or decision elsewhere.                                      |
 
 ## Categories
@@ -479,3 +479,41 @@ In summary:
     way that proposes new stable endpoints. Typically this is solved by
     a small table at the bottom mapping the various values from stable
     to unstable.
+
+## Proposal Tracking
+
+This is a living document generated from the list of proposals on the
+issue and pull request trackers of the
+[matrix-doc](https://github.com/matrix-org/matrix-doc) repo.
+
+We use labels and some metadata in MSC PR descriptions to generate this
+page. Labels are assigned by the Spec Core Team whilst triaging the
+proposals based on those which exist in the
+[matrix-doc](https://github.com/matrix-org/matrix-doc) repo already.
+
+It is worth mentioning that a previous version of the MSC process used a
+mixture of GitHub issues and PRs, leading to some MSC numbers deriving
+from GitHub issue IDs instead. A useful feature of GitHub is that it
+does automatically resolve to an issue, if an issue ID is placed in a
+pull URL. This means that
+<https://github.com/matrix-org/matrix-doc/pull/$MSCID> will correctly
+resolve to the desired MSC, whether it started as an issue or a PR.
+
+Other metadata:
+
+-   The MSC number is taken from the GitHub Pull Request ID. This is
+    carried for the lifetime of the proposal. These IDs do not necessary
+    represent a chronological order.
+-   The GitHub PR title will act as the MSC's title.
+-   Please link to the spec PR (if any) by adding a "PRs: \#1234" line
+    in the issue description.
+-   The creation date is taken from the GitHub PR, but can be overridden
+    by adding a "Date: yyyy-mm-dd" line in the PR description.
+-   Updated Date is taken from GitHub.
+-   Author is the creator of the MSC PR, but can be overridden by adding
+    a "Author: @username" line in the body of the issue description.
+    Please make sure @username is a GitHub user (include the @!)
+-   A shepherd can be assigned by adding a "Shepherd: @username" line in
+    the issue description. Again, make sure this is a real GitHub user.
+
+{{% proposal-tables %}}

layouts/shortcodes/proposal-tables.html

@@ -0,0 +1,81 @@
+{{/*
+
+  This template is used to render tables of MSC proposals.
+
+  It expects there to be a "proposals.json" under /data/msc.
+  It expects "proposals.json" to contain an array of objects,
+  one for each MSC state. Each object contains:
+  * `title`: human-readable title for the state, like "Proposal In Review"
+  * `label`: the GitHub label used for the state, like "proposal-in-review"
+  * `proposals`: an array of objects, each of which represents an MSC and contains:
+      * `number`: GitHub issue number
+			* `url`: GitHub URL for this issue
+			* `title`: Issue title
+		  * `created_at`: issue creation date
+			* `updated_at`: issue last-updated date
+			* `authors`: array of GitHub user names representing authors of this MSC
+			* `shepherd`: GitHub user name representing the shepherd of this MSC, or null
+			* `documentation`: Links to further documentation referenced in the GitHub issue
+
+  This data is scraped from GitHub using the /scripts/proposals.js Node script.
+  The script is run in CI: so typically if you run a local server the data will
+  be missing and no tables will be generated. If you do want to see the tables locally,
+  you can run the script locally:
+
+    npm install
+    npm run get-proposals
+
+  If this template does find the data, it renders one table for each MSC state,
+  containing a row for each MSC in that state.
+
+*/}}
+
+{{ $states := .Site.Data.msc.proposals }}
+
+{{ range $states }}
+<h3 id="{{.label}}" class="proposal-table-title">{{ .title }}</h3>
+    {{ if .proposals }}
+<table class="msc-table table">
+  <thead>
+    <tr>
+      <th>MSC</th>
+      <th>Title</th>
+      <th>Created at</th>
+      <th>Updated at</th>
+      <th>Docs</th>
+      <th>Author</th>
+      <th>Shepherd</th>
+    </tr>
+  </thead>
+
+  <tbody>
+        {{ range .proposals }}
+
+            {{ $index := 0 }}
+            {{ $docs_links_list := slice }}
+            {{ range .documentation }}
+                {{ $index = add $index 1 }}
+                {{ $docs_link := printf "<a href=\"%s\">%d</a>" . $index }}
+                {{ $docs_links_list = $docs_links_list | append $docs_link }}
+            {{ end }}
+            {{ $docs_links := delimit $docs_links_list ", " }}
+
+            {{ $authors_list := apply .authors "printf" "<a href=\"https://github.com/%s\">@%s</a>" "." "." }}
+            {{ $authors := delimit $authors_list ", " }}
+
+  <tr>
+      <td><a href="{{ .url }}">{{ .number }}</a></td>
+      <td>{{ .title }}</td>
+      <td>{{ .created_at }}</td>
+      <td>{{ .updated_at }}</td>
+      <td>{{ with $docs_links }}{{ $docs_links }}{{ end }}</td>
+      <td>{{ $authors }}</td>
+      <td>{{ with .shepherd }}<a href="https://github.com/{{ . }}">@{{ . }}</a>{{ end }}</td>
+  </tr>
+        {{ end }}
+  </tbody>
+</table>
+    {{ else }}
+<p>No MSCs are currently in this state.</p>
+    {{ end }}
+{{ end }}

package-lock.json

@@ -569,6 +569,12 @@
         "picomatch": "^2.0.5"
       }
     },
+    "node-fetch": {
+      "version": "2.6.1",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz",
+      "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==",
+      "dev": true
+    },
     "node-releases": {
       "version": "1.1.60",
       "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-1.1.60.tgz",

package.json

@@ -4,6 +4,7 @@
   "description": "Hugo theme for the Matrix specification.",
   "main": "none.js",
   "scripts": {
+    "get-proposals": "node ./scripts/proposals.js",
     "test": "echo \"Error: no test specified\" && exit 1"
   },
   "repository": {
@@ -20,6 +21,7 @@
   "dependencies": {},
   "devDependencies": {
     "autoprefixer": "^9.8.6",
+    "node-fetch": "^2.6.1",
     "postcss-cli": "^7.1.2"
   }
 }

scripts/proposals.js

@@ -0,0 +1,219 @@
+"use strict";
+
+/**
+ * This Node script fetches MSC proposals and stores them in /data/msc,
+ * so they can be used by a Hugo template to render summary tables of them
+ * in the specification.
+ *
+ * In detail, it:
+ * - fetches all GitHub issues from matrix-doc that have the `proposal` label
+ * - groups them by their state in the MSC process
+ * - does some light massaging of them so it's easier for the Hugo template to work with them
+ * - store them at /data/msc
+ */
+
+// built-in modules
+const path = require('path');
+const fs = require('fs');
+
+// third-party modules
+const fetch = require('node-fetch');
+
+// We will write proposals into the /data/msc directory
+const outputDir = path.join(__dirname, "../data/msc");
+
+/**
+ * This defines the different proposal lifecycle states.
+ * Each state has:
+ * - `label`: a GitHub label used to identify issues in this state
+ * - `title`: used for things like headings in renderings of the proposals
+ */
+const states = [
+  {
+    label: "proposal-in-review",
+    title: "Proposal In Review"
+  },
+  {
+    label: "proposed-final-comment-period",
+    title: "Proposed Final Comment Period"
+  },
+  {
+    label: "final-comment-period",
+    title: "Final Comment Period"
+  },
+  {
+    label: "finished-final-comment-period",
+    title: "Finished Final Comment Period"
+  },
+  {
+    label: "spec-pr-missing",
+    title: "Spec PR Missing"
+  },
+  {
+    label: "spec-pr-in-review",
+    title: "Spec PR In Review"
+  },
+  {
+    label: "merged",
+    title: "Merged"
+  },
+  {
+    label: "proposal-postoned",
+    title: "Postponed"
+  },
+  {
+    label: "abandoned",
+    title: "Abandoned"
+  },
+  {
+    label: "obsolete",
+    title: "Obsolete"
+  }
+];
+
+let issues = [];
+
+/**
+ * Fetch all the MSC proposals from GitHub.
+ *
+ * GitHub only lets us fetch 100 items at a time, and it gives us a `link`
+ * response header containing the URL for the next batch.
+ * So we will keep fetching until the response doesn't contain the "next" link.
+ */
+async function getIssues() {
+
+  /**
+   * A pretty ugly function to get us the "next" link in the header if there
+   * was one, or `null` otherwise.
+   */
+  function getNextLink(header) {
+    const links = header.split(",");
+    for (const link of links) {
+      const linkPieces = link.split(";");
+      if (linkPieces[1] == ` rel=\"next\"`) {
+        const next = linkPieces[0].trim();
+        return next.substring(1, next.length-1);
+      }
+    }
+    return null;
+  }
+
+  let pageLink = "https://api.github.com/repos/matrix-org/matrix-doc/issues?state=all&labels=proposal&per_page=100";
+  while (pageLink) {
+    const response = await fetch(pageLink);
+    const issuesForPage = await response.json();
+    issues = issues.concat(issuesForPage);
+    const linkHeader = response.headers.get("link");
+    pageLink = getNextLink(linkHeader);
+  }
+}
+
+getIssues().then(processIssues);
+
+/**
+ * Rather than just store the complete issue, we'll extract
+ * only the pieces we need.
+ * We'll also do some transformation of the issues, jsut because
+ * it's easier to do in JS than in the template.
+ */
+function getProposalFromIssue(issue) {
+
+  /**
+   * A helper function to fetch the contents of special
+   * directives in the issue body.
+   * Looks for a directive in the format:
+   * `^${directiveName}: (.+?)$`, returning the matched
+   *  part or null if the directive wasn't found.
+   */
+  function getDirective(directiveName, issue) {
+    const re = new RegExp(`^${directiveName}: (.+?)$`, "m");
+    const found = issue.body.match(re);
+    return found? found[1]: null;
+  }
+
+  function getDocumentation(issue) {
+    const found = getDirective("Documentation", issue);
+    if (found) {
+      return found.split(",").map(a => a.trim());
+    } else {
+      return [];
+    }
+  }
+
+  function getAuthors(issue) {
+    const found = getDirective("Author", issue);
+    if (found) {
+      return found.split(",").map(a => a.trim().substr(1));
+    } else {
+      return [`${issue.user.login}`];
+    }
+  }
+
+  function getShepherd(issue) {
+    const found = getDirective("Shepherd", issue);
+    if (found) {
+      return found.substr(1);
+    } else {
+      return null;
+    }
+  }
+
+  return {
+    number: issue.number,
+    url: issue.html_url,
+    title: issue.title,
+    created_at: issue.created_at.substr(0, 10),
+    updated_at: issue.updated_at.substr(0, 10),
+    authors: getAuthors(issue),
+    shepherd: getShepherd(issue),
+    documentation: getDocumentation(issue)
+  }
+}
+
+/**
+ * Returns the intersection of two arrays.
+ */
+function intersection(array1, array2) {
+  return array1.filter(i => array2.includes(i));
+}
+
+/**
+ * Given all the GitHub issues with a "proposal" label:
+ * - group issues by the state they are in, and for each group:
+ *     - extract the bits we need from each issue
+ *     - write the result under /data/msc
+ */
+function processIssues()  {
+  if (!fs.existsSync(outputDir)){
+    fs.mkdirSync(outputDir);
+  }
+  const output = [];
+  // make a group of "work in progress" proposals,
+  // which are identified by not having any of the state labels
+  const stateLabels = states.map(s => s.label);
+  const worksInProgress = issues.filter(issue => {
+    const labelsForIssue = issue.labels.map(l => l.name);
+    return intersection(labelsForIssue, stateLabels).length === 0;
+  });
+  output.push({
+    title: "Work In Progress",
+    label: "work-in-progress",
+    proposals: worksInProgress.map(issue => getProposalFromIssue(issue))
+  });
+  // for each defined state
+  for (const state of states) {
+    // get the set of issues for that state
+    const issuesForState = issues.filter(msc => {
+      return msc.labels.some(l => l.name === state.label);
+    });
+    // store it in /data
+    output.push({
+      title: state.title,
+      label: state.label,
+      proposals: issuesForState.map(issue => getProposalFromIssue(issue))
+    });
+  }
+  const outputData = JSON.stringify(output, null, '\t');
+  const outputFile = path.join(outputDir, `proposals.json`);
+  fs.writeFileSync(outputFile, outputData);
+}