Merge pull request #184 from matrix-org/daniel/splitspec

Split spec into page-per-section

scripts/gendoc.py

@@ -276,6 +276,12 @@ def run_through_template(input, set_verbose):
         raise
 
 
+def get_build_targets(targets_listing):
+    with open(targets_listing, "r") as targ_file:
+        all_targets = yaml.load(targ_file.read())
+    return all_targets["targets"].keys()
+
+
 """
 Extract and resolve groups for the given target in the given targets listing.
 Args:
@@ -386,21 +392,34 @@ def cleanup_env():
     shutil.rmtree("./tmp")
 
 
-def main(target_name, keep_intermediates):
+def main(requested_target_name, keep_intermediates):
     prepare_env()
-    log("Building spec [target=%s]" % target_name)
+    log("Building spec [target=%s]" % requested_target_name)
-    target = get_build_target("../specification/targets.yaml", target_name)
+
-    build_spec(target=target, out_filename="tmp/templated_spec.rst")
+    targets = [requested_target_name]
-    run_through_template("tmp/templated_spec.rst", VERBOSE)
+    if requested_target_name == "all":
-    fix_relative_titles(
+        targets = get_build_targets("../specification/targets.yaml")
-        target=target, filename="tmp/templated_spec.rst",
+
-        out_filename="tmp/full_spec.rst"
+    for target_name in targets:
-    )
+        templated_file = "tmp/templated_%s.rst" % (target_name,)
-    shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst")
+        rst_file = "tmp/spec_%s.rst" % (target_name,)
-    run_through_template("tmp/howto.rst", False)  # too spammy to mark -v on this
+        html_file = "gen/%s.html" % (target_name,)
-    rst2html("tmp/full_spec.rst", "gen/specification.html")
+
-    addAnchors("gen/specification.html")
+        target = get_build_target("../specification/targets.yaml", target_name)
-    rst2html("tmp/howto.rst", "gen/howtos.html")
+        build_spec(target=target, out_filename=templated_file)
+        run_through_template(templated_file, VERBOSE)
+        fix_relative_titles(
+            target=target, filename=templated_file,
+            out_filename=rst_file,
+        )
+        rst2html(rst_file, html_file)
+        addAnchors(html_file)
+
+    if requested_target_name == "all":
+        shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst")
+        run_through_template("tmp/howto.rst", False)  # too spammy to mark -v on this
+        rst2html("tmp/howto.rst", "gen/howtos.html")
+
     if not keep_intermediates:
         cleanup_env()
 
@@ -414,8 +433,9 @@ if __name__ == '__main__':
         help="Do not delete intermediate files. They will be found in tmp/"
     )
     parser.add_argument(
-        "--target", "-t", default="main",
+        "--target", "-t", default="all",
-        help="Specify the build target to build from specification/targets.yaml"
+        help="Specify the build target to build from specification/targets.yaml. " +
+             "The value 'all' will build all of the targets therein."
     )
     parser.add_argument(
         "--verbose", "-v", action="store_true",

scripts/speculator/main.go

@@ -57,8 +57,8 @@ var (
 	port            = flag.Int("port", 9000, "Port on which to listen for HTTP")
 	includesDir     = flag.String("includes_dir", "", "Directory containing include files for styling like matrix.org")
 	allowedMembers  map[string]bool
-	specCache       *lru.Cache // string -> []byte
+	specCache       *lru.Cache // string -> map[string][]byte filename -> contents
-	styledSpecCache *lru.Cache // string -> []byte
+	styledSpecCache *lru.Cache // string -> map[string][]byte filename -> contents
 )
 
 func (u *User) IsTrusted() bool {
@@ -105,10 +105,7 @@ func lookupPullRequest(url url.URL, pathPrefix string) (*PullRequest, error) {
 	if !strings.HasPrefix(url.Path, pathPrefix+"/") {
 		return nil, fmt.Errorf("invalid path passed: %s expect %s/123", url.Path, pathPrefix)
 	}
-	prNumber := url.Path[len(pathPrefix)+1:]
+	prNumber := strings.Split(url.Path[len(pathPrefix)+1:], "/")[0]
-	if strings.Contains(prNumber, "/") {
-		return nil, fmt.Errorf("invalid path passed: %s expect %s/123", url.Path, pathPrefix)
-	}
 
 	resp, err := http.Get(fmt.Sprintf("%s/%s", pullsPrefix, prNumber))
 	defer resp.Body.Close()
@@ -196,6 +193,17 @@ func (s *server) getSHAOf(ref string) (string, error) {
 	return strings.TrimSpace(b.String()), nil
 }
 
+func extractPath(path, base string) string {
+	// Assume exactly one flat directory
+	max := strings.Count(base, "/") + 2
+	parts := strings.SplitN(path, "/", max)
+
+	if len(parts) < max || parts[max-1] == "" {
+		return "index.html"
+	}
+	return parts[max-1]
+}
+
 func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) {
 	var sha string
 
@@ -206,7 +214,7 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) {
 		return
 	}
 
-	if strings.ToLower(req.URL.Path) == "/spec/head" {
+	if strings.HasPrefix(strings.ToLower(req.URL.Path), "/spec/head") {
 		// err may be non-nil here but if headSha is non-empty we will serve a possibly-stale result in favour of erroring.
 		// This is to deal with cases like where github is down but we still want to serve the spec.
 		if headSha, err := s.lookupHeadSHA(); headSha == "" {
@@ -236,36 +244,59 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) {
 		cache = styledSpecCache
 	}
 
+	var pathToContent map[string][]byte
+
 	if cached, ok := cache.Get(sha); ok {
-		w.Write(cached.([]byte))
+		pathToContent = cached.(map[string][]byte)
-		return
+	} else {
+		dst, err := s.generateAt(sha)
+		defer os.RemoveAll(dst)
+		if err != nil {
+			writeError(w, 500, err)
+			return
+		}
+
+		if styleLikeMatrixDotOrg {
+			cmd := exec.Command("./add-matrix-org-stylings.sh", *includesDir)
+			cmd.Dir = path.Join(dst, "scripts")
+			var b bytes.Buffer
+			cmd.Stderr = &b
+			if err := cmd.Run(); err != nil {
+				writeError(w, 500, fmt.Errorf("error styling spec: %v\nOutput:\n%v", err, b.String()))
+				return
+			}
+		}
+
+		fis, err := ioutil.ReadDir(path.Join(dst, "scripts", "gen"))
+		if err != nil {
+			writeError(w, 500, fmt.Errorf("Error reading directory: %v", err))
+		}
+		pathToContent = make(map[string][]byte)
+		for _, fi := range fis {
+			b, err := ioutil.ReadFile(path.Join(dst, "scripts", "gen", fi.Name()))
+			if err != nil {
+				writeError(w, 500, fmt.Errorf("Error reading spec: %v", err))
+				return
+			}
+			pathToContent[fi.Name()] = b
+		}
+		cache.Add(sha, pathToContent)
 	}
 
-	dst, err := s.generateAt(sha)
+	requestedPath := extractPath(req.URL.Path, "/spec/pr")
-	defer os.RemoveAll(dst)
+	if b, ok := pathToContent[requestedPath]; ok {
-	if err != nil {
+		w.Write(b)
-		writeError(w, 500, err)
 		return
 	}
-
+	if requestedPath == "index.html" {
-	if styleLikeMatrixDotOrg {
+		// Fall back to single-page spec for old PRs
-		cmd := exec.Command("./add-matrix-org-stylings.sh", *includesDir)
+		if b, ok := pathToContent["specification.html"]; ok {
-		cmd.Dir = path.Join(dst, "scripts")
+			w.Write(b)
-		var b bytes.Buffer
-		cmd.Stderr = &b
-		if err := cmd.Run(); err != nil {
-			writeError(w, 500, fmt.Errorf("error styling spec: %v\nOutput:\n%v", err, b.String()))
 			return
 		}
 	}
-
+	w.WriteHeader(404)
-	b, err := ioutil.ReadFile(path.Join(dst, "scripts/gen/specification.html"))
+	w.Write([]byte("Not found"))
-	if err != nil {
-		writeError(w, 500, fmt.Errorf("Error reading spec: %v", err))
-		return
-	}
-	w.Write(b)
-	cache.Add(sha, b)
 }
 
 // lookupHeadSHA looks up what origin/master's HEAD SHA is.
@@ -322,7 +353,7 @@ func (s *server) serveRSTDiff(w http.ResponseWriter, req *http.Request) {
 		return
 	}
 
-	diffCmd := exec.Command("diff", "-u", path.Join(base, "scripts", "tmp", "full_spec.rst"), path.Join(head, "scripts", "tmp", "full_spec.rst"))
+	diffCmd := exec.Command("diff", "-r", "-u", path.Join(base, "scripts", "tmp"), path.Join(head, "scripts", "tmp"))
 	var diff bytes.Buffer
 	diffCmd.Stdout = &diff
 	if err := ignoreExitCodeOne(diffCmd.Run()); err != nil {
@@ -366,7 +397,8 @@ func (s *server) serveHTMLDiff(w http.ResponseWriter, req *http.Request) {
 		return
 	}
 
-	cmd := exec.Command(htmlDiffer, path.Join(base, "scripts", "gen", "specification.html"), path.Join(head, "scripts", "gen", "specification.html"))
+	requestedPath := extractPath(req.URL.Path, "/diff/spec/pr")
+	cmd := exec.Command(htmlDiffer, path.Join(base, "scripts", "gen", requestedPath), path.Join(head, "scripts", "gen", requestedPath))
 	var b bytes.Buffer
 	cmd.Stdout = &b
 	if err := cmd.Run(); err != nil {

specification/intro.rst

@@ -8,6 +8,16 @@ https://github.com/matrix-org/matrix-doc using
 https://github.com/matrix-org/matrix-doc/blob/master/scripts/gendoc.py as of
 revision ``{{git_version}}`` - https://github.com/matrix-org/matrix-doc/tree/{{git_rev}}
 
+APIs
+~~~~
+The following APIs are documented in this specification:
+
+- `Client-Server API <client_server.html>`_ for writing Matrix clients.
+- `Server-Server API <server_server.html>`_ for writing servers which can federate with Matrix.
+- `Application Service API <application_service.html>`_ for writing privileged plugins to servers.
+
+There are also some `appendices <appendices.html>`_.
+
 Changelog
 ~~~~~~~~~
 {{spec_changelog}}

specification/modules/typing_notifications.rst

@@ -5,10 +5,8 @@ Typing Notifications
 
 Users may wish to be informed when another user is typing in a room. This can be
 achieved using typing notifications. These are ephemeral events scoped to a
-``room_id``. This means they do not form part of the `Event Graph`_ but still
+``room_id``. This means they do not form part of the
-have a ``room_id`` key.
+`Event Graph <index.html#event-graphs>`_ but still have a ``room_id`` key.
-
-.. _Event Graph: `sect:event-graph`_
 
 Events
 ------

specification/targets.yaml

@@ -1,16 +1,23 @@
 targets:
-  main:  # arbitrary name to identify this build target
+  index:
     files:  # the sort order of files to cat
       - intro.rst
+  client_server:
+    files:
       - client_server_api.rst
       - { 1: events.rst }
       - { 1: event_signing.rst }
       - modules.rst
       - { 1: feature_profiles.rst }
       - { 1: "group:modules" }  # reference a group of files
+  application_service:
+    files:
       - application_service_api.rst
+  server_server:
+    files:
       - server_server_api.rst
-      - identity_servers.rst
+  appendices:
+    files:
       - appendices.rst
 groups:  # reusable blobs of files when prefixed with 'group:'
   modules: