)', replacement, line.rstrip())
+ f.write(line + "\n")
+
+
def run_through_template(input, set_verbose):
tmpfile = './tmp/output'
try:
@@ -387,6 +399,7 @@ def main(target_name, keep_intermediates):
shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst")
run_through_template("tmp/howto.rst", False) # too spammy to mark -v on this
rst2html("tmp/full_spec.rst", "gen/specification.html")
+ addAnchors("gen/specification.html")
rst2html("tmp/howto.rst", "gen/howtos.html")
if not keep_intermediates:
cleanup_env()
diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go
index 97e67c8c..74731ff6 100644
--- a/scripts/speculator/main.go
+++ b/scripts/speculator/main.go
@@ -23,6 +23,7 @@ import (
"path"
"strconv"
"strings"
+ "sync"
"syscall"
"time"
@@ -53,9 +54,11 @@ type User struct {
}
var (
- port = flag.Int("port", 9000, "Port on which to listen for HTTP")
- allowedMembers map[string]bool
- specCache *lru.Cache // string -> []byte
+ port = flag.Int("port", 9000, "Port on which to listen for HTTP")
+ includesDir = flag.String("includes_dir", "", "Directory containing include files for styling like matrix.org")
+ allowedMembers map[string]bool
+ specCache *lru.Cache // string -> []byte
+ styledSpecCache *lru.Cache // string -> []byte
)
func (u *User) IsTrusted() bool {
@@ -63,19 +66,22 @@ func (u *User) IsTrusted() bool {
}
const (
- pullsPrefix = "https://api.github.com/repos/matrix-org/matrix-doc/pulls"
- matrixDocCloneURL = "https://github.com/matrix-org/matrix-doc.git"
+ pullsPrefix = "https://api.github.com/repos/matrix-org/matrix-doc/pulls"
+ matrixDocCloneURL = "https://github.com/matrix-org/matrix-doc.git"
+ permissionsOwnerFull = 0700
)
func gitClone(url string, shared bool) (string, error) {
directory := path.Join("/tmp/matrix-doc", strconv.FormatInt(rand.Int63(), 10))
- cmd := exec.Command("git", "clone", url, directory)
- if shared {
- cmd.Args = append(cmd.Args, "--shared")
+ if err := os.MkdirAll(directory, permissionsOwnerFull); err != nil {
+ return "", fmt.Errorf("error making directory %s: %v", directory, err)
}
-
- if err := cmd.Run(); err != nil {
- return "", fmt.Errorf("error cloning repo: %v", err)
+ args := []string{"clone", url, directory}
+ if shared {
+ args = append(args, "--shared")
+ }
+ if err := runGitCommand(directory, args); err != nil {
+ return "", err
}
return directory, nil
}
@@ -84,15 +90,13 @@ func gitCheckout(path, sha string) error {
return runGitCommand(path, []string{"checkout", sha})
}
-func gitFetch(path string) error {
- return runGitCommand(path, []string{"fetch"})
-}
-
func runGitCommand(path string, args []string) error {
cmd := exec.Command("git", args...)
cmd.Dir = path
+ var b bytes.Buffer
+ cmd.Stderr = &b
if err := cmd.Run(); err != nil {
- return fmt.Errorf("error running %q: %v", strings.Join(cmd.Args, " "), err)
+ return fmt.Errorf("error running %q: %v (stderr: %s)", strings.Join(cmd.Args, " "), err, b.String())
}
return nil
}
@@ -136,17 +140,35 @@ func writeError(w http.ResponseWriter, code int, err error) {
}
type server struct {
+ mu sync.Mutex // Must be locked around any git command on matrixDocCloneURL
matrixDocCloneURL string
}
+func (s *server) updateBase() error {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ return runGitCommand(s.matrixDocCloneURL, []string{"fetch"})
+}
+
+// canCheckout returns whether a given sha can currently be checked out from s.matrixDocCloneURL.
+func (s *server) canCheckout(sha string) bool {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ return runGitCommand(s.matrixDocCloneURL, []string{"cat-file", "-e", sha + "^{commit}"}) == nil
+}
+
// generateAt generates spec from repo at sha.
// Returns the path where the generation was done.
func (s *server) generateAt(sha string) (dst string, err error) {
- err = gitFetch(s.matrixDocCloneURL)
- if err != nil {
- return
+ if !s.canCheckout(sha) {
+ err = s.updateBase()
+ if err != nil {
+ return
+ }
}
+ s.mu.Lock()
dst, err = gitClone(s.matrixDocCloneURL, true)
+ s.mu.Unlock()
if err != nil {
return
}
@@ -164,7 +186,10 @@ func (s *server) getSHAOf(ref string) (string, error) {
cmd.Dir = path.Join(s.matrixDocCloneURL)
var b bytes.Buffer
cmd.Stdout = &b
- if err := cmd.Run(); err != nil {
+ s.mu.Lock()
+ err := cmd.Run()
+ s.mu.Unlock()
+ if err != nil {
return "", fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String())
}
return strings.TrimSpace(b.String()), nil
@@ -173,17 +198,22 @@ func (s *server) getSHAOf(ref string) (string, error) {
func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) {
var sha string
+ var styleLikeMatrixDotOrg = req.URL.Query().Get("matrixdotorgstyle") != ""
+
+ if styleLikeMatrixDotOrg && *includesDir == "" {
+ writeError(w, 500, fmt.Errorf("Cannot style like matrix.org - no include dir specified"))
+ return
+ }
+
if strings.ToLower(req.URL.Path) == "/spec/head" {
- if err := gitFetch(s.matrixDocCloneURL); err != nil {
+ // err may be non-nil here but if headSha is non-empty we will serve a possibly-stale result in favour of erroring.
+ // This is to deal with cases like where github is down but we still want to serve the spec.
+ if headSha, err := s.lookupHeadSHA(); headSha == "" {
writeError(w, 500, err)
return
+ } else {
+ sha = headSha
}
- originHead, err := s.getSHAOf("origin/master")
- if err != nil {
- writeError(w, 500, err)
- return
- }
- sha = originHead
} else {
pr, err := lookupPullRequest(*req.URL, "/spec")
if err != nil {
@@ -199,7 +229,13 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) {
}
sha = pr.Head.SHA
}
- if cached, ok := specCache.Get(sha); ok {
+
+ var cache = specCache
+ if styleLikeMatrixDotOrg {
+ cache = styledSpecCache
+ }
+
+ if cached, ok := cache.Get(sha); ok {
w.Write(cached.([]byte))
return
}
@@ -211,13 +247,43 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) {
return
}
+ if styleLikeMatrixDotOrg {
+ cmd := exec.Command("./add-matrix-org-stylings.sh", *includesDir)
+ cmd.Dir = path.Join(dst, "scripts")
+ var b bytes.Buffer
+ cmd.Stderr = &b
+ if err := cmd.Run(); err != nil {
+ writeError(w, 500, fmt.Errorf("error styling spec: %v\nOutput:\n%v", err, b.String()))
+ return
+ }
+ }
+
b, err := ioutil.ReadFile(path.Join(dst, "scripts/gen/specification.html"))
if err != nil {
writeError(w, 500, fmt.Errorf("Error reading spec: %v", err))
return
}
w.Write(b)
- specCache.Add(sha, b)
+ cache.Add(sha, b)
+}
+
+// lookupHeadSHA looks up what origin/master's HEAD SHA is.
+// It attempts to `git fetch` before doing so.
+// If this fails, it may still return a stale sha, but will also return an error.
+func (s *server) lookupHeadSHA() (sha string, retErr error) {
+ retErr = s.updateBase()
+ if retErr != nil {
+ log.Printf("Error fetching: %v, attempting to fall back to current known value", retErr)
+ }
+ originHead, err := s.getSHAOf("origin/master")
+ if err != nil {
+ retErr = err
+ }
+ sha = originHead
+ if retErr != nil && originHead != "" {
+ log.Printf("Successfully fell back to possibly stale sha: %s", sha)
+ }
+ return
}
func checkAuth(pr *PullRequest) error {
@@ -344,6 +410,10 @@ func listPulls(w http.ResponseWriter, req *http.Request) {
pull.Number, pull.User.HTMLURL, pull.User.Login, pull.HTMLURL, pull.Title, pull.Number, pull.Number, pull.Number)
}
s += `
`
+ if *includesDir != "" {
+ s += `
`
+ }
+
io.WriteString(w, s)
}
@@ -383,7 +453,7 @@ func main() {
if err != nil {
log.Fatal(err)
}
- s := server{masterCloneDir}
+ s := server{matrixDocCloneURL: masterCloneDir}
http.HandleFunc("/spec/", forceHTML(s.serveSpec))
http.HandleFunc("/diff/rst/", s.serveRSTDiff)
http.HandleFunc("/diff/html/", forceHTML(s.serveHTMLDiff))
@@ -408,7 +478,10 @@ func serveText(s string) func(http.ResponseWriter, *http.Request) {
}
func initCache() error {
- c, err := lru.New(50) // Evict after 50 entries (i.e. 50 sha1s)
- specCache = c
+ c1, err := lru.New(50) // Evict after 50 entries (i.e. 50 sha1s)
+ specCache = c1
+
+ c2, err := lru.New(50) // Evict after 50 entries (i.e. 50 sha1s)
+ styledSpecCache = c2
return err
}
diff --git a/specification/modules/instant_messaging.rst b/specification/modules/instant_messaging.rst
index a58c762f..cd385001 100644
--- a/specification/modules/instant_messaging.rst
+++ b/specification/modules/instant_messaging.rst
@@ -62,10 +62,8 @@ resulting ``mxc://`` URI can then be used in the ``url`` key.
Recommendations when sending messages
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Clients can send messages using ``POST`` or ``PUT`` requests. Clients SHOULD use
-``PUT`` requests with `transaction IDs`_ to make requests idempotent. This
-ensures that messages are sent exactly once even under poor network conditions.
-Clients SHOULD retry requests using an exponential-backoff algorithm for a
+In the event of send failure, clients SHOULD retry requests using an
+exponential-backoff algorithm for a
certain amount of time T. It is recommended that T is no longer than 5 minutes.
After this time, the client should stop retrying and mark the message as "unsent".
Users should be able to manually resend unsent messages.
@@ -78,8 +76,6 @@ reduce the impact of head-of-line blocking, clients should use a queue per room
rather than a global queue, as ordering is only relevant within a single room
rather than between rooms.
-.. _`transaction IDs`: `sect:txn_ids`_
-
Local echo
~~~~~~~~~~
diff --git a/templating/matrix_templates/sections.py b/templating/matrix_templates/sections.py
index e75a75af..78aabca7 100644
--- a/templating/matrix_templates/sections.py
+++ b/templating/matrix_templates/sections.py
@@ -35,7 +35,7 @@ class MatrixSections(Sections):
if not filterFn(event_name):
continue
sections.append(template.render(
- example=examples[event_name],
+ examples=examples[event_name],
event=schemas[event_name],
title_kind=subtitle_title_char
))
@@ -136,7 +136,7 @@ class MatrixSections(Sections):
if not event_name.startswith("m.room.message#m."):
continue
sections.append(template.render(
- example=examples[event_name],
+ example=examples[event_name][0],
event=schemas[event_name],
title_kind=subtitle_title_char
))
diff --git a/templating/matrix_templates/templates/events.tmpl b/templating/matrix_templates/templates/events.tmpl
index 324a8f3e..cd165ff0 100644
--- a/templating/matrix_templates/templates/events.tmpl
+++ b/templating/matrix_templates/templates/events.tmpl
@@ -12,8 +12,10 @@
{{ tables.paramtable(table.rows, [(table.title or "Content") ~ " Key", "Type", "Description"]) }}
{% endfor %}
-Example:
+Example{% if examples | length > 1 %}s{% endif %}:
+{% for example in examples %}
.. code:: json
{{example | jsonify(4, 4)}}
+{% endfor %}
diff --git a/templating/matrix_templates/units.py b/templating/matrix_templates/units.py
index e5a7c319..73997c4e 100644
--- a/templating/matrix_templates/units.py
+++ b/templating/matrix_templates/units.py
@@ -572,9 +572,14 @@ class MatrixUnits(Units):
if not filename.startswith("m."):
continue
with open(os.path.join(path, filename), "r") as f:
- examples[filename] = json.loads(f.read())
- if filename == "m.room.message#m.text":
- examples["m.room.message"] = examples[filename]
+ event_name = filename.split("#")[0]
+ example = json.loads(f.read())
+
+ examples[filename] = examples.get(filename, [])
+ examples[filename].append(example)
+ if filename != event_name:
+ examples[event_name] = examples.get(event_name, [])
+ examples[event_name].append(example)
return examples
def load_event_schemas(self):