wholetrans/docs-matrix-spec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge branch 'master' into markjh/end_to_end_encryption

Browse source
This commit is contained in:
Richard van der Hoff 2015-11-27 11:49:11 +00:00
commit 51ca25a472
13 changed files with 224 additions and 133 deletions
Download patch file Download diff file Expand all files Collapse all files

47
api/client-server/v1/room_send.yaml
View file

@ -76,50 +76,3 @@ paths:
type: string
description: |-
A unique identifier for the event.
"/rooms/{roomId}/send/{eventType}":
post:
summary: Send a message event to the given room.
description: |-
This endpoint can be used to send a message event to a room; however
the lack of a transaction ID means that it is possible to cause message
duplication if events are resent on error, so it is preferable to use
`PUT /_matrix/client/api/v1/rooms/{roomId}/send/{eventType}/{txnId}`_.
security:
- accessToken: []
parameters:
- in: path
type: string
name: roomId
description: The room to send the event to.
required: true
x-example: "!636q39766251:example.com"
- in: path
type: string
name: eventType
description: The type of event to send.
required: true
x-example: "m.room.message"
- in: body
name: body
schema:
type: object
example: |-
{
"msgtype": "m.text",
"body": "hello"
}
responses:
200:
description: "An ID for the sent event."
examples:
application/json: |-
{
"event_id": "YUwRidLecu"
}
schema:
type: object
properties:
event_id:
type: string
description: |-
A unique identifier for the event.

20
drafts/markjh_end_to_end.rst
View file

@ -84,18 +84,22 @@ follows:
.. math::
\begin{align}
R_{2^24n,0} &= H_1\left(R_{2^24(i-1),0}\right) \\
R_{2^24n,1} &= H_2\left(R_{2^24(i-1),0}\right) \\
R_{2^24n,0} &= H_0\left(R_{2^24(i-1),0}\right) \\
R_{2^24n,1} &= H_1\left(R_{2^24(i-1),0}\right) \\
R_{2^24n,2} &= H_2\left(R_{2^24(i-1),0}\right) \\
R_{2^24n,3} &= H_3\left(R_{2^24(i-1),0}\right) \\
R_{2^16n,1} &= H_1\left(R_{2^16(i-1),1}\right) \\
R_{2^16n,2} &= H_2\left(R_{2^16(i-1),1}\right) \\
R_{2^8i,2} &= H_1\left(R_{2^8(i-1),2}\right) \\
R_{2^8i,3} &= H_2\left(R_{2^8(i-1),2}\right) \\
R_{i,3} &= H_1\left(R_{(i-1),3}\right)
R_{2^16n,3} &= H_3\left(R_{2^16(i-1),1}\right) \\
R_{2^8i,2} &= H_2\left(R_{2^8(i-1),2}\right) \\
R_{2^8i,3} &= H_3\left(R_{2^8(i-1),2}\right) \\
R_{i,3} &= H_3\left(R_{(i-1),3}\right)
\end{align}
Where :math:`H_1` and :math:`H_2` are different hash functions. For example
:math:`H_1` could be :math:`HMAC\left(X,\text{"\textbackslash x01"}\right)` and
:math:`H_2` could be :math:`HMAC\left(X,\text{"\textbackslash x02"}\right)`.
Where :math:`H_0`, :math:`H_1`, :math:`H_2`, and :math:`H_3`
are different hash functions. For example
:math:`H_0` could be :math:`HMAC\left(X,\text{"\textbackslash x00"}\right)` and
:math:`H_1` could be :math:`HMAC\left(X,\text{"\textbackslash x01"}\right)`.
So every :math:`2^24` iterations :math:`R_{n,1}` is reseeded from :math:`R_{n,0}`.
Every :math:`2^16` iterations :math:`R_{n,2}` is reseeded from :math:`R_{n,1}`.

2
event-schemas/check_examples.py
View file

@ -60,6 +60,8 @@ def check_example_dir(exampledir, schemadir):
continue
examplepath = os.path.join(root, filename)
schemapath = examplepath.replace(exampledir, schemadir)
if schemapath.find("#") >= 0:
schemapath = schemapath[:schemapath.find("#")]
try:
check_example_file(examplepath, schemapath)
except Exception as e:

30
event-schemas/examples/v1/m.room.member#invite_room_state Normal file
View file

@ -0,0 +1,30 @@
{
"age": 242352,
"content": {
"membership": "join",
"avatar_url": "mxc://localhost/SEsfnsuifSDFSSEF#auto",
"displayname": "Alice Margatroid"
},
"invite_room_state": [
{
"type": "m.room.name",
"state_key": "",
"content": {
"name": "Forest of Magic"
}
},
{
"type": "m.room.join_rules",
"state_key": "",
"content": {
"join_rules": "invite"
}
}
],
"state_key": "@alice:localhost",
"origin_server_ts": 1431961217939,
"event_id": "$WLGTSEFSEF:localhost",
"type": "m.room.member",
"room_id": "!Cuyf34gef24t:localhost",
"user_id": "@example:localhost"
}

25
event-schemas/examples/v1/m.room.member#third_party_invite Normal file
View file

@ -0,0 +1,25 @@
{
"age": 242352,
"content": {
"membership": "join",
"avatar_url": "mxc://localhost/SEsfnsuifSDFSSEF#auto",
"displayname": "Alice Margatroid",
"third_party_invite": {
"signed": {
"mxid": "@alice:localhost",
"signatures": {
"magic.forest": {
"ed25519:3": "fQpGIW1Snz+pwLZu6sTy2aHy/DYWWTspTJRPyNp0PKkymfIsNffysMl6ObMMFdIJhk6g6pwlIqZ54rxo8SLmAg"
}
},
"token": "abc123"
}
}
},
"state_key": "@alice:localhost",
"origin_server_ts": 1431961217939,
"event_id": "$WLGTSEFSEF:localhost",
"type": "m.room.member",
"room_id": "!Cuyf34gef24t:localhost",
"user_id": "@example:localhost"
}

9
jenkins.sh
View file

@ -7,3 +7,12 @@ set -ex
(cd scripts && ./gendoc.py -v)
(cd api && npm install && node validator.js -s "client-server/v1" && node validator.js -s "client-server/v2_alpha")
(cd event-schemas/ && ./check.sh)
: ${GOPATH:=${WORKSPACE}/.gopath}
mkdir -p "${GOPATH}"
export GOPATH
go get github.com/hashicorp/golang-lru
go get gopkg.in/fsnotify.v1
(cd scripts/continuserv && go build)
(cd scripts/speculator && go build)

47
scripts/matrix-org-gendoc.sh → scripts/add-matrix-org-stylings.sh
View file

@ -1,41 +1,20 @@
#! /bin/bash
#!/bin/bash -eu
if [ -z "$1" ]; then
echo "Expected /includes/head.html file as 1st arg."
exit 1
if [[ $# != 1 || ! -d $1 ]]; then
echo >&2 "Usage: $0 include_dir"
exit 1
fi
if [ -z "$2" ]; then
echo "Expected /includes/nav.html file as 2nd arg."
HEADER="$1/head.html"
NAV_BAR="$1/nav.html"
FOOTER="$1/footer.html"
for f in "$1"/{head,nav,footer}.html; do
if [[ ! -e "${f}" ]]; then
echo >&2 "Need ${f} to exist"
exit 1
fi
if [ -z "$3" ]; then
echo "Expected /includes/footer.html file as 3rd arg."
exit 1
fi
HEADER=$1
NAV_BAR=$2
FOOTER=$3
if [ ! -f $HEADER ]; then
echo $HEADER " does not exist"
exit 1
fi
if [ ! -f $NAV_BAR ]; then
echo $NAV_BAR " does not exist"
exit 1
fi
if [ ! -f $FOOTER ]; then
echo $FOOTER " does not exist"
exit 1
fi
python gendoc.py
fi
done
perl -MFile::Slurp -pi -e 'BEGIN { $header = read_file("'$HEADER'") } s#<head>#<head>$header
<link rel="stylesheet" href="//matrix.org/docs/guides/css/docs_overrides.css">

13
scripts/gendoc.py
View file

@ -238,6 +238,18 @@ def rst2html(i, o):
)
def addAnchors(path):
with open(path, "r") as f:
lines = f.readlines()
replacement = replacement = r'<p><a class="anchor" id="\3"></a></p>\n\1'
with open(path, "w") as f:
for line in lines:
line = re.sub(r'(<h\d id="#?(.*?)">)', replacement, line.rstrip())
line = re.sub(r'(<div class="section" (id)="(.*?)">)', replacement, line.rstrip())
f.write(line + "\n")
def run_through_template(input, set_verbose):
tmpfile = './tmp/output'
try:
@ -387,6 +399,7 @@ def main(target_name, keep_intermediates):
shutil.copy("../supporting-docs/howtos/client-server.rst", "tmp/howto.rst")
run_through_template("tmp/howto.rst", False) # too spammy to mark -v on this
rst2html("tmp/full_spec.rst", "gen/specification.html")
addAnchors("gen/specification.html")
rst2html("tmp/howto.rst", "gen/howtos.html")
if not keep_intermediates:
cleanup_env()

137
scripts/speculator/main.go
View file

@ -23,6 +23,7 @@ import (
"path"
"strconv"
"strings"
"sync"
"syscall"
"time"
@ -53,9 +54,11 @@ type User struct {
}
var (
port = flag.Int("port", 9000, "Port on which to listen for HTTP")
allowedMembers map[string]bool
specCache *lru.Cache // string -> []byte
port = flag.Int("port", 9000, "Port on which to listen for HTTP")
includesDir = flag.String("includes_dir", "", "Directory containing include files for styling like matrix.org")
allowedMembers map[string]bool
specCache *lru.Cache // string -> []byte
styledSpecCache *lru.Cache // string -> []byte
)
func (u *User) IsTrusted() bool {
@ -63,19 +66,22 @@ func (u *User) IsTrusted() bool {
}
const (
pullsPrefix = "https://api.github.com/repos/matrix-org/matrix-doc/pulls"
matrixDocCloneURL = "https://github.com/matrix-org/matrix-doc.git"
pullsPrefix = "https://api.github.com/repos/matrix-org/matrix-doc/pulls"
matrixDocCloneURL = "https://github.com/matrix-org/matrix-doc.git"
permissionsOwnerFull = 0700
)
func gitClone(url string, shared bool) (string, error) {
directory := path.Join("/tmp/matrix-doc", strconv.FormatInt(rand.Int63(), 10))
cmd := exec.Command("git", "clone", url, directory)
if shared {
cmd.Args = append(cmd.Args, "--shared")
if err := os.MkdirAll(directory, permissionsOwnerFull); err != nil {
return "", fmt.Errorf("error making directory %s: %v", directory, err)
}
if err := cmd.Run(); err != nil {
return "", fmt.Errorf("error cloning repo: %v", err)
args := []string{"clone", url, directory}
if shared {
args = append(args, "--shared")
}
if err := runGitCommand(directory, args); err != nil {
return "", err
}
return directory, nil
}
@ -84,15 +90,13 @@ func gitCheckout(path, sha string) error {
return runGitCommand(path, []string{"checkout", sha})
}
func gitFetch(path string) error {
return runGitCommand(path, []string{"fetch"})
}
func runGitCommand(path string, args []string) error {
cmd := exec.Command("git", args...)
cmd.Dir = path
var b bytes.Buffer
cmd.Stderr = &b
if err := cmd.Run(); err != nil {
return fmt.Errorf("error running %q: %v", strings.Join(cmd.Args, " "), err)
return fmt.Errorf("error running %q: %v (stderr: %s)", strings.Join(cmd.Args, " "), err, b.String())
}
return nil
}
@ -136,17 +140,35 @@ func writeError(w http.ResponseWriter, code int, err error) {
}
type server struct {
mu sync.Mutex // Must be locked around any git command on matrixDocCloneURL
matrixDocCloneURL string
}
func (s *server) updateBase() error {
s.mu.Lock()
defer s.mu.Unlock()
return runGitCommand(s.matrixDocCloneURL, []string{"fetch"})
}
// canCheckout returns whether a given sha can currently be checked out from s.matrixDocCloneURL.
func (s *server) canCheckout(sha string) bool {
s.mu.Lock()
defer s.mu.Unlock()
return runGitCommand(s.matrixDocCloneURL, []string{"cat-file", "-e", sha + "^{commit}"}) == nil
}
// generateAt generates spec from repo at sha.
// Returns the path where the generation was done.
func (s *server) generateAt(sha string) (dst string, err error) {
err = gitFetch(s.matrixDocCloneURL)
if err != nil {
return
if !s.canCheckout(sha) {
err = s.updateBase()
if err != nil {
return
}
}
s.mu.Lock()
dst, err = gitClone(s.matrixDocCloneURL, true)
s.mu.Unlock()
if err != nil {
return
}
@ -164,7 +186,10 @@ func (s *server) getSHAOf(ref string) (string, error) {
cmd.Dir = path.Join(s.matrixDocCloneURL)
var b bytes.Buffer
cmd.Stdout = &b
if err := cmd.Run(); err != nil {
s.mu.Lock()
err := cmd.Run()
s.mu.Unlock()
if err != nil {
return "", fmt.Errorf("error generating spec: %v\nOutput from gendoc:\n%v", err, b.String())
}
return strings.TrimSpace(b.String()), nil
@ -173,17 +198,22 @@ func (s *server) getSHAOf(ref string) (string, error) {
func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) {
var sha string
var styleLikeMatrixDotOrg = req.URL.Query().Get("matrixdotorgstyle") != ""
if styleLikeMatrixDotOrg && *includesDir == "" {
writeError(w, 500, fmt.Errorf("Cannot style like matrix.org - no include dir specified"))
return
}
if strings.ToLower(req.URL.Path) == "/spec/head" {
if err := gitFetch(s.matrixDocCloneURL); err != nil {
// err may be non-nil here but if headSha is non-empty we will serve a possibly-stale result in favour of erroring.
// This is to deal with cases like where github is down but we still want to serve the spec.
if headSha, err := s.lookupHeadSHA(); headSha == "" {
writeError(w, 500, err)
return
} else {
sha = headSha
}
originHead, err := s.getSHAOf("origin/master")
if err != nil {
writeError(w, 500, err)
return
}
sha = originHead
} else {
pr, err := lookupPullRequest(*req.URL, "/spec")
if err != nil {
@ -199,7 +229,13 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) {
}
sha = pr.Head.SHA
}
if cached, ok := specCache.Get(sha); ok {
var cache = specCache
if styleLikeMatrixDotOrg {
cache = styledSpecCache
}
if cached, ok := cache.Get(sha); ok {
w.Write(cached.([]byte))
return
}
@ -211,13 +247,43 @@ func (s *server) serveSpec(w http.ResponseWriter, req *http.Request) {
return
}
if styleLikeMatrixDotOrg {
cmd := exec.Command("./add-matrix-org-stylings.sh", *includesDir)
cmd.Dir = path.Join(dst, "scripts")
var b bytes.Buffer
cmd.Stderr = &b
if err := cmd.Run(); err != nil {
writeError(w, 500, fmt.Errorf("error styling spec: %v\nOutput:\n%v", err, b.String()))
return
}
}
b, err := ioutil.ReadFile(path.Join(dst, "scripts/gen/specification.html"))
if err != nil {
writeError(w, 500, fmt.Errorf("Error reading spec: %v", err))
return
}
w.Write(b)
specCache.Add(sha, b)
cache.Add(sha, b)
}
// lookupHeadSHA looks up what origin/master's HEAD SHA is.
// It attempts to `git fetch` before doing so.
// If this fails, it may still return a stale sha, but will also return an error.
func (s *server) lookupHeadSHA() (sha string, retErr error) {
retErr = s.updateBase()
if retErr != nil {
log.Printf("Error fetching: %v, attempting to fall back to current known value", retErr)
}
originHead, err := s.getSHAOf("origin/master")
if err != nil {
retErr = err
}
sha = originHead
if retErr != nil && originHead != "" {
log.Printf("Successfully fell back to possibly stale sha: %s", sha)
}
return
}
func checkAuth(pr *PullRequest) error {
@ -344,6 +410,10 @@ func listPulls(w http.ResponseWriter, req *http.Request) {
pull.Number, pull.User.HTMLURL, pull.User.Login, pull.HTMLURL, pull.Title, pull.Number, pull.Number, pull.Number)
}
s += `</ul><div><a href="spec/head">View the spec at head</a></div></body>`
if *includesDir != "" {
s += `</ul><div><a href="spec/head?matrixdotorgstyle=1">View the spec at head styled like matrix.org</a></div></body>`
}
io.WriteString(w, s)
}
@ -383,7 +453,7 @@ func main() {
if err != nil {
log.Fatal(err)
}
s := server{masterCloneDir}
s := server{matrixDocCloneURL: masterCloneDir}
http.HandleFunc("/spec/", forceHTML(s.serveSpec))
http.HandleFunc("/diff/rst/", s.serveRSTDiff)
http.HandleFunc("/diff/html/", forceHTML(s.serveHTMLDiff))
@ -408,7 +478,10 @@ func serveText(s string) func(http.ResponseWriter, *http.Request) {
}
func initCache() error {
c, err := lru.New(50) // Evict after 50 entries (i.e. 50 sha1s)
specCache = c
c1, err := lru.New(50) // Evict after 50 entries (i.e. 50 sha1s)
specCache = c1
c2, err := lru.New(50) // Evict after 50 entries (i.e. 50 sha1s)
styledSpecCache = c2
return err
}

8
specification/modules/instant_messaging.rst
View file

@ -62,10 +62,8 @@ resulting ``mxc://`` URI can then be used in the ``url`` key.
Recommendations when sending messages
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Clients can send messages using ``POST`` or ``PUT`` requests. Clients SHOULD use
``PUT`` requests with `transaction IDs`_ to make requests idempotent. This
ensures that messages are sent exactly once even under poor network conditions.
Clients SHOULD retry requests using an exponential-backoff algorithm for a
In the event of send failure, clients SHOULD retry requests using an
exponential-backoff algorithm for a
certain amount of time T. It is recommended that T is no longer than 5 minutes.
After this time, the client should stop retrying and mark the message as "unsent".
Users should be able to manually resend unsent messages.
@ -78,8 +76,6 @@ reduce the impact of head-of-line blocking, clients should use a queue per room
rather than a global queue, as ordering is only relevant within a single room
rather than between rooms.
.. _`transaction IDs`: `sect:txn_ids`_
Local echo
~~~~~~~~~~

4
templating/matrix_templates/sections.py
View file

@ -35,7 +35,7 @@ class MatrixSections(Sections):
if not filterFn(event_name):
continue
sections.append(template.render(
example=examples[event_name],
examples=examples[event_name],
event=schemas[event_name],
title_kind=subtitle_title_char
))
@ -136,7 +136,7 @@ class MatrixSections(Sections):
if not event_name.startswith("m.room.message#m."):
continue
sections.append(template.render(
example=examples[event_name],
example=examples[event_name][0],
event=schemas[event_name],
title_kind=subtitle_title_char
))

4
templating/matrix_templates/templates/events.tmpl
View file

@ -12,8 +12,10 @@
{{ tables.paramtable(table.rows, [(table.title or "Content") ~ " Key", "Type", "Description"]) }}
{% endfor %}
Example:
Example{% if examples | length > 1 %}s{% endif %}:
{% for example in examples %}
.. code:: json
{{example | jsonify(4, 4)}}
{% endfor %}

11
templating/matrix_templates/units.py
View file

@ -572,9 +572,14 @@ class MatrixUnits(Units):
if not filename.startswith("m."):
continue
with open(os.path.join(path, filename), "r") as f:
examples[filename] = json.loads(f.read())
if filename == "m.room.message#m.text":
examples["m.room.message"] = examples[filename]
event_name = filename.split("#")[0]
example = json.loads(f.read())
examples[filename] = examples.get(filename, [])
examples[filename].append(example)
if filename != event_name:
examples[event_name] = examples.get(event_name, [])
examples[event_name].append(example)
return examples
def load_event_schemas(self):