initial http api proposal for initialsync, eventstream and history

+ our theoretical design workflow atm

drafts/cs-v2-http-api.rst

@@ -28,8 +28,8 @@ POST /user/{userId}/filter
 {
     // selectors: (bluntly selecting on the unencrypted fields)
     types: [ "m.*", "net.arasphere.*" ],    // default: all
-    rooms: [ "!83wy7whi:matrix.org" ],      // default: all
+    rooms: [ "!83wy7whi:matrix.org" ],      // default: all (may be aliases or IDs. wildcards supported)
-    sender_ids: [ "@matthew:matrix.org" ],  // default: all (e.g. for narrowing down presence, and stalker mode)
+    sender_ids: [ "@matthew:matrix.org" ],  // default: all (e.g. for narrowing down presence, and stalker mode. wildcards supported)
     
     // XXX: do we want this per-query; is it valid to reuse it?
     // we probably don't need this as querying per-event-ID will be a parameter from a seperate API,
@@ -49,7 +49,7 @@ POST /user/{userId}/filter
     // sorting/pagination/threading
     select: [ "event_id", "origin_server_ts", "thread_id", "content", "content.body" ], 
     
-    // include bundled child-event updates
+    // include bundled child-event updates (default false)
     bundle_updates: true,
     
     // include bundled related events
@@ -91,6 +91,7 @@ POST /user/{userId}/filter
     ],
 }
 
+Returns:
 200 OK
 {
     "filter_id": "583e98c2d983",
@@ -100,28 +101,36 @@ POST /user/{userId}/filter
 Global initial sync API
 -----------------------
 
-XXX: need much more concrete general API definition first
-
 GET /initialSync
 
 GET parameters::
-    limit: maximum number of events
+    limit: maximum number of events per room to return
-    <generic pagination parameters (per room)>
+    sort: fieldname, direction (e.g. "sender_id,asc"). // default: "timeline,asc". may appear multiple times.
-    <some way of doing an incremental sync using streaming_tokens>
+    since: <chunk token> to request an incremental update (*not* pagination) since the specified chunk token
-    filter: filter_id (XXX: allow different filters per room?)
+        We call this 'since' rather than 'from' because it's not for pagination,
-    presence: true/false
+    backfill: true/false (default true): do we want to pull in state from federation if we have less than <limit> events available for a room?
+    presence: true/false (default true): return presence info
     compact: boolean (default false): factor out common events.
              XXX: I *really* think this should be turned on by default --matthew
+    filter: <filter_id> (XXX: allow different filters per room?)
+    # filter overrides:
+    filter_type: wildcard event type match e.g. "m.*": default, all.  may appear multiple times.
+    filter_room: wildcard room id/name match e.g. "!83wy7whi:matrix.org": default, all.  may appear multiple times.
+    filter_sender_id: wildcard sender id match e.g. "@matthew:matrix.org": default, all.  may appear multiple times.
+    filter_event_id: event id to match e.g. "$192318719:matrix.org" // default, all: may appear multiple times
+    filter_format: "federation" or "events"
+    filter_select: event fields to return: default, all.  may appear multiple times
+    filter_bundle_updates: true/false: default, false. bundle updates in events.
 
+// FIXME: kegan: how much does the v1 response actually change here?
 
-FIXME: example using v1 API just for compact proofing.  Needs to be updated with streaming_token voodoo
+Returns:
 200 OK
 // where compact is false:
 {
-    // XXX: does "end" die now?
+    "end": "s72595_4483_1934", // the chunk token we pass to from=
-    "end": "s72595_4483_1934",
     
-    // global presence info. (XXX: should we only send content deltas if this is an delta initialsync - e.g. to avoid re-sending all avatar_urls?)
+    // global presence info (if presence=true)
     "presence": [{
         "content": {
             "avatar_url": "http://matrix.tp.mu:8008/_matrix/content/QG1hdHRoZXc6dHAubXUOeJQMWFMvUdqdeLovZKsyaOT.aW1hZ2UvanBlZw==.jpeg",
@@ -135,7 +144,7 @@ FIXME: example using v1 API just for compact proofing.  Needs to be updated with
     
     "rooms": [{
         "membership": "join",
-        "messages": {
+        "eventStream": { // rename messages to eventstream as this is a list of all events, not just messages (non-state events)
             "chunk": [{
                 "content": {
                     "avatar_url": "https://matrix.org/_matrix/content/QG1hdHRoZXc6bWF0cml4Lm9yZwxaesQWnqdynuXIYaRisFnZdG.aW1hZ2UvanBlZw==.jpeg",
@@ -217,10 +226,10 @@ FIXME: example using v1 API just for compact proofing.  Needs to be updated with
             }
         },
         "membership": "join",
-        "messages": {
+        "eventStream": { // rename messages to eventstream as this is a list of all events, not just messages (non-state events)
             "chunk": [ "$1417731086506PgoVf:matrix.org" ],
             "end": "s72595_4483_1934",
-            "start": "t67-41151_4483_1934"
+            "start": "t67-41151_4483_1934" // XXX: do we need start?
         },
         "room_id": "!KrLWMLDnZAyTapqLWW:matrix.org",
         "state": [ "$1417731086506PgoVf:matrix.org" ],
@@ -231,17 +240,141 @@ FIXME: example using v1 API just for compact proofing.  Needs to be updated with
 Event Stream API
 ----------------
 
+GET /eventStream
+GET parameters::
+    from: chunk token to continue streaming from (e.g. "end" given by initialsync)
+    filter*: as per initialSync (XXX: do we inherit this from the chunk token?)
+    // N.B. there is no limit or sort param here, as we get events in timeline order as fast as they come.
+    access_token: identifies both user and device
+    timeout: maximum time to poll before returning the request
+    presence: "offline" // optional parameter to tell the server not to interpret this as coming online
+
+XXX: this needs to be updated from v1.  Presumably s/user_id/sender_id/?
+
+Returns:
+200 OK
+
+// events precisely as per a room's eventStream key as returned by initialSync
+// includes non-graph events like presence
+{
+    "chunk": [{
+        "content": {
+            "avatar_url": "https://matrix.org/_matrix/content/QG1hdHRoZXc6bWF0cml4Lm9yZwxaesQWnqdynuXIYaRisFnZdG.aW1hZ2UvanBlZw==.jpeg",
+            "displayname": "Matthew",
+            "last_active_ago": 1241,
+            "presence": "online",
+            "user_id": "@matthew:matrix.org"
+        },
+        "type": "m.presence"
+    }, {
+        "age": 2595,
+        "content": {
+            "body": "test",
+            "msgtype": "m.text"
+        },
+        "event_id": "$14211894201675TMbmz:matrix.org",
+        "origin_server_ts": 1421189420147,
+        "room_id": "!cURbafjkfsMDVwdRDQ:matrix.org",
+        "type": "m.room.message",
+        "user_id": "@matthew:matrix.org"
+    }],
+    "end": "s75460_2478_981",
+    "start": "s75459_2477_981" // XXX: do we need start here?
+}
+
 Room Creation API
 -----------------
 
 Joining API
 -----------
 
+Room History
+------------
+
 Scrollback API
---------------
+~~~~~~~~~~~~~~
+
+GET /rooms/<room_id>/events
+GET parameters::
+    from: the chunk token to paginate from
+    Otherwise same as initialSync, except "compact", "since" and "presence" are not implemented
+
+Returns:
+200 OK
+
+// events precisely as per a room's eventStream key as returned by initialSync
+{
+    "chunk": [{
+        "age": 28153452, // XXX: age and origin_server_ts are redundant here surely
+        "content": {
+            "body": "but obviously the XSF believes XMPP is the One True Way",
+            "msgtype": "m.text"
+        },
+        "event_id": "$1421165049511TJpDp:matrix.org",
+        "origin_server_ts": 1421165049435,
+        "room_id": "!cURbafjkfsMDVwdRDQ:matrix.org",
+        "type": "m.room.message",
+        "user_id": "@irc_Arathorn:matrix.org"
+    }, {
+        "age": 28167245,
+        "content": {
+            "body": "which is all fair enough",
+            "msgtype": "m.text"
+        },
+        "event_id": "$1421165035510CBwsU:matrix.org",
+        "origin_server_ts": 1421165035643,
+        "room_id": "!cURbafjkfsMDVwdRDQ:matrix.org",
+        "type": "m.room.message",
+        "user_id": "@irc_Arathorn:matrix.org"
+    }],
+    "end": "t9571-74545_2470_979",
+    "start": "t9601-75400_2470_979" // XXX: don't we just need end here as we can only paginate one way?
+}
 
 Contextual windowing API
-------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+GET /events/<event_id>
+GET parameters:
+    context: "before", "after" or "around"
+    Otherwise same as initialSync, except "since" and "presence" are not implemented
+    
+Returns:
+200 OK
+
+// the room in question, formatted exactly as a room entry returned by /initialSync
+// with the event in question present in the list as determined by the context param
+{
+    "events": {
+        "$1417731086506PgoVf:matrix.org": {
+            "content": {
+                "avatar_url": "https://matrix.org/_matrix/content/QG1hdHRoZXc6bWF0cml4Lm9yZwxaesQWnqdynuXIYaRisFnZdG.aW1hZ2UvanBlZw==.jpeg",
+                "displayname": "Matthew",
+                "membership": "join"
+            },
+            "membership": "join",
+            "origin_server_ts": 1417731086795,
+            "prev_state": [["$1416420706925RVAWP:matrix.org", {
+                "sha256": "zVzi02R5aeO2HQDnybu1XuuyR6yBG8utLE/i1Sv8eyA"
+            }
+            ]],
+            "room_id": "!KrLWMLDnZAyTapqLWW:matrix.org",
+            "state_key": "@matthew:matrix.org",
+            "type": "m.room.member",
+            "user_id": "@matthew:matrix.org"    
+        }
+    },
+    "membership": "join",
+    "eventStream": {
+        "chunk": [ "$1417731086506PgoVf:matrix.org" ],
+        "end": "s72595_4483_1934",
+        "start": "t67-41151_4483_1934"
+    },
+    "room_id": "!KrLWMLDnZAyTapqLWW:matrix.org",
+    "state": [ "$1417731086506PgoVf:matrix.org" ],
+    "visibility": "public"
+}
+
 
 Room Alias API
 --------------
@@ -270,6 +403,18 @@ Actions API
 Presence API
 ------------
 
+PUT /user/{userId}/presence/m.status // set DND/asleep/on holiday etc -
+// XXX: do we need to distinguish between internationalisable presets like DND
+// and free-form textual status messages?
+// XXX: should this be in /user/{userId}/data/m.status instead?
+// what's actually the difference? surely status is no different to avatar
+// updates in terms of needing to be pushed around
+
+PUT /device/{deviceId}/presence/m.presence // explicitly set online/idle/offline
+// or /presence/device/{deviceId}
+
+// XXX: need to remember how to handle activity notifications
+
 Typing API
 ----------
 

drafts/design_workflow.rst

@@ -0,0 +1,14 @@
+Matrix Spec Design Workflow
+===========================
+
+1. Write use cases
+
+2. Design data flows for use cases
+
+3. Design generic API (factoring out commonalities where possible)
+
+4. Design transport-specific API with justifications
+
+5. Formalise transport-specific API as swagger or similar
+
+6. Evolve the generic API design doc and transport-specific API into the actual spec.