Add tests for MSC2716 and backfilling history

dockerfiles/synapse/homeserver.yaml

@@ -107,3 +107,5 @@ experimental_features:
   msc2403_enabled: true
   # Enable spaces support
   spaces_enabled: true
+  # Enable history backfilling support
+  msc2716_enabled: true

dockerfiles/synapse/workers-shared.yaml

@@ -64,5 +64,7 @@ federation_rr_transactions_per_room_per_second: 9999
 experimental_features:
   # Enable knocking support
   msc2403_enabled: true
+  # Enable history backfilling support
+  msc2716_enabled: true
   # Enable spaces support
   spaces_enabled: true

internal/b/hs_with_application_service.go

@@ -2,7 +2,7 @@ package b
 
 // BlueprintHSWithApplicationService who has an application service to interact with
 var BlueprintHSWithApplicationService = MustValidate(Blueprint{
-	Name: "alice",
+	Name: "hs_with_application_service",
 	Homeservers: []Homeserver{
 		{
 			Name: "hs1",
@@ -21,5 +21,14 @@ var BlueprintHSWithApplicationService = MustValidate(Blueprint{
 				},
 			},
 		},
+		{
+			Name: "hs2",
+			Users: []User{
+				{
+					Localpart:   "@charlie",
+					DisplayName: "Charlie",
+				},
+			},
+		},
 	},
 })

internal/client/client.go

@@ -132,13 +132,13 @@ func (c *CSAPI) SendEventSynced(t *testing.T, roomID string, e b.Event) string {
 // Will time out after CSAPI.SyncUntilTimeout.
 func (c *CSAPI) SyncUntilTimelineHas(t *testing.T, roomID string, check func(gjson.Result) bool) {
 	t.Helper()
-	c.SyncUntil(t, "", "rooms.join."+GjsonEscape(roomID)+".timeline.events", check)
+	c.SyncUntil(t, "", "", "rooms.join."+GjsonEscape(roomID)+".timeline.events", check)
 }
 
 // SyncUntil blocks and continually calls /sync until the `check` function returns true.
 // If the `check` function fails the test, the failing event will be automatically logged.
 // Will time out after CSAPI.SyncUntilTimeout.
-func (c *CSAPI) SyncUntil(t *testing.T, since, key string, check func(gjson.Result) bool) {
+func (c *CSAPI) SyncUntil(t *testing.T, since, filter, key string, check func(gjson.Result) bool) {
 	t.Helper()
 	start := time.Now()
 	checkCounter := 0
@@ -152,6 +152,9 @@ func (c *CSAPI) SyncUntil(t *testing.T, since, key string, check func(gjson.Resu
 		if since != "" {
 			query["since"] = []string{since}
 		}
+		if filter != "" {
+			query["filter"] = []string{filter}
+		}
 		res := c.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "sync"}, WithQueries(query))
 		body := ParseJSON(t, res)
 		since = GetJSONFieldStr(t, body, "next_batch")
@@ -324,7 +327,7 @@ func (c *CSAPI) DoFunc(t *testing.T, method string, paths []string, opts ...Requ
 }
 
 // NewLoggedClient returns an http.Client which logs requests/responses
-func NewLoggedClient(t *testing.T, cli *http.Client) *http.Client {
+func NewLoggedClient(t *testing.T, hsName string, cli *http.Client) *http.Client {
 	t.Helper()
 	if cli == nil {
 		cli = &http.Client{
@@ -335,22 +338,23 @@ func NewLoggedClient(t *testing.T, cli *http.Client) *http.Client {
 	if transport == nil {
 		transport = http.DefaultTransport
 	}
-	cli.Transport = &loggedRoundTripper{t, transport}
+	cli.Transport = &loggedRoundTripper{t, hsName, transport}
 	return cli
 }
 
 type loggedRoundTripper struct {
-	t    *testing.T
-	wrap http.RoundTripper
+	t      *testing.T
+	hsName string
+	wrap   http.RoundTripper
 }
 
 func (t *loggedRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
 	start := time.Now()
 	res, err := t.wrap.RoundTrip(req)
 	if err != nil {
-		t.t.Logf("%s %s => error: %s (%s)", req.Method, req.URL.Path, err, time.Since(start))
+		t.t.Logf("%s %s%s => error: %s (%s)", req.Method, t.hsName, req.URL.Path, err, time.Since(start))
 	} else {
-		t.t.Logf("%s %s => %s (%s)", req.Method, req.URL.Path, res.Status, time.Since(start))
+		t.t.Logf("%s %s%s => %s (%s)", req.Method, t.hsName, req.URL.Path, res.Status, time.Since(start))
 	}
 	return res, err
 }
@@ -368,6 +372,29 @@ func GetJSONFieldStr(t *testing.T, body []byte, wantKey string) string {
 	return res.Str
 }
 
+func GetJSONFieldStringArray(t *testing.T, body []byte, wantKey string) []string {
+	t.Helper()
+
+	res := gjson.GetBytes(body, wantKey)
+
+	if !res.Exists() {
+		t.Fatalf("JSONFieldStr: key '%s' missing from %s", wantKey, string(body))
+	}
+
+	arrLength := len(res.Array())
+	arr := make([]string, arrLength)
+	i := 0
+	res.ForEach(func(key, value gjson.Result) bool {
+		arr[i] = value.Str
+
+		// Keep iterating
+		i++
+		return true
+	})
+
+	return arr
+}
+
 // ParseJSON parses a JSON-encoded HTTP Response body into a byte slice
 func ParseJSON(t *testing.T, res *http.Response) []byte {
 	t.Helper()

internal/docker/builder.go

@@ -476,7 +476,9 @@ func generateASRegistrationYaml(as b.ApplicationService) string {
 		fmt.Sprintf("sender_localpart: %s\n", as.SenderLocalpart) +
 		fmt.Sprintf("rate_limited: %v\n", as.RateLimited) +
 		"namespaces:\n" +
-		"  users: []\n" +
+		"  users:\n" +
+		"    - exclusive: false\n" +
+		"      regex: .*\n" +
 		"  rooms: []\n" +
 		"  aliases: []\n"
 }

internal/docker/deployment.go

@@ -53,7 +53,7 @@ func (d *Deployment) Client(t *testing.T, hsName, userID string) *client.CSAPI {
 		UserID:           userID,
 		AccessToken:      token,
 		BaseURL:          dep.BaseURL,
-		Client:           client.NewLoggedClient(t, nil),
+		Client:           client.NewLoggedClient(t, hsName, nil),
 		SyncUntilTimeout: 5 * time.Second,
 		Debug:            d.Deployer.debugLogging,
 	}
@@ -69,7 +69,7 @@ func (d *Deployment) RegisterUser(t *testing.T, hsName, localpart, password stri
 	}
 	client := &client.CSAPI{
 		BaseURL:          dep.BaseURL,
-		Client:           client.NewLoggedClient(t, nil),
+		Client:           client.NewLoggedClient(t, hsName, nil),
 		SyncUntilTimeout: 5 * time.Second,
 		Debug:            d.Deployer.debugLogging,
 	}

internal/match/json.go

@@ -55,25 +55,7 @@ func JSONKeyTypeEqual(wantKey string, wantType gjson.Type) JSON {
 	}
 }
 
-// JSONCheckOff returns a matcher which will loop over `wantKey` and ensure that the items
-// (which can be array elements or object keys)
-// are present exactly once in any order in `wantItems`. If there are unexpected items or items
-// appear more than once then the match fails. This matcher can be used to check off items in
-// an array/object. The `mapper` function should map the item to an interface which will be
-// comparable via `reflect.DeepEqual` with items in `wantItems`. The optional `fn` callback
-// allows more checks to be performed other than checking off the item from the list. It is
-// called with 2 args: the result of the `mapper` function and the element itself (or value if
-// it's an object).
-//
-// Usage: (ensures `events` has these events in any order, with the right event type)
-//    JSONCheckOff("events", []interface{}{"$foo:bar", "$baz:quuz"}, func(r gjson.Result) interface{} {
-//        return r.Get("event_id").Str
-//    }, func(eventID interface{}, eventBody gjson.Result) error {
-//        if eventBody.Get("type").Str != "m.room.message" {
-//	          return fmt.Errorf("expected event to be 'm.room.message'")
-//        }
-//    })
-func JSONCheckOff(wantKey string, wantItems []interface{}, mapper func(gjson.Result) interface{}, fn func(interface{}, gjson.Result) error) JSON {
+func jsonCheckOffInternal(wantKey string, wantItems []interface{}, allowUnwantedItems bool, mapper func(gjson.Result) interface{}, fn func(interface{}, gjson.Result) error) JSON {
 	return func(body []byte) error {
 		res := gjson.GetBytes(body, wantKey)
 		if !res.Exists() {
@@ -103,12 +85,15 @@ func JSONCheckOff(wantKey string, wantItems []interface{}, mapper func(gjson.Res
 					break
 				}
 			}
-			if want == -1 {
+			if !allowUnwantedItems && want == -1 {
 				err = fmt.Errorf("JSONCheckOff: unexpected item %s", item)
 				return false
 			}
-			// delete the wanted item
-			wantItems = append(wantItems[:want], wantItems[want+1:]...)
+
+			if want != -1 {
+				// delete the wanted item
+				wantItems = append(wantItems[:want], wantItems[want+1:]...)
+			}
 
 			// do further checks
 			if fn != nil {
@@ -130,6 +115,50 @@ func JSONCheckOff(wantKey string, wantItems []interface{}, mapper func(gjson.Res
 	}
 }
 
+// JSONCheckOffAllowUnwanted returns a matcher which will loop over `wantKey` and ensure that the items
+// (which can be array elements or object keys)
+// are present exactly once in any order in `wantItems`. Allows unexpected items or items
+// appear that more than once. This matcher can be used to check off items in
+// an array/object. The `mapper` function should map the item to an interface which will be
+// comparable via `reflect.DeepEqual` with items in `wantItems`. The optional `fn` callback
+// allows more checks to be performed other than checking off the item from the list. It is
+// called with 2 args: the result of the `mapper` function and the element itself (or value if
+// it's an object).
+//
+// Usage: (ensures `events` has these events in any order, with the right event type)
+//    JSONCheckOffAllowUnwanted("events", []interface{}{"$foo:bar", "$baz:quuz"}, func(r gjson.Result) interface{} {
+//        return r.Get("event_id").Str
+//    }, func(eventID interface{}, eventBody gjson.Result) error {
+//        if eventBody.Get("type").Str != "m.room.message" {
+//	          return fmt.Errorf("expected event to be 'm.room.message'")
+//        }
+//    })
+func JSONCheckOffAllowUnwanted(wantKey string, wantItems []interface{}, mapper func(gjson.Result) interface{}, fn func(interface{}, gjson.Result) error) JSON {
+	return jsonCheckOffInternal(wantKey, wantItems, true, mapper, fn)
+}
+
+// JSONCheckOff returns a matcher which will loop over `wantKey` and ensure that the items
+// (which can be array elements or object keys)
+// are present exactly once in any order in `wantItems`. If there are unexpected items or items
+// appear more than once then the match fails. This matcher can be used to check off items in
+// an array/object. The `mapper` function should map the item to an interface which will be
+// comparable via `reflect.DeepEqual` with items in `wantItems`. The optional `fn` callback
+// allows more checks to be performed other than checking off the item from the list. It is
+// called with 2 args: the result of the `mapper` function and the element itself (or value if
+// it's an object).
+//
+// Usage: (ensures `events` has these events in any order, with the right event type)
+//    JSONCheckOff("events", []interface{}{"$foo:bar", "$baz:quuz"}, func(r gjson.Result) interface{} {
+//        return r.Get("event_id").Str
+//    }, func(eventID interface{}, eventBody gjson.Result) error {
+//        if eventBody.Get("type").Str != "m.room.message" {
+//	          return fmt.Errorf("expected event to be 'm.room.message'")
+//        }
+//    })
+func JSONCheckOff(wantKey string, wantItems []interface{}, mapper func(gjson.Result) interface{}, fn func(interface{}, gjson.Result) error) JSON {
+	return jsonCheckOffInternal(wantKey, wantItems, false, mapper, fn)
+}
+
 // JSONArrayEach returns a matcher which will check that `wantKey` is an array then loops over each
 // item calling `fn`. If `fn` returns an error, iterating stops and an error is returned.
 func JSONArrayEach(wantKey string, fn func(gjson.Result) error) JSON {

tests/msc2403_test.go

@@ -143,6 +143,7 @@ func knockingBetweenTwoUsersTest(t *testing.T, roomID string, inRoomUser, knocki
 			knockingUser.SyncUntil(
 				t,
 				since,
+				"",
 				"rooms.leave."+client.GjsonEscape(roomID)+".timeline.events",
 				func(ev gjson.Result) bool {
 					if ev.Get("type").Str != "m.room.member" || ev.Get("sender").Str != knockingUser.UserID {
@@ -274,6 +275,7 @@ func knockOnRoomSynced(t *testing.T, c *client.CSAPI, roomID, reason string, ser
 	c.SyncUntil(
 		t,
 		"",
+		"",
 		"rooms.knock."+client.GjsonEscape(roomID)+".knock_state.events",
 		func(ev gjson.Result) bool {
 			// We don't currently define any required state event types to be sent.