Configuration tweaks (#2567)

This makes the following changes:

* The various `Defaults` functions are now responsible for setting sane defaults if `generate` is specified, rather than hiding them in `generate-config`
* Some configuration options have been marked as `omitempty` so that they don't appear in generated configs unnecessarily (monolith-specific vs. polylith-specific options)
* A new option `-polylith` has been added to `generate-config` to create a config that makes sense for polylith deployments (i.e. including the internal/external API listeners and per-component database sections)
* A new option `-normalise` has been added to `generate-config` to take an existing file and add any missing options and/or defaults

build/gobind-pinecone/monolith.go

@@ -243,7 +243,10 @@ func (m *DendriteMonolith) Start() {
 
 	prefix := hex.EncodeToString(pk)
 	cfg := &config.Dendrite{}
-	cfg.Defaults(true)
+	cfg.Defaults(config.DefaultOpts{
+		Generate:   true,
+		Monolithic: true,
+	})
 	cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
 	cfg.Global.PrivateKey = sk
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)

build/gobind-yggdrasil/monolith.go

@@ -82,7 +82,10 @@ func (m *DendriteMonolith) Start() {
 	m.YggdrasilNode = ygg
 
 	cfg := &config.Dendrite{}
-	cfg.Defaults(true)
+	cfg.Defaults(config.DefaultOpts{
+		Generate:   true,
+		Monolithic: true,
+	})
 	cfg.Global.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName())
 	cfg.Global.PrivateKey = ygg.PrivateKey()
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)

build/scripts/ComplementPostgres.Dockerfile

@@ -46,9 +46,8 @@ EXPOSE 8008 8448
 # At runtime, generate TLS cert based on the CA now mounted at /ca
 # At runtime, replace the SERVER_NAME with what we are told
 CMD /build/run_postgres.sh && ./generate-keys --keysize 1024 --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \
-    ./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
-    # Replace the connection string with a single postgres DB, using user/db = 'postgres' and no password, bump max_conns
-    sed -i "s%connection_string:.*$%connection_string: postgresql://postgres@localhost/postgres?sslmode=disable%g" dendrite.yaml && \
-    sed -i 's/max_open_conns:.*$/max_open_conns: 100/g' dendrite.yaml && \
+    ./generate-config -server $SERVER_NAME --ci --db postgresql://postgres@localhost/postgres?sslmode=disable > dendrite.yaml && \
+    # Bump max_open_conns up here in the global database config
+    sed -i 's/max_open_conns:.*$/max_open_conns: 1990/g' dendrite.yaml && \
     cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
     exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}

clientapi/routing/register_test.go

@@ -181,7 +181,10 @@ func TestValidationOfApplicationServices(t *testing.T) {
 
 	// Set up a config
 	fakeConfig := &config.Dendrite{}
-	fakeConfig.Defaults(true)
+	fakeConfig.Defaults(config.DefaultOpts{
+		Generate:   true,
+		Monolithic: true,
+	})
 	fakeConfig.Global.ServerName = "localhost"
 	fakeConfig.ClientAPI.Derived.ApplicationServices = []config.ApplicationService{fakeApplicationService}
 

cmd/dendrite-demo-pinecone/main.go

@@ -115,7 +115,10 @@ func main() {
 				panic("failed to load PEM key: " + err.Error())
 			}
 		}
-		cfg.Defaults(true)
+		cfg.Defaults(config.DefaultOpts{
+			Generate:   true,
+			Monolithic: true,
+		})
 		cfg.Global.PrivateKey = sk
 		cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName))
 		cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))

cmd/dendrite-demo-yggdrasil/main.go

@@ -78,7 +78,10 @@ func main() {
 	if configFlagSet {
 		cfg = setup.ParseFlags(true)
 	} else {
-		cfg.Defaults(true)
+		cfg.Defaults(config.DefaultOpts{
+			Generate:   true,
+			Monolithic: true,
+		})
 		cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName))
 		cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
 		cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))

cmd/generate-config/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"flag"
 	"fmt"
+	"path/filepath"
 
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -11,90 +12,81 @@ import (
 )
 
 func main() {
-	defaultsForCI := flag.Bool("ci", false, "sane defaults for CI testing")
+	defaultsForCI := flag.Bool("ci", false, "Populate the configuration with sane defaults for use in CI")
 	serverName := flag.String("server", "", "The domain name of the server if not 'localhost'")
-	dbURI := flag.String("db", "", "The DB URI to use for all components if not SQLite files")
+	dbURI := flag.String("db", "", "The DB URI to use for all components (PostgreSQL only)")
+	dirPath := flag.String("dir", "./", "The folder to use for paths (like SQLite databases, media storage)")
+	normalise := flag.String("normalise", "", "Normalise an existing configuration file by adding new/missing options and defaults")
+	polylith := flag.Bool("polylith", false, "Generate a config that makes sense for polylith deployments")
 	flag.Parse()
 
-	cfg := &config.Dendrite{
-		Version: config.Version,
-	}
-	cfg.Defaults(true)
-	if *serverName != "" {
-		cfg.Global.ServerName = gomatrixserverlib.ServerName(*serverName)
-	}
-	if *dbURI != "" {
-		cfg.FederationAPI.Database.ConnectionString = config.DataSource(*dbURI)
-		cfg.KeyServer.Database.ConnectionString = config.DataSource(*dbURI)
-		cfg.MSCs.Database.ConnectionString = config.DataSource(*dbURI)
-		cfg.MediaAPI.Database.ConnectionString = config.DataSource(*dbURI)
-		cfg.RoomServer.Database.ConnectionString = config.DataSource(*dbURI)
-		cfg.SyncAPI.Database.ConnectionString = config.DataSource(*dbURI)
-		cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(*dbURI)
-	}
-	cfg.Global.TrustedIDServers = []string{
-		"matrix.org",
-		"vector.im",
-	}
-	cfg.Logging = []config.LogrusHook{
-		{
-			Type:  "file",
-			Level: "info",
-			Params: map[string]interface{}{
-				"path": "/var/log/dendrite",
-			},
-		},
-	}
-	cfg.FederationAPI.KeyPerspectives = config.KeyPerspectives{
-		{
-			ServerName: "matrix.org",
-			Keys: []config.KeyPerspectiveTrustKey{
-				{
-					KeyID:     "ed25519:auto",
-					PublicKey: "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw",
-				},
-				{
-					KeyID:     "ed25519:a_RXGa",
-					PublicKey: "l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ",
+	var cfg *config.Dendrite
+	if *normalise == "" {
+		cfg = &config.Dendrite{
+			Version: config.Version,
+		}
+		cfg.Defaults(config.DefaultOpts{
+			Generate:   true,
+			Monolithic: !*polylith,
+		})
+		if *serverName != "" {
+			cfg.Global.ServerName = gomatrixserverlib.ServerName(*serverName)
+		}
+		uri := config.DataSource(*dbURI)
+		if *polylith || uri.IsSQLite() || uri == "" {
+			for name, db := range map[string]*config.DatabaseOptions{
+				"federationapi": &cfg.FederationAPI.Database,
+				"keyserver":     &cfg.KeyServer.Database,
+				"mscs":          &cfg.MSCs.Database,
+				"mediaapi":      &cfg.MediaAPI.Database,
+				"roomserver":    &cfg.RoomServer.Database,
+				"syncapi":       &cfg.SyncAPI.Database,
+				"userapi":       &cfg.UserAPI.AccountDatabase,
+			} {
+				if uri == "" {
+					path := filepath.Join(*dirPath, fmt.Sprintf("dendrite_%s.db", name))
+					db.ConnectionString = config.DataSource(fmt.Sprintf("file:%s", path))
+				} else {
+					db.ConnectionString = uri
+				}
+			}
+		} else {
+			cfg.Global.DatabaseOptions.ConnectionString = uri
+		}
+		cfg.Logging = []config.LogrusHook{
+			{
+				Type:  "file",
+				Level: "info",
+				Params: map[string]interface{}{
+					"path": filepath.Join(*dirPath, "log"),
 				},
 			},
-		},
-	}
-	cfg.MediaAPI.ThumbnailSizes = []config.ThumbnailSize{
-		{
-			Width:        32,
-			Height:       32,
-			ResizeMethod: "crop",
-		},
-		{
-			Width:        96,
-			Height:       96,
-			ResizeMethod: "crop",
-		},
-		{
-			Width:        640,
-			Height:       480,
-			ResizeMethod: "scale",
-		},
-	}
-
-	if *defaultsForCI {
-		cfg.AppServiceAPI.DisableTLSValidation = true
-		cfg.ClientAPI.RateLimiting.Enabled = false
-		cfg.FederationAPI.DisableTLSValidation = false
-		// don't hit matrix.org when running tests!!!
-		cfg.FederationAPI.KeyPerspectives = config.KeyPerspectives{}
-		cfg.MSCs.MSCs = []string{"msc2836", "msc2946", "msc2444", "msc2753"}
-		cfg.Logging[0].Level = "trace"
-		cfg.Logging[0].Type = "std"
-		cfg.UserAPI.BCryptCost = bcrypt.MinCost
-		cfg.Global.JetStream.InMemory = true
-		cfg.ClientAPI.RegistrationDisabled = false
-		cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
-		cfg.ClientAPI.RegistrationSharedSecret = "complement"
-		cfg.Global.Presence = config.PresenceOptions{
-			EnableInbound:  true,
-			EnableOutbound: true,
+		}
+		if *defaultsForCI {
+			cfg.AppServiceAPI.DisableTLSValidation = true
+			cfg.ClientAPI.RateLimiting.Enabled = false
+			cfg.FederationAPI.DisableTLSValidation = false
+			// don't hit matrix.org when running tests!!!
+			cfg.FederationAPI.KeyPerspectives = config.KeyPerspectives{}
+			cfg.MediaAPI.BasePath = config.Path(filepath.Join(*dirPath, "media"))
+			cfg.MSCs.MSCs = []string{"msc2836", "msc2946", "msc2444", "msc2753"}
+			cfg.Logging[0].Level = "trace"
+			cfg.Logging[0].Type = "std"
+			cfg.UserAPI.BCryptCost = bcrypt.MinCost
+			cfg.Global.JetStream.InMemory = true
+			cfg.Global.JetStream.StoragePath = config.Path(*dirPath)
+			cfg.ClientAPI.RegistrationDisabled = false
+			cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
+			cfg.ClientAPI.RegistrationSharedSecret = "complement"
+			cfg.Global.Presence = config.PresenceOptions{
+				EnableInbound:  true,
+				EnableOutbound: true,
+			}
+		}
+	} else {
+		var err error
+		if cfg, err = config.Load(*normalise, !*polylith); err != nil {
+			panic(err)
 		}
 	}
 

federationapi/federationapi_keys_test.go

@@ -75,7 +75,10 @@ func TestMain(m *testing.M) {
 			// Draw up just enough Dendrite config for the server key
 			// API to work.
 			cfg := &config.Dendrite{}
-			cfg.Defaults(true)
+			cfg.Defaults(config.DefaultOpts{
+				Generate:   true,
+				Monolithic: true,
+			})
 			cfg.Global.ServerName = gomatrixserverlib.ServerName(s.name)
 			cfg.Global.PrivateKey = testPriv
 			cfg.Global.JetStream.InMemory = true

federationapi/federationapi_test.go

@@ -263,7 +263,10 @@ func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) {
 func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
 	_, privKey, _ := ed25519.GenerateKey(nil)
 	cfg := &config.Dendrite{}
-	cfg.Defaults(true)
+	cfg.Defaults(config.DefaultOpts{
+		Generate:   true,
+		Monolithic: true,
+	})
 	cfg.Global.KeyID = gomatrixserverlib.KeyID("ed25519:auto")
 	cfg.Global.ServerName = gomatrixserverlib.ServerName("localhost")
 	cfg.Global.PrivateKey = privKey

setup/config/config.go

@@ -211,7 +211,10 @@ func loadConfig(
 	monolithic bool,
 ) (*Dendrite, error) {
 	var c Dendrite
-	c.Defaults(false)
+	c.Defaults(DefaultOpts{
+		Generate:   false,
+		Monolithic: monolithic,
+	})
 	c.IsMonolith = monolithic
 
 	var err error
@@ -295,21 +298,25 @@ func (config *Dendrite) Derive() error {
 	return nil
 }
 
+type DefaultOpts struct {
+	Generate   bool
+	Monolithic bool
+}
+
 // SetDefaults sets default config values if they are not explicitly set.
-func (c *Dendrite) Defaults(generate bool) {
+func (c *Dendrite) Defaults(opts DefaultOpts) {
 	c.Version = Version
 
-	c.Global.Defaults(generate)
-	c.ClientAPI.Defaults(generate)
-	c.FederationAPI.Defaults(generate)
-	c.KeyServer.Defaults(generate)
-	c.MediaAPI.Defaults(generate)
-	c.RoomServer.Defaults(generate)
-	c.SyncAPI.Defaults(generate)
-	c.UserAPI.Defaults(generate)
-	c.AppServiceAPI.Defaults(generate)
-	c.MSCs.Defaults(generate)
-
+	c.Global.Defaults(opts)
+	c.ClientAPI.Defaults(opts)
+	c.FederationAPI.Defaults(opts)
+	c.KeyServer.Defaults(opts)
+	c.MediaAPI.Defaults(opts)
+	c.RoomServer.Defaults(opts)
+	c.SyncAPI.Defaults(opts)
+	c.UserAPI.Defaults(opts)
+	c.AppServiceAPI.Defaults(opts)
+	c.MSCs.Defaults(opts)
 	c.Wiring()
 }
 

setup/config/config_appservice.go

@@ -29,7 +29,7 @@ type AppServiceAPI struct {
 	Matrix  *Global  `yaml:"-"`
 	Derived *Derived `yaml:"-"` // TODO: Nuke Derived from orbit
 
-	InternalAPI InternalAPIOptions `yaml:"internal_api"`
+	InternalAPI InternalAPIOptions `yaml:"internal_api,omitempty"`
 
 	// DisableTLSValidation disables the validation of X.509 TLS certs
 	// on appservice endpoints. This is not recommended in production!
@@ -38,9 +38,11 @@ type AppServiceAPI struct {
 	ConfigFiles []string `yaml:"config_files"`
 }
 
-func (c *AppServiceAPI) Defaults(generate bool) {
-	c.InternalAPI.Listen = "http://localhost:7777"
-	c.InternalAPI.Connect = "http://localhost:7777"
+func (c *AppServiceAPI) Defaults(opts DefaultOpts) {
+	if !opts.Monolithic {
+		c.InternalAPI.Listen = "http://localhost:7777"
+		c.InternalAPI.Connect = "http://localhost:7777"
+	}
 }
 
 func (c *AppServiceAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {

setup/config/config_clientapi.go

@@ -9,8 +9,8 @@ type ClientAPI struct {
 	Matrix  *Global  `yaml:"-"`
 	Derived *Derived `yaml:"-"` // TODO: Nuke Derived from orbit
 
-	InternalAPI InternalAPIOptions `yaml:"internal_api"`
-	ExternalAPI ExternalAPIOptions `yaml:"external_api"`
+	InternalAPI InternalAPIOptions `yaml:"internal_api,omitempty"`
+	ExternalAPI ExternalAPIOptions `yaml:"external_api,omitempty"`
 
 	// If set disables new users from registering (except via shared
 	// secrets)
@@ -48,13 +48,15 @@ type ClientAPI struct {
 	// Rate-limiting options
 	RateLimiting RateLimiting `yaml:"rate_limiting"`
 
-	MSCs *MSCs `yaml:"mscs"`
+	MSCs *MSCs `yaml:"-"`
 }
 
-func (c *ClientAPI) Defaults(generate bool) {
-	c.InternalAPI.Listen = "http://localhost:7771"
-	c.InternalAPI.Connect = "http://localhost:7771"
-	c.ExternalAPI.Listen = "http://[::]:8071"
+func (c *ClientAPI) Defaults(opts DefaultOpts) {
+	if !opts.Monolithic {
+		c.InternalAPI.Listen = "http://localhost:7771"
+		c.InternalAPI.Connect = "http://localhost:7771"
+		c.ExternalAPI.Listen = "http://[::]:8071"
+	}
 	c.RegistrationSharedSecret = ""
 	c.RecaptchaPublicKey = ""
 	c.RecaptchaPrivateKey = ""