Propagate client config to queries running in the client. (#4047)

This propagates things like prevent_execve into the query scope to
ensure it is inspected at all levels.

Also implemented a plugin deny list as a more robust way to prevent some
plugins and functions from running on the endpoint. This extends the
idea of prevent_execve to other potentially dangerous plugins (e.g. rm).
It is implemented as a deny list in addition to the previous allow_list
to make it easier to use.

actions/vql.go

@@ -148,6 +148,7 @@ func (self VQLClientAction) StartQuery(
 
 	builder := services.ScopeBuilder{
 		Config: &config_proto.Config{
+			Client:     config_obj.Client,
 			Remappings: config_obj.Remappings,
 		},
 		Ctx: ctx,

actions/vql_test.go

@@ -11,15 +11,26 @@ import (
 	artifacts_proto "www.velocidex.com/golang/velociraptor/artifacts/proto"
 	crypto_proto "www.velocidex.com/golang/velociraptor/crypto/proto"
 	"www.velocidex.com/golang/velociraptor/file_store/test_utils"
+	"www.velocidex.com/golang/velociraptor/logging"
 	"www.velocidex.com/golang/velociraptor/responder"
 	"www.velocidex.com/golang/velociraptor/vtesting"
 	"www.velocidex.com/golang/velociraptor/vtesting/assert"
+
+	// For execve and query
+	_ "www.velocidex.com/golang/velociraptor/vql/common"
+	_ "www.velocidex.com/golang/velociraptor/vql/tools"
 )
 
 type ClientVQLTestSuite struct {
 	test_utils.TestSuite
 }
 
+func (self *ClientVQLTestSuite) SetupTest() {
+	self.ConfigObj = self.LoadConfig()
+	self.ConfigObj.Client.PreventExecve = true
+	self.TestSuite.SetupTest()
+}
+
 func (self *ClientVQLTestSuite) TestCPUThrottler() {
 	request := &actions_proto.VQLCollectorArgs{
 		Query: []*actions_proto.VQLRequest{
@@ -129,6 +140,49 @@ func (self *ClientVQLTestSuite) TestMaxRows() {
 	})
 }
 
+func (self *ClientVQLTestSuite) TestExecve() {
+	resp := responder.TestResponderWithFlowId(self.ConfigObj, "TestMaxRows")
+
+	logging.ClearMemoryLogs()
+
+	actions.VQLClientAction{}.StartQuery(self.ConfigObj, self.Sm.Ctx, resp,
+		&actions_proto.VQLCollectorArgs{
+			MaxRow: 10,
+			Query: []*actions_proto.VQLRequest{
+				{
+					Name: "Query",
+					VQL:  "SELECT * FROM execve(argv='ls')",
+					//					VQL:  "SELECT * FROM query(query={ SELECT * FROM execve(argv='ls') })",
+				},
+			},
+		})
+
+	vtesting.WaitUntil(time.Second, self.T(), func() bool {
+		return vtesting.MemoryLogsContainRegex(
+			"execve: Not allowed to execve by configuration.")
+	})
+
+	logging.ClearMemoryLogs()
+
+	// Make sure the query() plugin propagates the execve flag
+	actions.VQLClientAction{}.StartQuery(self.ConfigObj, self.Sm.Ctx, resp,
+		&actions_proto.VQLCollectorArgs{
+			MaxRow: 10,
+			Query: []*actions_proto.VQLRequest{
+				{
+					Name: "Query",
+					VQL:  "SELECT * FROM query(query={ SELECT * FROM execve(argv='ls') })",
+				},
+			},
+		})
+
+	vtesting.WaitUntil(time.Second, self.T(), func() bool {
+		return vtesting.MemoryLogsContainRegex(
+			"execve: Not allowed to execve by configuration.")
+	})
+
+}
+
 func (self *ClientVQLTestSuite) TestMaxWait() {
 	assert.True(self.T(), test_utils.Retry(self.T(), 5, time.Millisecond*500,
 		func(r *test_utils.R) {

config/proto/config.proto

@@ -1160,6 +1160,11 @@ message Defaults {
     repeated string allowed_functions = 11;
     repeated string allowed_accessors = 12;
 
+    // Alternatively, it might be easier to deny specific plugins and
+    // functions.
+    repeated string denied_plugins = 54;
+    repeated string denied_functions = 55;
+
     // How long to cache ACL policies (default 60 sec)
     uint64 acl_lru_timeout_sec = 13;
 

docs/references/sample_config/main.go

@@ -21,7 +21,8 @@ var (
 		"Yaml filename to read (server.config.yaml)").Required().String()
 	tagRegEx = regexp.MustCompile("json:\"([^,]+)")
 
-	// Usually deprecated fields we dont want people to use.
+	// Usually deprecated fields we dont want people to use so we dont
+	// document them.
 	hidden_fields = []string{
 		"sub_authenticators",
 		"autocert_domain",
@@ -74,6 +75,7 @@ var (
 		"Client.disable_compression",
 		"Client.default_server_flow_stats_update",
 		"defaults.max_vfs_directory_size",
+		"Datastore.remote_datastore_rpc_deadline",
 
 		// Fields that are already handled but their default value is
 		// false or 0.
@@ -112,6 +114,8 @@ var (
 		"debug_mode",
 		"Client.proxy_config.ignore_environment",
 		"Frontend.proxy_config.ignore_environment",
+		"defaults.disable_active_inflight_checks",
+		"defaults.write_internal_events",
 	}
 )
 

docs/references/server.config.yaml

@@ -564,6 +564,12 @@ GUI:
     # part of the multi authenticator so having a name here helps keep
     # them recognizable.
     oidc_name: "Company Name"
+
+    # Additional URL parameters that should be added to the OIDC
+    # redirect URL.
+    oidc_auth_url_params:
+      Key: Value
+
     avatar: http://www.example.com/icon.png
 
     # These are required for the oauth flow - get from the OIDC provider.
@@ -1270,6 +1276,14 @@ defaults:
   allowed_accessors:
     - auto
 
+  # Alternatively, it might be easier to deny specific plugins and
+  # functions.
+  denied_plugins:
+    - execve
+
+  denied_functions:
+    - rm
+
   # How long to cache ACL policies (default 60 sec)
   acl_lru_timeout_sec: 60
 
@@ -1366,6 +1380,17 @@ defaults:
   indexed_client_metadata:
     - department
 
+  # If this is set we do not actively check the status of inflight
+  # collections. This is a new feature to 0.73 and may need to be
+  # disabled in some large deployments due to additional overheads.
+  disable_active_inflight_checks: false
+
+  # Normally internal event artifacts are not written to disk but
+  # passed internally. For debugging it is useful to have a written
+  # record though. Enabling this will also write them to
+  # disk. Probably only useful for debugging.
+  write_internal_events: false
+
 # The Velociraptor server may be placed into "lockdown" mode. While in
 # lockdown mode certain permissions are denied - even for
 # administrators. This additional protection mode helps to mitigate

services/launcher/launcher_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/Velocidex/ordereddict"
 	"github.com/go-errors/errors"
 	"www.velocidex.com/golang/velociraptor/constants"
+	"www.velocidex.com/golang/velociraptor/datastore"
 	"www.velocidex.com/golang/velociraptor/file_store"
 	"www.velocidex.com/golang/velociraptor/result_sets"
 	"www.velocidex.com/golang/velociraptor/utils"
@@ -1386,6 +1387,8 @@ func (self *LauncherTestSuite) _TestDelete(t *assert.R) {
 	idx_flow_id, _ := idx[0].GetString("FlowId")
 	assert.Equal(t, flow_id, idx_flow_id)
 
+	datastore.FlushDatastore(self.ConfigObj)
+
 	// However GetFlows omits the deleted flow immediately because it
 	// can not find it (The actual flow object is removed but the
 	// index is out of step).
@@ -1420,6 +1423,8 @@ func (self *LauncherTestSuite) _TestDelete(t *assert.R) {
 		})
 	assert.NoError(t, err)
 
+	datastore.FlushDatastore(self.ConfigObj)
+
 	// This time the index is reset immediately.
 	idx = self.getIndex("server")
 	assert.Equal(t, len(idx), 0)

startup/allowlists.go

@@ -0,0 +1,46 @@
+package startup
+
+import (
+	"www.velocidex.com/golang/velociraptor/accessors"
+	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
+	"www.velocidex.com/golang/velociraptor/logging"
+	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
+)
+
+// Potentially restrict server functionality.
+func MaybeEnforceAllowLists(config_obj *config_proto.Config) error {
+	if config_obj.Defaults == nil {
+		return nil
+	}
+
+	if len(config_obj.Defaults.AllowedPlugins) > 0 {
+		logger := logging.GetLogger(config_obj, &logging.FrontendComponent)
+		logger.Info("Restricting VQL plugins to set %v and functions to set %v\n",
+			config_obj.Defaults.AllowedPlugins, config_obj.Defaults.AllowedFunctions)
+	}
+
+	if len(config_obj.Defaults.DeniedPlugins) > 0 {
+		logger := logging.GetLogger(config_obj, &logging.FrontendComponent)
+		logger.Info("Removing VQL plugins to set %v and functions to set %v\n",
+			config_obj.Defaults.DeniedPlugins, config_obj.Defaults.DeniedPlugins)
+	}
+
+	err := vql_subsystem.EnforceVQLAllowList(
+		config_obj.Defaults.AllowedPlugins,
+		config_obj.Defaults.AllowedFunctions,
+		config_obj.Defaults.DeniedPlugins,
+		config_obj.Defaults.DeniedFunctions)
+	if err != nil {
+		return err
+	}
+
+	if len(config_obj.Defaults.AllowedAccessors) > 0 {
+		err = accessors.EnforceAccessorAllowList(
+			config_obj.Defaults.AllowedAccessors)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

startup/client.go

@@ -19,9 +19,6 @@ func StartClientServices(
 	on_error func(ctx context.Context,
 		config_obj *config_proto.Config)) (*services.Service, error) {
 
-	scope := vql_subsystem.MakeScope()
-	vql_subsystem.InstallUnimplemented(scope)
-
 	// Create a suitable service plan.
 	if config_obj.Services == nil {
 		config_obj.Services = services.ClientServicesSpec()
@@ -31,8 +28,16 @@ func StartClientServices(
 	// before we begin the comms.
 	sm := services.NewServiceManager(ctx, config_obj)
 
+	err := MaybeEnforceAllowLists(config_obj)
+	if err != nil {
+		return sm, err
+	}
+
+	scope := vql_subsystem.MakeScope()
+	vql_subsystem.InstallUnimplemented(scope)
+
 	// Start encrypted logs service if possible
-	err := encrypted_logs.StartEncryptedLog(sm.Ctx, sm.Wg, sm.Config)
+	err = encrypted_logs.StartEncryptedLog(sm.Ctx, sm.Wg, sm.Config)
 	if err != nil {
 		return sm, err
 	}

startup/frontend.go

@@ -3,10 +3,8 @@ package startup
 import (
 	"context"
 
-	"www.velocidex.com/golang/velociraptor/accessors"
 	"www.velocidex.com/golang/velociraptor/api"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
-	"www.velocidex.com/golang/velociraptor/logging"
 	"www.velocidex.com/golang/velociraptor/services"
 	"www.velocidex.com/golang/velociraptor/services/orgs"
 	"www.velocidex.com/golang/velociraptor/utils/tempfile"
@@ -18,14 +16,21 @@ func StartFrontendServices(
 	ctx context.Context,
 	config_obj *config_proto.Config) (*services.Service, error) {
 
-	scope := vql_subsystem.MakeScope()
-	vql_subsystem.InstallUnimplemented(scope)
-
 	// Set the temp directory if needed
 	tempfile.SetTempfile(config_obj)
 
 	sm := services.NewServiceManager(ctx, config_obj)
-	_, err := orgs.NewOrgManager(sm.Ctx, sm.Wg, config_obj)
+
+	// Potentially restrict server functionality.
+	err := MaybeEnforceAllowLists(config_obj)
+	if err != nil {
+		return sm, err
+	}
+
+	scope := vql_subsystem.MakeScope()
+	vql_subsystem.InstallUnimplemented(scope)
+
+	_, err = orgs.NewOrgManager(sm.Ctx, sm.Wg, config_obj)
 	if err != nil {
 		return sm, err
 	}
@@ -44,29 +49,5 @@ func StartFrontendServices(
 		}
 	}
 
-	// Potentially restrict server functionality.
-	if config_obj.Defaults != nil {
-		if len(config_obj.Defaults.AllowedPlugins) > 0 {
-			logger := logging.GetLogger(config_obj, &logging.FrontendComponent)
-			logger.Info("Restricting VQL plugins to set %v and functions to set %v\n",
-				config_obj.Defaults.AllowedPlugins, config_obj.Defaults.AllowedFunctions)
-
-			err = vql_subsystem.EnforceVQLAllowList(
-				config_obj.Defaults.AllowedPlugins,
-				config_obj.Defaults.AllowedFunctions)
-			if err != nil {
-				return sm, err
-			}
-		}
-
-		if len(config_obj.Defaults.AllowedAccessors) > 0 {
-			err = accessors.EnforceAccessorAllowList(
-				config_obj.Defaults.AllowedAccessors)
-			if err != nil {
-				return sm, err
-			}
-		}
-	}
-
 	return sm, server_builder.StartServer(sm.Ctx, sm.Wg)
 }

startup/tool.go

@@ -18,7 +18,13 @@ func StartToolServices(
 	vql_subsystem.InstallUnimplemented(scope)
 
 	sm := services.NewServiceManager(ctx, config_obj)
-	_, err := orgs.NewOrgManager(sm.Ctx, sm.Wg, config_obj)
+
+	err := MaybeEnforceAllowLists(config_obj)
+	if err != nil {
+		return sm, err
+	}
+
+	_, err = orgs.NewOrgManager(sm.Ctx, sm.Wg, config_obj)
 	if err != nil {
 		return sm, err
 	}