From 0468a64f102f888d7513af64970cf6bfd2f415b8 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Wed, 20 Dec 2023 03:24:49 +1000 Subject: [PATCH] Added symlink processing to linux file accessor (#3173) --- accessors/file/accessor_common.go | 19 +++- accessors/file/accessor_linux_test.go | 98 +++++++++++++++++++ accessors/file/accessor_test.go | 1 + .../file/fixtures/TestLinuxSymlinks.golden | 6 ++ glob/glob.go | 15 ++- services/ddclient/ddclient.go | 3 +- services/inventory/dummy.go | 5 +- services/inventory/inventory.go | 3 +- services/notebook/initial.go | 6 +- vql/networking/http_client.go | 4 + vql/tools/logscale/logscale.go | 3 +- vql/tools/webdav_upload.go | 5 + 12 files changed, 155 insertions(+), 13 deletions(-) create mode 100644 accessors/file/accessor_linux_test.go create mode 100644 accessors/file/fixtures/TestLinuxSymlinks.golden diff --git a/accessors/file/accessor_common.go b/accessors/file/accessor_common.go index 6a925982a98..191d0cd8728 100644 --- a/accessors/file/accessor_common.go +++ b/accessors/file/accessor_common.go @@ -336,16 +336,33 @@ func (self OSFileSystemAccessor) ReadDirWithOSPath( } else { // If it is a symlink, we need to check the target of the // symlink and make sure it is a directory. - target, err := os.Readlink(dir) + target, err := filepath.EvalSymlinks(dir) if err == nil { + // The target is interpreted relative to the directory of + // the link. + if !strings.HasPrefix(target, "/") { + target = full_path.Dirname().PathSpec().Path + "/" + target + } lstat, err := os.Lstat(target) + // Target of the link is not there or inaccessible or // points to something that is not a directory - just // ignore it with no errors. if err != nil || !lstat.IsDir() { return nil, nil } + + sys, ok := lstat.Sys().(*syscall.Stat_t) + if ok { + // Keep track of the links we visited. + if self.context.WasLinkVisited( + uint64(sys.Dev), sys.Ino) { + return nil, errors.New("Symlink cycle detected") + } + self.context.LinkVisited(uint64(sys.Dev), sys.Ino) + } } + dir = target } dirfstype := getFSType(dir) diff --git a/accessors/file/accessor_linux_test.go b/accessors/file/accessor_linux_test.go new file mode 100644 index 00000000000..72132a6f2b3 --- /dev/null +++ b/accessors/file/accessor_linux_test.go @@ -0,0 +1,98 @@ +// +build linux + +package file + +import ( + "context" + "io/ioutil" + "log" + "os" + "path/filepath" + "sort" + "testing" + + "github.com/Velocidex/ordereddict" + "github.com/alecthomas/assert" + "github.com/sebdah/goldie" + "github.com/stretchr/testify/suite" + "www.velocidex.com/golang/velociraptor/accessors" + "www.velocidex.com/golang/velociraptor/config" + "www.velocidex.com/golang/velociraptor/glob" + "www.velocidex.com/golang/velociraptor/json" + vql_subsystem "www.velocidex.com/golang/velociraptor/vql" + "www.velocidex.com/golang/velociraptor/vql/acl_managers" +) + +type AccessorLinuxTestSuite struct { + suite.Suite + tmpdir string +} + +func (self *AccessorLinuxTestSuite) TestLinuxSymlinks() { + tmpdir, err := ioutil.TempDir("", "accessor_test") + assert.NoError(self.T(), err) + + // Create two symlinks. + // tmp/second_bin/ -> tmp/zbin + // tmp/zbin -> /bin/ + + err = os.Symlink("/bin", filepath.Join(tmpdir, "zbin")) + assert.NoError(self.T(), err) + + err = os.Symlink(filepath.Join(tmpdir, "zbin"), + filepath.Join(tmpdir, "second_bin")) + assert.NoError(self.T(), err) + + // Create a symlink cycle: + // tmp/subdir is a directory + // tmp/sym1 -> tmp/subdir + // tmp/subdir/sym2 -> tmp + + dirname := filepath.Join(tmpdir, "subdir") + err = os.Mkdir(dirname, 0777) + assert.NoError(self.T(), err) + + err = os.Mkdir(filepath.Join(dirname, "ls"), 0777) + assert.NoError(self.T(), err) + + err = os.Symlink(dirname, filepath.Join(tmpdir, "sym1")) + assert.NoError(self.T(), err) + + err = os.Symlink(tmpdir, filepath.Join(dirname, "sym2")) + assert.NoError(self.T(), err) + + scope := vql_subsystem.MakeScope().AppendVars(ordereddict.NewDict(). + Set(vql_subsystem.ACL_MANAGER_VAR, acl_managers.NullACLManager{})) + scope.SetLogger(log.New(os.Stderr, " ", 0)) + + glob_path, _ := accessors.NewLinuxOSPath("/**/ls") + tmp_path, _ := accessors.NewLinuxOSPath(tmpdir) + + options := glob.GlobOptions{ + DoNotFollowSymlinks: false, + } + globber := glob.NewGlobber().WithOptions(options) + + globber.Add(glob_path) + + accessor, err := accessors.GetAccessor("file", scope) + assert.NoError(self.T(), err) + + config_obj := config.GetDefaultConfig() + hits := []string{} + for hit := range globber.ExpandWithContext( + context.Background(), scope, + config_obj, tmp_path, accessor) { + hits = append(hits, hit.OSPath().TrimComponents( + tmp_path.Components...).String()) + } + + sort.Strings(hits) + + goldie.Assert(self.T(), "TestLinuxSymlinks", json.MustMarshalIndent(hits)) +} + +// Test Linux specific File accessor. +func TestFileLinux(t *testing.T) { + suite.Run(t, &AccessorLinuxTestSuite{}) +} diff --git a/accessors/file/accessor_test.go b/accessors/file/accessor_test.go index d108aae3977..7095ce1e874 100644 --- a/accessors/file/accessor_test.go +++ b/accessors/file/accessor_test.go @@ -182,6 +182,7 @@ func (self *AccessorWindowsTestSuite) TestSymlinks() { } +// Test both the Windows and Linux File accessor. func TestWindowsLinux(t *testing.T) { suite.Run(t, &AccessorWindowsTestSuite{}) } diff --git a/accessors/file/fixtures/TestLinuxSymlinks.golden b/accessors/file/fixtures/TestLinuxSymlinks.golden new file mode 100644 index 00000000000..a5cc295b752 --- /dev/null +++ b/accessors/file/fixtures/TestLinuxSymlinks.golden @@ -0,0 +1,6 @@ +[ + "/second_bin/ls", + "/subdir/ls", + "/subdir/sym2/subdir/ls", + "/sym1/ls" +] \ No newline at end of file diff --git a/glob/glob.go b/glob/glob.go index 82ef2e14444..b0f7277b53b 100644 --- a/glob/glob.go +++ b/glob/glob.go @@ -200,7 +200,9 @@ func (self *Globber) _add_filter(components []_PathFilterer, globs []string) err } func (self *Globber) is_dir_or_link( - f accessors.FileInfo, accessor accessors.FileSystemAccessor, depth int) bool { + scope vfilter.Scope, root *accessors.OSPath, + f accessors.FileInfo, + accessor accessors.FileSystemAccessor, depth int) bool { // Do not follow symlinks to symlinks deeply. if depth > 10 { return false @@ -220,7 +222,11 @@ func (self *Globber) is_dir_or_link( } target, err := f.GetLink() - if err == nil { + if err != nil { + //scope.Log("Globber: %v while processing %v", + // err, root.String()) + + } else { target_info, err := accessor.Lstat(target.String()) if err == nil { // Check if the target is on a different filesystem @@ -235,7 +241,8 @@ func (self *Globber) is_dir_or_link( } } - return self.is_dir_or_link(target_info, accessor, depth+1) + return self.is_dir_or_link( + scope, root, target_info, accessor, depth+1) } // Hmm we failed to lstat the target - assume @@ -311,7 +318,7 @@ func (self *Globber) ExpandWithContext( } // Only recurse into directories. - if self.is_dir_or_link(f, accessor, 0) { + if self.is_dir_or_link(scope, root, f, accessor, 0) { item := []*Globber{next} prev_item, pres := children[basename] if pres { diff --git a/services/ddclient/ddclient.go b/services/ddclient/ddclient.go index 99df6e7cf29..d1b8d141f40 100644 --- a/services/ddclient/ddclient.go +++ b/services/ddclient/ddclient.go @@ -14,6 +14,7 @@ import ( "time" config_proto "www.velocidex.com/golang/velociraptor/config/proto" + "www.velocidex.com/golang/velociraptor/constants" "www.velocidex.com/golang/velociraptor/logging" "www.velocidex.com/golang/velociraptor/vql/networking" ) @@ -200,7 +201,7 @@ func UpdateDDNSRecord(config_obj *config_proto.Config, if err != nil { return err } - req.Header.Set("User-Agent", "") + req.Header.Set("User-Agent", constants.USER_AGENT) req.SetBasicAuth(user, pw) resp, err := client.Do(req) diff --git a/services/inventory/dummy.go b/services/inventory/dummy.go index c84c53e88c7..945408fad8a 100644 --- a/services/inventory/dummy.go +++ b/services/inventory/dummy.go @@ -19,6 +19,7 @@ import ( "google.golang.org/protobuf/proto" artifacts_proto "www.velocidex.com/golang/velociraptor/artifacts/proto" config_proto "www.velocidex.com/golang/velociraptor/config/proto" + "www.velocidex.com/golang/velociraptor/constants" "www.velocidex.com/golang/velociraptor/json" "www.velocidex.com/golang/velociraptor/logging" "www.velocidex.com/golang/velociraptor/services" @@ -187,7 +188,7 @@ func (self *Dummy) materializeTool( if err != nil { return err } - request.Header.Set("User-Agent", "") + request.Header.Set("User-Agent", constants.USER_AGENT) res, err := self.Client.Do(request) if err != nil { return err @@ -224,7 +225,7 @@ func getGithubRelease(ctx context.Context, Client networking.HTTPClient, return "", err } - request.Header.Set("User-Agent", "") + request.Header.Set("User-Agent", constants.USER_AGENT) logger.Info("Resolving latest Github release for %v", tool.Name) res, err := Client.Do(request) if err != nil { diff --git a/services/inventory/inventory.go b/services/inventory/inventory.go index 067bc117cd2..5af1fbe5289 100644 --- a/services/inventory/inventory.go +++ b/services/inventory/inventory.go @@ -49,6 +49,7 @@ import ( "google.golang.org/protobuf/proto" artifacts_proto "www.velocidex.com/golang/velociraptor/artifacts/proto" config_proto "www.velocidex.com/golang/velociraptor/config/proto" + "www.velocidex.com/golang/velociraptor/constants" "www.velocidex.com/golang/velociraptor/datastore" "www.velocidex.com/golang/velociraptor/logging" "www.velocidex.com/golang/velociraptor/paths" @@ -352,7 +353,7 @@ func (self *InventoryService) materializeTool( if err != nil { return err } - request.Header.Set("User-Agent", "") + request.Header.Set("User-Agent", constants.USER_AGENT) res, err := self.Client.Do(request) if err != nil { return err diff --git a/services/notebook/initial.go b/services/notebook/initial.go index 60ea5e7f5b2..9629bc41202 100644 --- a/services/notebook/initial.go +++ b/services/notebook/initial.go @@ -365,7 +365,7 @@ LET ERRORS = SELECT ClientId, FlowId, Flow.start_time As StartedTime, Flow.state AS FlowState, Flow.status as FlowStatus, Flow.execution_duration as Duration, - Flow.total_collected_bytes as TotalBytes, + Flow.total_uploaded_bytes as TotalBytes, Flow.total_collected_rows as TotalRows FROM hunt_flows(hunt_id=HuntId) WHERE FlowState =~ 'ERROR' @@ -390,7 +390,7 @@ SELECT ClientId, FlowId, Flow.start_time As StartedTime, Flow.state AS FlowState, Flow.status as FlowStatus, Flow.execution_duration as Duration, - Flow.total_collected_bytes as TotalBytes, + Flow.total_uploaded_bytes as TotalBytes, Flow.total_collected_rows as TotalRows FROM hunt_flows(hunt_id=HuntId) WHERE FlowState =~ 'RUNNING' @@ -403,7 +403,7 @@ SELECT ClientId, FlowId, Flow.start_time As StartedTime, Flow.state AS FlowState, Flow.status as FlowStatus, Flow.execution_duration as Duration, - Flow.total_collected_bytes as TotalBytes, + Flow.total_uploaded_bytes as TotalBytes, Flow.total_collected_rows as TotalRows FROM hunt_flows(hunt_id=HuntId) WHERE FlowState =~ 'Finished' diff --git a/vql/networking/http_client.go b/vql/networking/http_client.go index 670099fd994..0f6400e4179 100644 --- a/vql/networking/http_client.go +++ b/vql/networking/http_client.go @@ -34,6 +34,7 @@ import ( "www.velocidex.com/golang/velociraptor/acls" "www.velocidex.com/golang/velociraptor/artifacts" config_proto "www.velocidex.com/golang/velociraptor/config/proto" + "www.velocidex.com/golang/velociraptor/constants" "www.velocidex.com/golang/velociraptor/utils" "www.velocidex.com/golang/velociraptor/vql" vql_subsystem "www.velocidex.com/golang/velociraptor/vql" @@ -308,6 +309,9 @@ func (self *_HttpPlugin) Call( } scope.Log("Fetching %v\n", arg.Url) + if arg.UserAgent == "" { + arg.UserAgent = constants.USER_AGENT + } req.Header.Set("User-Agent", arg.UserAgent) diff --git a/vql/tools/logscale/logscale.go b/vql/tools/logscale/logscale.go index dca808f9a5e..2d149a4f5bf 100644 --- a/vql/tools/logscale/logscale.go +++ b/vql/tools/logscale/logscale.go @@ -17,6 +17,7 @@ import ( "github.com/Velocidex/ordereddict" "github.com/hashicorp/go-retryablehttp" config_proto "www.velocidex.com/golang/velociraptor/config/proto" + "www.velocidex.com/golang/velociraptor/constants" "www.velocidex.com/golang/velociraptor/file_store/api" "www.velocidex.com/golang/velociraptor/file_store/directory" "www.velocidex.com/golang/velociraptor/json" @@ -470,7 +471,7 @@ func (self *LogScaleQueue) postBytes(scope vfilter.Scope, data []byte, count int return nil, err } - req.Header.Set("User-Agent", "") + req.Header.Set("User-Agent", constants.USER_AGENT) req.Header.Add("Accept", "application/json") req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", self.authToken)) diff --git a/vql/tools/webdav_upload.go b/vql/tools/webdav_upload.go index 35d9bf3961e..a6f6fb81c05 100644 --- a/vql/tools/webdav_upload.go +++ b/vql/tools/webdav_upload.go @@ -15,6 +15,7 @@ import ( "www.velocidex.com/golang/velociraptor/accessors" "www.velocidex.com/golang/velociraptor/acls" "www.velocidex.com/golang/velociraptor/artifacts" + "www.velocidex.com/golang/velociraptor/constants" "www.velocidex.com/golang/velociraptor/uploads" "www.velocidex.com/golang/velociraptor/vql" vql_subsystem "www.velocidex.com/golang/velociraptor/vql" @@ -52,6 +53,10 @@ func (self *WebDAVUploadFunction) Call(ctx context.Context, scope.Log("upload_webdav: NoVerifyCert is deprecated, please use SkipVerify instead") } + if arg.UserAgent == "" { + arg.UserAgent = constants.USER_AGENT + } + err = vql_subsystem.CheckFilesystemAccess(scope, arg.Accessor) if err != nil { scope.Log("upload_webdav: %s", err)