Added symlink processing to linux file accessor (#3173)

accessors/file/accessor_common.go

@@ -336,16 +336,33 @@ func (self OSFileSystemAccessor) ReadDirWithOSPath(
 	} else {
 		// If it is a symlink, we need to check the target of the
 		// symlink and make sure it is a directory.
-		target, err := os.Readlink(dir)
+		target, err := filepath.EvalSymlinks(dir)
 		if err == nil {
+			// The target is interpreted relative to the directory of
+			// the link.
+			if !strings.HasPrefix(target, "/") {
+				target = full_path.Dirname().PathSpec().Path + "/" + target
+			}
 			lstat, err := os.Lstat(target)
+
 			// Target of the link is not there or inaccessible or
 			// points to something that is not a directory - just
 			// ignore it with no errors.
 			if err != nil || !lstat.IsDir() {
 				return nil, nil
 			}
+
+			sys, ok := lstat.Sys().(*syscall.Stat_t)
+			if ok {
+				// Keep track of the links we visited.
+				if self.context.WasLinkVisited(
+					uint64(sys.Dev), sys.Ino) {
+					return nil, errors.New("Symlink cycle detected")
+				}
+				self.context.LinkVisited(uint64(sys.Dev), sys.Ino)
+			}
 		}
+		dir = target
 	}
 
 	dirfstype := getFSType(dir)

accessors/file/accessor_linux_test.go

@@ -0,0 +1,98 @@
+// +build linux
+
+package file
+
+import (
+	"context"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"sort"
+	"testing"
+
+	"github.com/Velocidex/ordereddict"
+	"github.com/alecthomas/assert"
+	"github.com/sebdah/goldie"
+	"github.com/stretchr/testify/suite"
+	"www.velocidex.com/golang/velociraptor/accessors"
+	"www.velocidex.com/golang/velociraptor/config"
+	"www.velocidex.com/golang/velociraptor/glob"
+	"www.velocidex.com/golang/velociraptor/json"
+	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
+	"www.velocidex.com/golang/velociraptor/vql/acl_managers"
+)
+
+type AccessorLinuxTestSuite struct {
+	suite.Suite
+	tmpdir string
+}
+
+func (self *AccessorLinuxTestSuite) TestLinuxSymlinks() {
+	tmpdir, err := ioutil.TempDir("", "accessor_test")
+	assert.NoError(self.T(), err)
+
+	// Create two symlinks.
+	// tmp/second_bin/ -> tmp/zbin
+	// tmp/zbin -> /bin/
+
+	err = os.Symlink("/bin", filepath.Join(tmpdir, "zbin"))
+	assert.NoError(self.T(), err)
+
+	err = os.Symlink(filepath.Join(tmpdir, "zbin"),
+		filepath.Join(tmpdir, "second_bin"))
+	assert.NoError(self.T(), err)
+
+	// Create a symlink cycle:
+	// tmp/subdir is a directory
+	// tmp/sym1 -> tmp/subdir
+	// tmp/subdir/sym2 -> tmp
+
+	dirname := filepath.Join(tmpdir, "subdir")
+	err = os.Mkdir(dirname, 0777)
+	assert.NoError(self.T(), err)
+
+	err = os.Mkdir(filepath.Join(dirname, "ls"), 0777)
+	assert.NoError(self.T(), err)
+
+	err = os.Symlink(dirname, filepath.Join(tmpdir, "sym1"))
+	assert.NoError(self.T(), err)
+
+	err = os.Symlink(tmpdir, filepath.Join(dirname, "sym2"))
+	assert.NoError(self.T(), err)
+
+	scope := vql_subsystem.MakeScope().AppendVars(ordereddict.NewDict().
+		Set(vql_subsystem.ACL_MANAGER_VAR, acl_managers.NullACLManager{}))
+	scope.SetLogger(log.New(os.Stderr, " ", 0))
+
+	glob_path, _ := accessors.NewLinuxOSPath("/**/ls")
+	tmp_path, _ := accessors.NewLinuxOSPath(tmpdir)
+
+	options := glob.GlobOptions{
+		DoNotFollowSymlinks: false,
+	}
+	globber := glob.NewGlobber().WithOptions(options)
+
+	globber.Add(glob_path)
+
+	accessor, err := accessors.GetAccessor("file", scope)
+	assert.NoError(self.T(), err)
+
+	config_obj := config.GetDefaultConfig()
+	hits := []string{}
+	for hit := range globber.ExpandWithContext(
+		context.Background(), scope,
+		config_obj, tmp_path, accessor) {
+		hits = append(hits, hit.OSPath().TrimComponents(
+			tmp_path.Components...).String())
+	}
+
+	sort.Strings(hits)
+
+	goldie.Assert(self.T(), "TestLinuxSymlinks", json.MustMarshalIndent(hits))
+}
+
+// Test Linux specific File accessor.
+func TestFileLinux(t *testing.T) {
+	suite.Run(t, &AccessorLinuxTestSuite{})
+}

accessors/file/accessor_test.go

@@ -182,6 +182,7 @@ func (self *AccessorWindowsTestSuite) TestSymlinks() {
 
 }
 
+// Test both the Windows and Linux File accessor.
 func TestWindowsLinux(t *testing.T) {
 	suite.Run(t, &AccessorWindowsTestSuite{})
 }

accessors/file/fixtures/TestLinuxSymlinks.golden

@@ -0,0 +1,6 @@
+[
+ "/second_bin/ls",
+ "/subdir/ls",
+ "/subdir/sym2/subdir/ls",
+ "/sym1/ls"
+]

glob/glob.go

@@ -200,7 +200,9 @@ func (self *Globber) _add_filter(components []_PathFilterer, globs []string) err
 }
 
 func (self *Globber) is_dir_or_link(
-	f accessors.FileInfo, accessor accessors.FileSystemAccessor, depth int) bool {
+	scope vfilter.Scope, root *accessors.OSPath,
+	f accessors.FileInfo,
+	accessor accessors.FileSystemAccessor, depth int) bool {
 	// Do not follow symlinks to symlinks deeply.
 	if depth > 10 {
 		return false
@@ -220,7 +222,11 @@ func (self *Globber) is_dir_or_link(
 		}
 
 		target, err := f.GetLink()
-		if err == nil {
+		if err != nil {
+			//scope.Log("Globber: %v while processing %v",
+			//	err, root.String())
+
+		} else {
 			target_info, err := accessor.Lstat(target.String())
 			if err == nil {
 				// Check if the target is on a different filesystem
@@ -235,7 +241,8 @@ func (self *Globber) is_dir_or_link(
 					}
 				}
 
-				return self.is_dir_or_link(target_info, accessor, depth+1)
+				return self.is_dir_or_link(
+					scope, root, target_info, accessor, depth+1)
 			}
 
 			// Hmm we failed to lstat the target - assume
@@ -311,7 +318,7 @@ func (self *Globber) ExpandWithContext(
 				}
 
 				// Only recurse into directories.
-				if self.is_dir_or_link(f, accessor, 0) {
+				if self.is_dir_or_link(scope, root, f, accessor, 0) {
 					item := []*Globber{next}
 					prev_item, pres := children[basename]
 					if pres {

services/ddclient/ddclient.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
+	"www.velocidex.com/golang/velociraptor/constants"
 	"www.velocidex.com/golang/velociraptor/logging"
 	"www.velocidex.com/golang/velociraptor/vql/networking"
 )
@@ -200,7 +201,7 @@ func UpdateDDNSRecord(config_obj *config_proto.Config,
 	if err != nil {
 		return err
 	}
-	req.Header.Set("User-Agent", "")
+	req.Header.Set("User-Agent", constants.USER_AGENT)
 	req.SetBasicAuth(user, pw)
 
 	resp, err := client.Do(req)

services/inventory/dummy.go

@@ -19,6 +19,7 @@ import (
 	"google.golang.org/protobuf/proto"
 	artifacts_proto "www.velocidex.com/golang/velociraptor/artifacts/proto"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
+	"www.velocidex.com/golang/velociraptor/constants"
 	"www.velocidex.com/golang/velociraptor/json"
 	"www.velocidex.com/golang/velociraptor/logging"
 	"www.velocidex.com/golang/velociraptor/services"
@@ -187,7 +188,7 @@ func (self *Dummy) materializeTool(
 	if err != nil {
 		return err
 	}
-	request.Header.Set("User-Agent", "")
+	request.Header.Set("User-Agent", constants.USER_AGENT)
 	res, err := self.Client.Do(request)
 	if err != nil {
 		return err
@@ -224,7 +225,7 @@ func getGithubRelease(ctx context.Context, Client networking.HTTPClient,
 		return "", err
 	}
 
-	request.Header.Set("User-Agent", "")
+	request.Header.Set("User-Agent", constants.USER_AGENT)
 	logger.Info("Resolving latest Github release for <green>%v</>", tool.Name)
 	res, err := Client.Do(request)
 	if err != nil {

services/inventory/inventory.go

@@ -49,6 +49,7 @@ import (
 	"google.golang.org/protobuf/proto"
 	artifacts_proto "www.velocidex.com/golang/velociraptor/artifacts/proto"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
+	"www.velocidex.com/golang/velociraptor/constants"
 	"www.velocidex.com/golang/velociraptor/datastore"
 	"www.velocidex.com/golang/velociraptor/logging"
 	"www.velocidex.com/golang/velociraptor/paths"
@@ -352,7 +353,7 @@ func (self *InventoryService) materializeTool(
 	if err != nil {
 		return err
 	}
-	request.Header.Set("User-Agent", "")
+	request.Header.Set("User-Agent", constants.USER_AGENT)
 	res, err := self.Client.Do(request)
 	if err != nil {
 		return err

services/notebook/initial.go

@@ -365,7 +365,7 @@ LET ERRORS = SELECT ClientId,
        FlowId, Flow.start_time As StartedTime,
        Flow.state AS FlowState, Flow.status as FlowStatus,
        Flow.execution_duration as Duration,
-       Flow.total_collected_bytes as TotalBytes,
+       Flow.total_uploaded_bytes as TotalBytes,
        Flow.total_collected_rows as TotalRows
 FROM hunt_flows(hunt_id=HuntId)
 WHERE FlowState =~ 'ERROR'
@@ -390,7 +390,7 @@ SELECT ClientId,
        FlowId, Flow.start_time As StartedTime,
        Flow.state AS FlowState, Flow.status as FlowStatus,
        Flow.execution_duration as Duration,
-       Flow.total_collected_bytes as TotalBytes,
+       Flow.total_uploaded_bytes as TotalBytes,
        Flow.total_collected_rows as TotalRows
 FROM hunt_flows(hunt_id=HuntId)
 WHERE FlowState =~ 'RUNNING'
@@ -403,7 +403,7 @@ SELECT ClientId,
        FlowId, Flow.start_time As StartedTime,
        Flow.state AS FlowState, Flow.status as FlowStatus,
        Flow.execution_duration as Duration,
-       Flow.total_collected_bytes as TotalBytes,
+       Flow.total_uploaded_bytes as TotalBytes,
        Flow.total_collected_rows as TotalRows
 FROM hunt_flows(hunt_id=HuntId)
 WHERE FlowState =~ 'Finished'

vql/networking/http_client.go

@@ -34,6 +34,7 @@ import (
 	"www.velocidex.com/golang/velociraptor/acls"
 	"www.velocidex.com/golang/velociraptor/artifacts"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
+	"www.velocidex.com/golang/velociraptor/constants"
 	"www.velocidex.com/golang/velociraptor/utils"
 	"www.velocidex.com/golang/velociraptor/vql"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
@@ -308,6 +309,9 @@ func (self *_HttpPlugin) Call(
 		}
 
 		scope.Log("Fetching %v\n", arg.Url)
+		if arg.UserAgent == "" {
+			arg.UserAgent = constants.USER_AGENT
+		}
 
 		req.Header.Set("User-Agent", arg.UserAgent)
 

vql/tools/logscale/logscale.go

@@ -17,6 +17,7 @@ import (
 	"github.com/Velocidex/ordereddict"
 	"github.com/hashicorp/go-retryablehttp"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
+	"www.velocidex.com/golang/velociraptor/constants"
 	"www.velocidex.com/golang/velociraptor/file_store/api"
 	"www.velocidex.com/golang/velociraptor/file_store/directory"
 	"www.velocidex.com/golang/velociraptor/json"
@@ -470,7 +471,7 @@ func (self *LogScaleQueue) postBytes(scope vfilter.Scope, data []byte, count int
 		return nil, err
 	}
 
-	req.Header.Set("User-Agent", "")
+	req.Header.Set("User-Agent", constants.USER_AGENT)
 	req.Header.Add("Accept", "application/json")
 	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", self.authToken))
 

vql/tools/webdav_upload.go

@@ -15,6 +15,7 @@ import (
 	"www.velocidex.com/golang/velociraptor/accessors"
 	"www.velocidex.com/golang/velociraptor/acls"
 	"www.velocidex.com/golang/velociraptor/artifacts"
+	"www.velocidex.com/golang/velociraptor/constants"
 	"www.velocidex.com/golang/velociraptor/uploads"
 	"www.velocidex.com/golang/velociraptor/vql"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
@@ -52,6 +53,10 @@ func (self *WebDAVUploadFunction) Call(ctx context.Context,
 		scope.Log("upload_webdav: NoVerifyCert is deprecated, please use SkipVerify instead")
 	}
 
+	if arg.UserAgent == "" {
+		arg.UserAgent = constants.USER_AGENT
+	}
+
 	err = vql_subsystem.CheckFilesystemAccess(scope, arg.Accessor)
 	if err != nil {
 		scope.Log("upload_webdav: %s", err)