Sync to Master

Makefile

@@ -92,7 +92,7 @@ lint:
 	golangci-lint run
 
 KapeFilesSync:
-	python3 scripts/kape_files.py -t win ~/projects/KapeFiles/ > artifacts/definitions/Windows/KapeFiles/Targets.yaml
+	python3 scripts/kape_files.py -t win --state_file_path scripts/templates/kape_files_state.json ~/projects/KapeFiles/ > artifacts/definitions/Windows/KapeFiles/Targets.yaml
 
 SQLECmdSync:
 	python3 scripts/sqlecmd_convert.py ~/projects/SQLECmd/ ~/projects/KapeFiles/ artifacts/definitions/Generic/Collectors/SQLECmd.yaml

accessors/manipulators.go

@@ -25,9 +25,9 @@ var (
 
 // This is a generic Path manipulator that implements the escaping
 // standard as used by Velociraptor:
-// 1. Path separators are / but will be able to use \\ to parse.
-// 2. Each component is optionally quoted if it contains special
-//    characters (like path separators).
+//  1. Path separators are / but will be able to use \\ to parse.
+//  2. Each component is optionally quoted if it contains special
+//     characters (like path separators).
 type GenericPathManipulator struct {
 	Sep string
 }
@@ -124,6 +124,16 @@ func (self LinuxPathManipulator) PathParse(path string, result *OSPath) error {
 	return nil
 }
 
+func (self LinuxPathManipulator) PathJoin(path *OSPath) string {
+	osPathSerializations.Inc()
+
+	result := self.AsPathSpec(path)
+	if result.GetDelegateAccessor() == "" && result.GetDelegatePath() == "" {
+		return result.Path
+	}
+	return result.String()
+}
+
 func (self LinuxPathManipulator) AsPathSpec(path *OSPath) *PathSpec {
 	result := path.pathspec
 	if result == nil {

accessors/manipulators_test.go

@@ -12,11 +12,43 @@ type testcase struct {
 	expected_path   string
 }
 
+var generic_testcases = []testcase{
+	// Generic paths try to take a good guess of the path type:
+	// 1. Use / or \ as path separator
+	// 2. Quotes represent unbroken paths.
+	{"/bin/file\\1.txt", []string{"bin", "file", "1.txt"}, "/bin/file/1.txt"},
+
+	// Quotes in the filename are escaped by doubling up and enclosing
+	// the component with a single quote.
+	{"/bin/file\"1\".txt", []string{"bin", "file\"1\".txt"},
+		`/bin/"file""1"".txt"`},
+
+	{`/bin/"file""1"".txt"`, []string{"bin", "file\"1\".txt"},
+		`/bin/"file""1"".txt"`},
+
+	// Enclosing a path in quotes treats it as a single literal
+	// component.
+	{"/bin/\"file\\1.txt\"", []string{"bin", "file\\1.txt"}, "/bin/\"file\\1.txt\""},
+}
+
+func TestGenericManipulators(t *testing.T) {
+	for _, testcase := range generic_testcases {
+		path, err := NewGenericOSPath(testcase.serialized_path)
+		assert.NoError(t, err)
+		assert.Equal(t, testcase.components, path.Components)
+		assert.Equal(t, testcase.expected_path, path.String())
+	}
+}
+
 var linux_testcases = []testcase{
 	{"/bin/ls", []string{"bin", "ls"}, "/bin/ls"},
 	{"bin////ls", []string{"bin", "ls"}, "/bin/ls"},
 	{"/bin/ls////", []string{"bin", "ls"}, "/bin/ls"},
 
+	// Files with non-path backslash characters should be parsed as
+	// one filename. They should also be serialized as a single file.
+	{"/bin/file\\1.txt", []string{"bin", "file\\1.txt"}, "/bin/file\\1.txt"},
+
 	// Ignore and dont support directory traversal at all
 	{"/bin/../../../.././../../ls", []string{"bin", "ls"}, "/bin/ls"},
 

accessors/vfs/vfs.go

@@ -2,7 +2,6 @@ package vfs
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
 
 	"github.com/Velocidex/ordereddict"
@@ -11,14 +10,17 @@ import (
 	api_proto "www.velocidex.com/golang/velociraptor/api/proto"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
 	flows_proto "www.velocidex.com/golang/velociraptor/flows/proto"
+	"www.velocidex.com/golang/velociraptor/json"
 	"www.velocidex.com/golang/velociraptor/services"
+	"www.velocidex.com/golang/velociraptor/utils"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
 	"www.velocidex.com/golang/vfilter"
 )
 
 var (
 	ErrNotFound     = errors.New("file not found")
 	ErrNotAvailable = errors.New("File content not available")
+	ErrInvalidRow   = errors.New("Stored row is invalid")
 )
 
 type VFSFileSystemAccessor struct {
@@ -97,14 +99,18 @@ func (self VFSFileSystemAccessor) LstatWithOSPath(filename *accessors.OSPath) (
 	if err != nil {
 		return nil, err
 	}
+
 	// Find the row that matches this filename
 	for _, r := range res.Rows {
-		if len(r.Cell) < 12 {
+		var row []interface{}
+		_ = json.Unmarshal([]byte(r.Json), &row)
+		if len(row) < 12 {
 			continue
 		}
 
-		if r.Cell[5] == filename.Basename() {
-			return rowCellToFSInfo(r.Cell)
+		name, ok := row[5].(string)
+		if ok && name == filename.Basename() {
+			return rowCellToFSInfo(row)
 		}
 	}
 
@@ -147,11 +153,13 @@ func (self VFSFileSystemAccessor) ReadDirWithOSPath(
 
 	result := []accessors.FileInfo{}
 	for _, r := range res.Rows {
-		if len(r.Cell) < 12 {
+		var row []interface{}
+		_ = json.Unmarshal([]byte(r.Json), &row)
+		if len(row) < 12 {
 			continue
 		}
 
-		fs_info, err := rowCellToFSInfo(r.Cell)
+		fs_info, err := rowCellToFSInfo(row)
 		if err != nil {
 			continue
 		}
@@ -191,14 +199,21 @@ func (self VFSFileSystemAccessor) OpenWithOSPath(filename *accessors.OSPath) (
 
 	// Find the row that matches this filename
 	for _, r := range res.Rows {
-		if len(r.Cell) < 12 {
+		var row []interface{}
+		_ = json.Unmarshal([]byte(r.Json), &row)
+		if len(row) < 12 {
+			continue
+		}
+
+		name, ok := row[5].(string)
+		if !ok {
 			continue
 		}
 
-		if r.Cell[5] == filename.Basename() {
+		if name == filename.Basename() {
 			// Check if it has a download link
 			record := &flows_proto.VFSDownloadInfo{}
-			err = json.Unmarshal([]byte(r.Cell[0]), record)
+			err = utils.ParseIntoProtobuf(row[0], record)
 			if err != nil || record.Name == "" {
 				return nil, ErrNotAvailable
 			}
@@ -211,23 +226,31 @@ func (self VFSFileSystemAccessor) OpenWithOSPath(filename *accessors.OSPath) (
 	return nil, ErrNotFound
 }
 
-func rowCellToFSInfo(cell []string) (accessors.FileInfo, error) {
-	components := []string{}
-	err := json.Unmarshal([]byte(cell[2]), &components)
-	if err != nil {
-		return nil, err
+func rowCellToFSInfo(cell []interface{}) (accessors.FileInfo, error) {
+	components := utils.ConvertToStringSlice(cell[2])
+	if len(components) == 0 {
+		return nil, ErrInvalidRow
 	}
 
-	size := int64(0)
-	_ = json.Unmarshal([]byte(cell[6]), &size)
+	size, ok := utils.ToInt64(cell[6])
+	if !ok {
+		return nil, ErrInvalidRow
+	}
 
-	is_dir := false
-	if len(cell[7]) > 1 && cell[7][0] == 'd' {
-		is_dir = true
+	mode, ok := cell[7].(string)
+	if !ok {
+		return nil, ErrInvalidRow
 	}
 
+	is_dir := len(mode) > 1 && mode[0] == 'd'
+
 	// The Accessor + components is the path of the item
-	path := accessors.MustNewGenericOSPath(cell[3]).Append(components...)
+	ospath, ok := cell[3].(string)
+	if !ok {
+		return nil, ErrInvalidRow
+	}
+
+	path := accessors.MustNewGenericOSPath(ospath).Append(components...)
 	fs_info := &accessors.VirtualFileInfo{
 		Path:   path,
 		IsDir_: is_dir,
@@ -237,7 +260,7 @@ func rowCellToFSInfo(cell []string) (accessors.FileInfo, error) {
 
 	// The download pointer allows us to fetch the file itself.
 	record := &flows_proto.VFSDownloadInfo{}
-	err = json.Unmarshal([]byte(cell[0]), record)
+	err := utils.ParseIntoProtobuf(cell[0], record)
 	if err == nil {
 		fs_info.Data_.Set("DownloadInfo", record)
 	}

api/api.go

@@ -38,11 +38,11 @@ import (
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/reflection"
 	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/emptypb"
 	"www.velocidex.com/golang/velociraptor/acls"
 	actions_proto "www.velocidex.com/golang/velociraptor/actions/proto"
 	"www.velocidex.com/golang/velociraptor/api/authenticators"
-	"www.velocidex.com/golang/velociraptor/api/proto"
 	api_proto "www.velocidex.com/golang/velociraptor/api/proto"
 	"www.velocidex.com/golang/velociraptor/api/tables"
 	artifacts_proto "www.velocidex.com/golang/velociraptor/artifacts/proto"
@@ -62,7 +62,7 @@ import (
 )
 
 type ApiServer struct {
-	proto.UnimplementedAPIServer
+	api_proto.UnimplementedAPIServer
 	server_obj         *server.Server
 	ca_pool            *x509.CertPool
 	wg                 *sync.WaitGroup
@@ -713,8 +713,14 @@ func (self *ApiServer) GetArtifacts(
 
 		for _, name := range in.Names {
 			artifact, pres := repository.Get(ctx, org_config_obj, name)
+			artifact_clone := proto.Clone(artifact).(*artifacts_proto.Artifact)
+			for _, s := range artifact_clone.Sources {
+				s.Queries = nil
+			}
+			artifact_clone.Raw = ""
+
 			if pres {
-				result.Items = append(result.Items, artifact)
+				result.Items = append(result.Items, artifact_clone)
 			}
 		}
 		return result, nil

api/authenticators/common.go

@@ -46,7 +46,7 @@ func getSignedJWTTokenCookie(
 	// replay sessioon cookies past expiry.
 	expiry := time.Now().Add(time.Minute * time.Duration(expiry_min))
 
-	// Enfore the JWT to expire
+	// Enforce the JWT to expire
 	claims.Expires = float64(expiry.Unix())
 
 	// Make a JWT and sign it.

api/flows.go

@@ -79,20 +79,27 @@ func (self *ApiServer) GetClientFlows(
 		if flow.Request == nil {
 			continue
 		}
-		row_data := []string{
+		row_data := []interface{}{
 			flow.State.String(),
 			flow.SessionId,
-			json.AnyToString(flow.Request.Artifacts, vjson.DefaultEncOpts()),
-			json.AnyToString(flow.CreateTime, vjson.DefaultEncOpts()),
-			json.AnyToString(flow.ActiveTime, vjson.DefaultEncOpts()),
-			json.AnyToString(flow.Request.Creator, vjson.DefaultEncOpts()),
-			json.AnyToString(flow.TotalUploadedBytes, vjson.DefaultEncOpts()),
-			json.AnyToString(flow.TotalCollectedRows, vjson.DefaultEncOpts()),
-			json.MustMarshalProtobufString(flow, vjson.DefaultEncOpts()),
-			json.AnyToString(flow.Request.Urgent, vjson.DefaultEncOpts()),
-			json.AnyToString(flow.ArtifactsWithResults, vjson.DefaultEncOpts()),
+			flow.Request.Artifacts,
+			flow.CreateTime,
+			flow.ActiveTime,
+			flow.Request.Creator,
+			flow.TotalUploadedBytes,
+			flow.TotalCollectedRows,
+			json.ConvertProtoToOrderedDict(flow),
+			flow.Request.Urgent,
+			flow.ArtifactsWithResults,
 		}
-		result.Rows = append(result.Rows, &api_proto.Row{Cell: row_data})
+		opts := vjson.DefaultEncOpts()
+		serialized, err := json.MarshalWithOptions(row_data, opts)
+		if err != nil {
+			continue
+		}
+		result.Rows = append(result.Rows, &api_proto.Row{
+			Json: string(serialized),
+		})
 	}
 
 	return result, nil

api/hunts.go

@@ -74,18 +74,26 @@ func (self *ApiServer) GetHuntFlows(
 			continue
 		}
 
-		row_data := []string{
+		row_data := []interface{}{
 			flow.Context.ClientId,
 			services.GetHostname(ctx, org_config_obj, flow.Context.ClientId),
 			flow.Context.SessionId,
-			json.AnyToString(flow.Context.StartTime/1000, vjson.DefaultEncOpts()),
+			flow.Context.StartTime / 1000,
 			flow.Context.State.String(),
-			json.AnyToString(flow.Context.ExecutionDuration/1000000000,
-				vjson.DefaultEncOpts()),
-			json.AnyToString(flow.Context.TotalUploadedBytes, vjson.DefaultEncOpts()),
-			json.AnyToString(flow.Context.TotalCollectedRows, vjson.DefaultEncOpts())}
+			flow.Context.ExecutionDuration / 1000000000,
+			flow.Context.TotalUploadedBytes,
+			flow.Context.TotalCollectedRows,
+		}
+
+		opts := vjson.DefaultEncOpts()
+		serialized, err := json.MarshalWithOptions(row_data, opts)
+		if err != nil {
+			continue
+		}
 
-		result.Rows = append(result.Rows, &api_proto.Row{Cell: row_data})
+		result.Rows = append(result.Rows, &api_proto.Row{
+			Json: string(serialized),
+		})
 
 		if uint64(len(result.Rows)) > in.Rows {
 			break
@@ -142,19 +150,25 @@ func (self *ApiServer) GetHuntTable(
 			total_clients_scheduled = hunt.Stats.TotalClientsScheduled
 		}
 
-		row_data := []string{
+		row_data := []interface{}{
 			fmt.Sprintf("%v", hunt.State),
-			json.AnyToString(hunt.Tags, vjson.DefaultEncOpts()),
+			hunt.Tags,
 			hunt.HuntId,
 			hunt.HuntDescription,
-			json.AnyToString(hunt.CreateTime, vjson.DefaultEncOpts()),
-			json.AnyToString(hunt.StartTime, vjson.DefaultEncOpts()),
-			json.AnyToString(hunt.Expires, vjson.DefaultEncOpts()),
-			fmt.Sprintf("%v", total_clients_scheduled),
+			hunt.CreateTime,
+			hunt.StartTime,
+			hunt.Expires,
+			total_clients_scheduled,
 			hunt.Creator,
 		}
-
-		result.Rows = append(result.Rows, &api_proto.Row{Cell: row_data})
+		opts := vjson.DefaultEncOpts()
+		serialized, err := json.MarshalWithOptions(row_data, opts)
+		if err != nil {
+			continue
+		}
+		result.Rows = append(result.Rows, &api_proto.Row{
+			Json: string(serialized),
+		})
 
 		if uint64(len(result.Rows)) > in.Rows {
 			break