diff --git a/accessors/vmdk/cache.go b/accessors/vmdk/cache.go index 068704c7bd2..dc40a094efd 100644 --- a/accessors/vmdk/cache.go +++ b/accessors/vmdk/cache.go @@ -93,6 +93,9 @@ func getCachedVMDKFile( return utils.MakeReaderAtter(fd), func() { fd.Close() }, nil }) + if err != nil { + return nil, err + } vmdk_file := &VMDKFile{ reader: vmdk_ctx, diff --git a/go.mod b/go.mod index b43f0dcad0e..fabdf798970 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Velocidex/amsi v0.0.0-20200608120838-e5d93b76f119 github.com/Velocidex/etw v0.0.0-20250201014319-bf4266efe992 github.com/Velocidex/go-elasticsearch/v7 v7.3.1-0.20191001125819-fee0ef9cac6b - github.com/Velocidex/go-magic v0.0.0-20211018155418-c5dc48282f28 + github.com/Velocidex/go-magic v0.0.0-20250203094020-32f94b14f00f github.com/Velocidex/go-yara v1.1.10-0.20240309155455-3f491847cec9 github.com/Velocidex/grpc-go-pool v1.2.2-0.20241016164850-ff0cb80037a8 github.com/Velocidex/json v0.0.0-20220224052537-92f3c0326e5a @@ -92,7 +92,7 @@ require ( www.velocidex.com/golang/go-pe v0.1.1-0.20250101153735-7a925ba8334b www.velocidex.com/golang/go-prefetch v0.0.0-20240910051453-2385582c1c22 www.velocidex.com/golang/oleparse v0.0.0-20230217092320-383a0121aafe - www.velocidex.com/golang/regparser v0.0.0-20240404115756-2169ac0e3c09 + www.velocidex.com/golang/regparser v0.0.0-20250203141505-31e704a67ef7 www.velocidex.com/golang/vfilter v0.0.0-20241123123542-6b030f4d2090 ) diff --git a/go.sum b/go.sum index dc1eb850f60..eba1746514d 100644 --- a/go.sum +++ b/go.sum @@ -56,8 +56,6 @@ github.com/Velocidex/chroma v0.6.8-0.20200418131129-82edc291369c h1:ipQHX4FX5HKR github.com/Velocidex/chroma v0.6.8-0.20200418131129-82edc291369c/go.mod h1:sko8vR34/90zvl5QdcUdvzL3J8NKjAUx9va9jPuFNoM= github.com/Velocidex/errors v0.0.0-20221019164655-9ace6bf61e26 h1:VwbeNpMRuS3bRieg7WLaSYIMaI8RjH/wGxd37oj6H1g= github.com/Velocidex/errors v0.0.0-20221019164655-9ace6bf61e26/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= -github.com/Velocidex/etw v0.0.0-20250102072915-dae077e67c53 h1:uySW0T4zQyJSe4xcJ94KB14NYXm1cCAOFSJCKXLYb1k= -github.com/Velocidex/etw v0.0.0-20250102072915-dae077e67c53/go.mod h1:1flhaSAPAtB7fS7InVaBJxxmLttgWOlx7rXG6Sftg5U= github.com/Velocidex/etw v0.0.0-20250201014319-bf4266efe992 h1:LbwyRalm+ZNdpvuoH+UXUPacJPGmXK+tMvSWa5wWOUk= github.com/Velocidex/etw v0.0.0-20250201014319-bf4266efe992/go.mod h1:1flhaSAPAtB7fS7InVaBJxxmLttgWOlx7rXG6Sftg5U= github.com/Velocidex/file-rotatelogs v0.0.0-20211221020724-d12e4dae4e11 h1:pQY9p6hvmbFKXJg8suzGSG9/t8Ij9ece32GUFIdHgqg= @@ -74,8 +72,8 @@ github.com/Velocidex/go-fat v0.0.0-20230923165230-3e6c4265297a h1:dWHPlB3C86vh+M github.com/Velocidex/go-fat v0.0.0-20230923165230-3e6c4265297a/go.mod h1:g74FCv59tsVP48V2o1eyIK8aKbNKPLJIJ+HuiUPVc6E= github.com/Velocidex/go-journalctl v0.0.0-20241004063153-cc1c858415bd h1:CSTW6zYoG1IFxaGM3N42wSwruigV1xZ4gNzjLgb2xIc= github.com/Velocidex/go-journalctl v0.0.0-20241004063153-cc1c858415bd/go.mod h1:5WxXsCtLdEvnc4FsFa+QfMwOWYkfey3nlQbPssZWqjc= -github.com/Velocidex/go-magic v0.0.0-20211018155418-c5dc48282f28 h1:3FMhXfGzZR4oNHmV8NizrviyaTv+2SmLuj+43cMJCUQ= -github.com/Velocidex/go-magic v0.0.0-20211018155418-c5dc48282f28/go.mod h1:n9o/44DFcqU/E55pWoIt4sKkxBC3k4JVNqvTAb9kZlI= +github.com/Velocidex/go-magic v0.0.0-20250203094020-32f94b14f00f h1:KCDi0hKrkDrn0DI2L8cSMkrF0yWj57c6VIhAKmmQFV8= +github.com/Velocidex/go-magic v0.0.0-20250203094020-32f94b14f00f/go.mod h1:2oVfOYRdtA0yuSZiN9ai8PRgxvkw6SLUlUXy1Sm76qk= github.com/Velocidex/go-mscfb v0.0.0-20240618091452-31f4ccc54002 h1:FWeeVb/x+XvaIKZyMdxwB+JYmj4dSATSuu+DBfCXFVU= github.com/Velocidex/go-mscfb v0.0.0-20240618091452-31f4ccc54002/go.mod h1:YvYAfyK6Jg2WIaqvK42KPmVDfU8FSVxoSiZSVJfihDo= github.com/Velocidex/go-vhdx v0.0.0-20240601014259-b204818c95fd h1:znnjIQdOK6aqsG/crrEBAWBJzYdg1+jn/IGLdozC0qU= @@ -975,8 +973,8 @@ www.velocidex.com/golang/go-prefetch v0.0.0-20240910051453-2385582c1c22 h1:Re+Yl www.velocidex.com/golang/go-prefetch v0.0.0-20240910051453-2385582c1c22/go.mod h1:UNIUmQhflpSTt7TH4o/6O/GiMCjSzIALXe9/zzTKFCw= www.velocidex.com/golang/oleparse v0.0.0-20230217092320-383a0121aafe h1:o9jQWSwKTLhBeavfOk054/HK5yNi6Ni9VHQ6rxYZEi4= www.velocidex.com/golang/oleparse v0.0.0-20230217092320-383a0121aafe/go.mod h1:R7IisRzDO7q5LVRJsCQf1xA50LrIavsPWzAjVE4THyY= -www.velocidex.com/golang/regparser v0.0.0-20240404115756-2169ac0e3c09 h1:G1RWYBXP2lSzxKcrAU1YhiUlBetZ7hGIzIiWuuazvfo= -www.velocidex.com/golang/regparser v0.0.0-20240404115756-2169ac0e3c09/go.mod h1:pxSECT5mWM3goJ4sxB4HCJNKnKqiAlpyT8XnvBwkLGU= +www.velocidex.com/golang/regparser v0.0.0-20250203141505-31e704a67ef7 h1:BMX/37sYwX+8JhHt+YNbPfbx7dXG1w1L1mXonNBtjt0= +www.velocidex.com/golang/regparser v0.0.0-20250203141505-31e704a67ef7/go.mod h1:pxSECT5mWM3goJ4sxB4HCJNKnKqiAlpyT8XnvBwkLGU= www.velocidex.com/golang/vfilter v0.0.0-20241123123542-6b030f4d2090 h1:AuS8qXwIxLUKy3wfNKD4Li14iu3PIKDOPJoKpDL3SFk= www.velocidex.com/golang/vfilter v0.0.0-20241123123542-6b030f4d2090/go.mod h1:P50KPQr2LpWVAu7ilGH8CBLBASGtOJ2971yA9YhR8rY= www.velocidex.com/golang/vtypes v0.0.0-20240123105603-069d4a7f435c h1:rL/It+Ig+mvIhmy9vl5gg5b6CX2J12x0v2SXIT2RoWE= diff --git a/services/launcher/launcher_test.go b/services/launcher/launcher_test.go index d3fd851fbc9..08a1cd013d4 100644 --- a/services/launcher/launcher_test.go +++ b/services/launcher/launcher_test.go @@ -1336,6 +1336,7 @@ func getReqName(in *actions_proto.VQLCollectorArgs) string { func (self *LauncherTestSuite) TestDelete() { assert.Retry(self.T(), 3, time.Second, self._TestDelete) } + func (self *LauncherTestSuite) _TestDelete(t *assert.R) { launcher, err := services.GetLauncher(self.ConfigObj) assert.NoError(t, err) @@ -1396,7 +1397,6 @@ func (self *LauncherTestSuite) _TestDelete(t *assert.R) { res, err = launcher.GetFlows(self.Ctx, self.ConfigObj, "server", result_sets.ResultSetOptions{}, 0, 10) assert.NoError(t, err) - fmt.Printf("Flows %v\n", res) time.Sleep(time.Second) return len(res.Items) == 0 }) diff --git a/vql/sigma/evaluator/evaluate.go b/vql/sigma/evaluator/evaluate.go index 3fc375f5873..33a5840f8cc 100644 --- a/vql/sigma/evaluator/evaluate.go +++ b/vql/sigma/evaluator/evaluate.go @@ -32,22 +32,17 @@ type VQLRuleEvaluator struct { lambda *vfilter.Lambda lambda_args *ordereddict.Dict - fieldmappings []FieldMappingRecord + fieldmappings *FieldMappingResolver // If this rule has a correlator, then forward the match to the // correlator. Correlator *SigmaCorrelator `json:"correlator,omitempty" yaml:"correlator,omitempty"` } -type FieldMappingRecord struct { - Name string - Lambda *vfilter.Lambda -} - func NewVQLRuleEvaluator( scope types.Scope, rule sigma.Rule, - fieldmappings []FieldMappingRecord) *VQLRuleEvaluator { + fieldmappings *FieldMappingResolver) *VQLRuleEvaluator { result := &VQLRuleEvaluator{ scope: scope, Rule: rule, diff --git a/vql/sigma/evaluator/evaluate_search.go b/vql/sigma/evaluator/evaluate_search.go index f47ee62488f..4a1ca03175a 100644 --- a/vql/sigma/evaluator/evaluate_search.go +++ b/vql/sigma/evaluator/evaluate_search.go @@ -178,10 +178,9 @@ func (self *VQLRuleEvaluator) GetFieldValuesFromEvent( field string, event *Event) ([]interface{}, error) { // There is a field mapping - lets evaluate it - for _, m := range self.fieldmappings { - if m.Name == field { - return toGenericSlice(event.Reduce(ctx, scope, field, m.Lambda)), nil - } + lambda, err := self.fieldmappings.Get(field) + if err == nil { + return toGenericSlice(event.Reduce(ctx, scope, field, lambda)), nil } value, ok := event.Get(field) diff --git a/vql/sigma/evaluator/event.go b/vql/sigma/evaluator/event.go index 090fe14a387..7b990fdb584 100644 --- a/vql/sigma/evaluator/event.go +++ b/vql/sigma/evaluator/event.go @@ -55,6 +55,45 @@ func (self *Event) Copy() *ordereddict.Dict { return result } +func (self *Event) Get(key string) (interface{}, bool) { + self.mu.Lock() + defer self.mu.Unlock() + + cached, pres := self.cache[key] + if pres { + return cached, true + } + + // The following supports the special case where the field has dot + // notation. This aleviate the need to have pre-defined field + // mappings and allows us to access fields directly. We only + // support dict style events this way. This method is actually + // faster than the VQL lambda as we dont need to use VQL scopes to + // access the fields. + var value interface{} = self.Dict + + for _, part := range strings.Split(key, ".") { + // We only allow dereferencing of dict events by default. For + // other data structures use a lambda which will use the + // entire VQL machinery to dereference fields properly. + value_dict, ok := value.(*ordereddict.Dict) + if !ok { + // It is not a dict - can not dereference it. + return nil, false + } + + next_value, pres := value_dict.Get(part) + if !pres { + return nil, false + } + + value = next_value + } + + self.cache[key] = value + return value, true +} + func (self *Event) Reduce( ctx context.Context, scope types.Scope, field string, lambda *vfilter.Lambda) types.Any { diff --git a/vql/sigma/evaluator/fieldmapping.go b/vql/sigma/evaluator/fieldmapping.go new file mode 100644 index 00000000000..5817d0ac40d --- /dev/null +++ b/vql/sigma/evaluator/fieldmapping.go @@ -0,0 +1,49 @@ +package evaluator + +import ( + "sync" + + "www.velocidex.com/golang/velociraptor/utils" + "www.velocidex.com/golang/vfilter" +) + +type FieldMappingRecord struct { + Name string + Lambda *vfilter.Lambda +} + +type FieldMappingResolver struct { + mappings map[string]FieldMappingRecord + mu sync.Mutex +} + +func (self *FieldMappingResolver) Get(name string) (*vfilter.Lambda, error) { + self.mu.Lock() + defer self.mu.Unlock() + + res, pres := self.mappings[name] + if !pres { + return nil, utils.NotFoundError + } + return res.Lambda, nil +} + +func (self *FieldMappingResolver) Len() int { + self.mu.Lock() + defer self.mu.Unlock() + + return len(self.mappings) +} + +func (self *FieldMappingResolver) Set(name string, lambda *vfilter.Lambda) { + self.mu.Lock() + defer self.mu.Unlock() + + self.mappings[name] = FieldMappingRecord{Name: name, Lambda: lambda} +} + +func NewFieldMappingResolver() *FieldMappingResolver { + return &FieldMappingResolver{ + mappings: make(map[string]FieldMappingRecord), + } +} diff --git a/vql/sigma/fixtures/TestSigma.golden b/vql/sigma/fixtures/TestSigma.golden index d943cee036f..546b10be900 100644 --- a/vql/sigma/fixtures/TestSigma.golden +++ b/vql/sigma/fixtures/TestSigma.golden @@ -1884,5 +1884,57 @@ } } } + ], + "Automatic Field Mappings": [ + { + "Foo": { + "Bar": { + "Baz": "Hello world" + } + }, + "Details": null, + "_Match": { + "match": true, + "search_results": { + "automaticField": true + }, + "condition_results": [ + true + ] + }, + "_Rule": { + "Title": "Automatic Field Mappings", + "Logsource": { + "Product": "windows", + "Service": "application" + }, + "Detection": { + "Searches": { + "automaticField": { + "event_matchers": [ + [ + { + "field": "Foo.Bar.Baz", + "modifiers": [ + "contains" + ], + "values": [ + "Hello" + ] + } + ] + ] + } + }, + "Condition": [ + { + "Search": { + "Name": "automaticField" + } + } + ] + } + } + } ] } \ No newline at end of file diff --git a/vql/sigma/runner.go b/vql/sigma/runner.go index b42893e6a31..08ce97ca331 100644 --- a/vql/sigma/runner.go +++ b/vql/sigma/runner.go @@ -46,7 +46,7 @@ type SigmaContext struct { // Map between sigma field names to event. The lambda will be // passed the event. For example EID can be the lambda // x=>x.System.EventID.Value - fieldmappings []evaluator.FieldMappingRecord + fieldmappings *evaluator.FieldMappingResolver mu sync.Mutex debug bool @@ -147,9 +147,10 @@ func NewSigmaContext( output_chan: output_chan, default_details: default_details, debug: debug, + fieldmappings: evaluator.NewFieldMappingResolver(), } - // Compile the field mappings. NOTE: The compiled_fieldmappings + // Compile the field mappings. NOTE: The compiled fieldmappings // is shared between all the worker goroutines. Benchmarking shows // it is faster to make a slice copy than having to use a mutex to // protect it. This is O(1) but lock free. Using map copies uses @@ -167,8 +168,7 @@ func NewSigmaContext( if err != nil { return nil, fmt.Errorf("fieldmapping for %s is not a valid VQL Lambda: %v", k, err) } - self.fieldmappings = append(self.fieldmappings, - evaluator.FieldMappingRecord{Name: k, Lambda: lambda}) + self.fieldmappings.Set(k, lambda) } } diff --git a/vql/sigma/sigma.go b/vql/sigma/sigma.go index a8279824042..51d2e6734b7 100644 --- a/vql/sigma/sigma.go +++ b/vql/sigma/sigma.go @@ -101,7 +101,7 @@ func (self SigmaPlugin) Call( scope.Log("INFO:sigma: Loaded %v rules (from %v) into %v log sources and %v field mappings", sigma_context.total_rules, len(rules), len(sigma_context.runners), - len(sigma_context.fieldmappings)) + sigma_context.fieldmappings.Len()) for row := range sigma_context.Rows(ctx, scope) { output_chan <- row diff --git a/vql/sigma/sigma_test.go b/vql/sigma/sigma_test.go index 6df12d03dc7..ba647305d38 100644 --- a/vql/sigma/sigma_test.go +++ b/vql/sigma/sigma_test.go @@ -563,6 +563,28 @@ detection: Set("Proc", 1), }, }, + { + description: "Automatic Field Mappings", + rule: ` +title: Automatic Field Mappings +logsource: + product: windows + service: application + +detection: + automaticField: + Foo.Bar.Baz|contains: Hello + + condition: automaticField +`, + fieldmappings: ordereddict.NewDict(), + rows: []*ordereddict.Dict{ + ordereddict.NewDict(). + Set("Foo", ordereddict.NewDict(). + Set("Bar", ordereddict.NewDict(). + Set("Baz", "Hello world"))), + }, + }, } ) @@ -582,7 +604,7 @@ func (self *SigmaTestSuite) TestSigmaModifiers() { plugin := SigmaPlugin{} for _, test_case := range sigmaTestCases { - if false && test_case.description != "Test Conditions" { + if false && test_case.description != "Automatic Field Mappings" { continue } diff --git a/vql/tools/magic.go b/vql/tools/magic.go index 244529f0cb9..a5fa1983f4e 100644 --- a/vql/tools/magic.go +++ b/vql/tools/magic.go @@ -96,7 +96,12 @@ func (self MagicFunction) Call( // Just let libmagic handle the path if arg.Accessor == "" { - return handle.File(arg.Path.String()) + magic, err := handle.File(arg.Path.String()) + if err != nil { + scope.Log("magic: %v", err) + return vfilter.Null{} + } + return magic } err = vql_subsystem.CheckFilesystemAccess(scope, arg.Accessor)