Added Sigma Default Field Mappings

Normally the Sigma Model needs to specify the field mappings into the
event. This makes it difficult to write rules targeting fields which
are present in the event but have no mapping yet. It also means most
mappings are fairly obvious.

For example consider the event:

{
...
    "EventData": {
        "ImageFileName": "WmiPrvSE.exe"
    }
}

To refer to the ImageFileName we would need a field mapping like:

ImageFileName =  x=>x.EventData.ImageFileName

However in a Sigma rule it is just as easy to specify:

detection:
   filename_condition:
      EventData.ImageFileName|contains: WmiPrvSE

This is both clear and unambigious.

This PR allows field names to have "." and refer directly to the event
itself.

accessors/vmdk/cache.go

@@ -93,6 +93,9 @@ func getCachedVMDKFile(
 			return utils.MakeReaderAtter(fd),
 				func() { fd.Close() }, nil
 		})
+	if err != nil {
+		return nil, err
+	}
 
 	vmdk_file := &VMDKFile{
 		reader: vmdk_ctx,

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/Velocidex/amsi v0.0.0-20200608120838-e5d93b76f119
 	github.com/Velocidex/etw v0.0.0-20250201014319-bf4266efe992
 	github.com/Velocidex/go-elasticsearch/v7 v7.3.1-0.20191001125819-fee0ef9cac6b
-	github.com/Velocidex/go-magic v0.0.0-20211018155418-c5dc48282f28
+	github.com/Velocidex/go-magic v0.0.0-20250203094020-32f94b14f00f
 	github.com/Velocidex/go-yara v1.1.10-0.20240309155455-3f491847cec9
 	github.com/Velocidex/grpc-go-pool v1.2.2-0.20241016164850-ff0cb80037a8
 	github.com/Velocidex/json v0.0.0-20220224052537-92f3c0326e5a
@@ -92,7 +92,7 @@ require (
 	www.velocidex.com/golang/go-pe v0.1.1-0.20250101153735-7a925ba8334b
 	www.velocidex.com/golang/go-prefetch v0.0.0-20240910051453-2385582c1c22
 	www.velocidex.com/golang/oleparse v0.0.0-20230217092320-383a0121aafe
-	www.velocidex.com/golang/regparser v0.0.0-20240404115756-2169ac0e3c09
+	www.velocidex.com/golang/regparser v0.0.0-20250203141505-31e704a67ef7
 	www.velocidex.com/golang/vfilter v0.0.0-20241123123542-6b030f4d2090
 )
 

go.sum

@@ -56,8 +56,6 @@ github.com/Velocidex/chroma v0.6.8-0.20200418131129-82edc291369c h1:ipQHX4FX5HKR
 github.com/Velocidex/chroma v0.6.8-0.20200418131129-82edc291369c/go.mod h1:sko8vR34/90zvl5QdcUdvzL3J8NKjAUx9va9jPuFNoM=
 github.com/Velocidex/errors v0.0.0-20221019164655-9ace6bf61e26 h1:VwbeNpMRuS3bRieg7WLaSYIMaI8RjH/wGxd37oj6H1g=
 github.com/Velocidex/errors v0.0.0-20221019164655-9ace6bf61e26/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
-github.com/Velocidex/etw v0.0.0-20250102072915-dae077e67c53 h1:uySW0T4zQyJSe4xcJ94KB14NYXm1cCAOFSJCKXLYb1k=
-github.com/Velocidex/etw v0.0.0-20250102072915-dae077e67c53/go.mod h1:1flhaSAPAtB7fS7InVaBJxxmLttgWOlx7rXG6Sftg5U=
 github.com/Velocidex/etw v0.0.0-20250201014319-bf4266efe992 h1:LbwyRalm+ZNdpvuoH+UXUPacJPGmXK+tMvSWa5wWOUk=
 github.com/Velocidex/etw v0.0.0-20250201014319-bf4266efe992/go.mod h1:1flhaSAPAtB7fS7InVaBJxxmLttgWOlx7rXG6Sftg5U=
 github.com/Velocidex/file-rotatelogs v0.0.0-20211221020724-d12e4dae4e11 h1:pQY9p6hvmbFKXJg8suzGSG9/t8Ij9ece32GUFIdHgqg=
@@ -74,8 +72,8 @@ github.com/Velocidex/go-fat v0.0.0-20230923165230-3e6c4265297a h1:dWHPlB3C86vh+M
 github.com/Velocidex/go-fat v0.0.0-20230923165230-3e6c4265297a/go.mod h1:g74FCv59tsVP48V2o1eyIK8aKbNKPLJIJ+HuiUPVc6E=
 github.com/Velocidex/go-journalctl v0.0.0-20241004063153-cc1c858415bd h1:CSTW6zYoG1IFxaGM3N42wSwruigV1xZ4gNzjLgb2xIc=
 github.com/Velocidex/go-journalctl v0.0.0-20241004063153-cc1c858415bd/go.mod h1:5WxXsCtLdEvnc4FsFa+QfMwOWYkfey3nlQbPssZWqjc=
-github.com/Velocidex/go-magic v0.0.0-20211018155418-c5dc48282f28 h1:3FMhXfGzZR4oNHmV8NizrviyaTv+2SmLuj+43cMJCUQ=
-github.com/Velocidex/go-magic v0.0.0-20211018155418-c5dc48282f28/go.mod h1:n9o/44DFcqU/E55pWoIt4sKkxBC3k4JVNqvTAb9kZlI=
+github.com/Velocidex/go-magic v0.0.0-20250203094020-32f94b14f00f h1:KCDi0hKrkDrn0DI2L8cSMkrF0yWj57c6VIhAKmmQFV8=
+github.com/Velocidex/go-magic v0.0.0-20250203094020-32f94b14f00f/go.mod h1:2oVfOYRdtA0yuSZiN9ai8PRgxvkw6SLUlUXy1Sm76qk=
 github.com/Velocidex/go-mscfb v0.0.0-20240618091452-31f4ccc54002 h1:FWeeVb/x+XvaIKZyMdxwB+JYmj4dSATSuu+DBfCXFVU=
 github.com/Velocidex/go-mscfb v0.0.0-20240618091452-31f4ccc54002/go.mod h1:YvYAfyK6Jg2WIaqvK42KPmVDfU8FSVxoSiZSVJfihDo=
 github.com/Velocidex/go-vhdx v0.0.0-20240601014259-b204818c95fd h1:znnjIQdOK6aqsG/crrEBAWBJzYdg1+jn/IGLdozC0qU=
@@ -975,8 +973,8 @@ www.velocidex.com/golang/go-prefetch v0.0.0-20240910051453-2385582c1c22 h1:Re+Yl
 www.velocidex.com/golang/go-prefetch v0.0.0-20240910051453-2385582c1c22/go.mod h1:UNIUmQhflpSTt7TH4o/6O/GiMCjSzIALXe9/zzTKFCw=
 www.velocidex.com/golang/oleparse v0.0.0-20230217092320-383a0121aafe h1:o9jQWSwKTLhBeavfOk054/HK5yNi6Ni9VHQ6rxYZEi4=
 www.velocidex.com/golang/oleparse v0.0.0-20230217092320-383a0121aafe/go.mod h1:R7IisRzDO7q5LVRJsCQf1xA50LrIavsPWzAjVE4THyY=
-www.velocidex.com/golang/regparser v0.0.0-20240404115756-2169ac0e3c09 h1:G1RWYBXP2lSzxKcrAU1YhiUlBetZ7hGIzIiWuuazvfo=
-www.velocidex.com/golang/regparser v0.0.0-20240404115756-2169ac0e3c09/go.mod h1:pxSECT5mWM3goJ4sxB4HCJNKnKqiAlpyT8XnvBwkLGU=
+www.velocidex.com/golang/regparser v0.0.0-20250203141505-31e704a67ef7 h1:BMX/37sYwX+8JhHt+YNbPfbx7dXG1w1L1mXonNBtjt0=
+www.velocidex.com/golang/regparser v0.0.0-20250203141505-31e704a67ef7/go.mod h1:pxSECT5mWM3goJ4sxB4HCJNKnKqiAlpyT8XnvBwkLGU=
 www.velocidex.com/golang/vfilter v0.0.0-20241123123542-6b030f4d2090 h1:AuS8qXwIxLUKy3wfNKD4Li14iu3PIKDOPJoKpDL3SFk=
 www.velocidex.com/golang/vfilter v0.0.0-20241123123542-6b030f4d2090/go.mod h1:P50KPQr2LpWVAu7ilGH8CBLBASGtOJ2971yA9YhR8rY=
 www.velocidex.com/golang/vtypes v0.0.0-20240123105603-069d4a7f435c h1:rL/It+Ig+mvIhmy9vl5gg5b6CX2J12x0v2SXIT2RoWE=

services/launcher/launcher_test.go

@@ -1336,6 +1336,7 @@ func getReqName(in *actions_proto.VQLCollectorArgs) string {
 func (self *LauncherTestSuite) TestDelete() {
 	assert.Retry(self.T(), 3, time.Second, self._TestDelete)
 }
+
 func (self *LauncherTestSuite) _TestDelete(t *assert.R) {
 	launcher, err := services.GetLauncher(self.ConfigObj)
 	assert.NoError(t, err)
@@ -1396,7 +1397,6 @@ func (self *LauncherTestSuite) _TestDelete(t *assert.R) {
 		res, err = launcher.GetFlows(self.Ctx, self.ConfigObj, "server",
 			result_sets.ResultSetOptions{}, 0, 10)
 		assert.NoError(t, err)
-		fmt.Printf("Flows %v\n", res)
 		time.Sleep(time.Second)
 		return len(res.Items) == 0
 	})

vql/sigma/evaluator/evaluate.go

@@ -32,22 +32,17 @@ type VQLRuleEvaluator struct {
 	lambda      *vfilter.Lambda
 	lambda_args *ordereddict.Dict
 
-	fieldmappings []FieldMappingRecord
+	fieldmappings *FieldMappingResolver
 
 	// If this rule has a correlator, then forward the match to the
 	// correlator.
 	Correlator *SigmaCorrelator `json:"correlator,omitempty" yaml:"correlator,omitempty"`
 }
 
-type FieldMappingRecord struct {
-	Name   string
-	Lambda *vfilter.Lambda
-}
-
 func NewVQLRuleEvaluator(
 	scope types.Scope,
 	rule sigma.Rule,
-	fieldmappings []FieldMappingRecord) *VQLRuleEvaluator {
+	fieldmappings *FieldMappingResolver) *VQLRuleEvaluator {
 	result := &VQLRuleEvaluator{
 		scope:         scope,
 		Rule:          rule,

vql/sigma/evaluator/evaluate_search.go

@@ -178,10 +178,9 @@ func (self *VQLRuleEvaluator) GetFieldValuesFromEvent(
 	field string, event *Event) ([]interface{}, error) {
 
 	// There is a field mapping - lets evaluate it
-	for _, m := range self.fieldmappings {
-		if m.Name == field {
-			return toGenericSlice(event.Reduce(ctx, scope, field, m.Lambda)), nil
-		}
+	lambda, err := self.fieldmappings.Get(field)
+	if err == nil {
+		return toGenericSlice(event.Reduce(ctx, scope, field, lambda)), nil
 	}
 
 	value, ok := event.Get(field)

vql/sigma/evaluator/event.go

@@ -55,6 +55,45 @@ func (self *Event) Copy() *ordereddict.Dict {
 	return result
 }
 
+func (self *Event) Get(key string) (interface{}, bool) {
+	self.mu.Lock()
+	defer self.mu.Unlock()
+
+	cached, pres := self.cache[key]
+	if pres {
+		return cached, true
+	}
+
+	// The following supports the special case where the field has dot
+	// notation. This aleviate the need to have pre-defined field
+	// mappings and allows us to access fields directly. We only
+	// support dict style events this way. This method is actually
+	// faster than the VQL lambda as we dont need to use VQL scopes to
+	// access the fields.
+	var value interface{} = self.Dict
+
+	for _, part := range strings.Split(key, ".") {
+		// We only allow dereferencing of dict events by default. For
+		// other data structures use a lambda which will use the
+		// entire VQL machinery to dereference fields properly.
+		value_dict, ok := value.(*ordereddict.Dict)
+		if !ok {
+			// It is not a dict - can not dereference it.
+			return nil, false
+		}
+
+		next_value, pres := value_dict.Get(part)
+		if !pres {
+			return nil, false
+		}
+
+		value = next_value
+	}
+
+	self.cache[key] = value
+	return value, true
+}
+
 func (self *Event) Reduce(
 	ctx context.Context, scope types.Scope,
 	field string, lambda *vfilter.Lambda) types.Any {

vql/sigma/evaluator/fieldmapping.go

@@ -0,0 +1,49 @@
+package evaluator
+
+import (
+	"sync"
+
+	"www.velocidex.com/golang/velociraptor/utils"
+	"www.velocidex.com/golang/vfilter"
+)
+
+type FieldMappingRecord struct {
+	Name   string
+	Lambda *vfilter.Lambda
+}
+
+type FieldMappingResolver struct {
+	mappings map[string]FieldMappingRecord
+	mu       sync.Mutex
+}
+
+func (self *FieldMappingResolver) Get(name string) (*vfilter.Lambda, error) {
+	self.mu.Lock()
+	defer self.mu.Unlock()
+
+	res, pres := self.mappings[name]
+	if !pres {
+		return nil, utils.NotFoundError
+	}
+	return res.Lambda, nil
+}
+
+func (self *FieldMappingResolver) Len() int {
+	self.mu.Lock()
+	defer self.mu.Unlock()
+
+	return len(self.mappings)
+}
+
+func (self *FieldMappingResolver) Set(name string, lambda *vfilter.Lambda) {
+	self.mu.Lock()
+	defer self.mu.Unlock()
+
+	self.mappings[name] = FieldMappingRecord{Name: name, Lambda: lambda}
+}
+
+func NewFieldMappingResolver() *FieldMappingResolver {
+	return &FieldMappingResolver{
+		mappings: make(map[string]FieldMappingRecord),
+	}
+}

vql/sigma/fixtures/TestSigma.golden

@@ -1884,5 +1884,57 @@
     }
    }
   }
+ ],
+ "Automatic Field Mappings": [
+  {
+   "Foo": {
+    "Bar": {
+     "Baz": "Hello world"
+    }
+   },
+   "Details": null,
+   "_Match": {
+    "match": true,
+    "search_results": {
+     "automaticField": true
+    },
+    "condition_results": [
+     true
+    ]
+   },
+   "_Rule": {
+    "Title": "Automatic Field Mappings",
+    "Logsource": {
+     "Product": "windows",
+     "Service": "application"
+    },
+    "Detection": {
+     "Searches": {
+      "automaticField": {
+       "event_matchers": [
+        [
+         {
+          "field": "Foo.Bar.Baz",
+          "modifiers": [
+           "contains"
+          ],
+          "values": [
+           "Hello"
+          ]
+         }
+        ]
+       ]
+      }
+     },
+     "Condition": [
+      {
+       "Search": {
+        "Name": "automaticField"
+       }
+      }
+     ]
+    }
+   }
+  }
  ]
 }

vql/sigma/runner.go

@@ -46,7 +46,7 @@ type SigmaContext struct {
 	// Map between sigma field names to event. The lambda will be
 	// passed the event. For example EID can be the lambda
 	// x=>x.System.EventID.Value
-	fieldmappings []evaluator.FieldMappingRecord
+	fieldmappings *evaluator.FieldMappingResolver
 
 	mu          sync.Mutex
 	debug       bool
@@ -147,9 +147,10 @@ func NewSigmaContext(
 		output_chan:     output_chan,
 		default_details: default_details,
 		debug:           debug,
+		fieldmappings:   evaluator.NewFieldMappingResolver(),
 	}
 
-	// Compile the field mappings.  NOTE: The compiled_fieldmappings
+	// Compile the field mappings.  NOTE: The compiled fieldmappings
 	// is shared between all the worker goroutines. Benchmarking shows
 	// it is faster to make a slice copy than having to use a mutex to
 	// protect it. This is O(1) but lock free. Using map copies uses
@@ -167,8 +168,7 @@ func NewSigmaContext(
 			if err != nil {
 				return nil, fmt.Errorf("fieldmapping for %s is not a valid VQL Lambda: %v", k, err)
 			}
-			self.fieldmappings = append(self.fieldmappings,
-				evaluator.FieldMappingRecord{Name: k, Lambda: lambda})
+			self.fieldmappings.Set(k, lambda)
 		}
 	}
 

vql/sigma/sigma.go

@@ -101,7 +101,7 @@ func (self SigmaPlugin) Call(
 
 		scope.Log("INFO:sigma: Loaded %v rules (from %v) into %v log sources and %v field mappings",
 			sigma_context.total_rules, len(rules), len(sigma_context.runners),
-			len(sigma_context.fieldmappings))
+			sigma_context.fieldmappings.Len())
 
 		for row := range sigma_context.Rows(ctx, scope) {
 			output_chan <- row

vql/sigma/sigma_test.go

@@ -563,6 +563,28 @@ detection:
 					Set("Proc", 1),
 			},
 		},
+		{
+			description: "Automatic Field Mappings",
+			rule: `
+title: Automatic Field Mappings
+logsource:
+  product: windows
+  service: application
+
+detection:
+   automaticField:
+      Foo.Bar.Baz|contains: Hello
+
+   condition: automaticField
+`,
+			fieldmappings: ordereddict.NewDict(),
+			rows: []*ordereddict.Dict{
+				ordereddict.NewDict().
+					Set("Foo", ordereddict.NewDict().
+						Set("Bar", ordereddict.NewDict().
+							Set("Baz", "Hello world"))),
+			},
+		},
 	}
 )
 
@@ -582,7 +604,7 @@ func (self *SigmaTestSuite) TestSigmaModifiers() {
 	plugin := SigmaPlugin{}
 
 	for _, test_case := range sigmaTestCases {
-		if false && test_case.description != "Test Conditions" {
+		if false && test_case.description != "Automatic Field Mappings" {
 			continue
 		}
 

vql/tools/magic.go

@@ -96,7 +96,12 @@ func (self MagicFunction) Call(
 
 	// Just let libmagic handle the path
 	if arg.Accessor == "" {
-		return handle.File(arg.Path.String())
+		magic, err := handle.File(arg.Path.String())
+		if err != nil {
+			scope.Log("magic: %v", err)
+			return vfilter.Null{}
+		}
+		return magic
 	}
 
 	err = vql_subsystem.CheckFilesystemAccess(scope, arg.Accessor)