Bugfix: yara_lint() should remove invalid imports from yara rules. (#…

…3987)

api/authenticators/oidc.go

@@ -11,7 +11,6 @@ import (
 	api_utils "www.velocidex.com/golang/velociraptor/api/utils"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
 	"www.velocidex.com/golang/velociraptor/logging"
-	"www.velocidex.com/golang/velociraptor/utils"
 )
 
 type OIDCConnector interface {
@@ -140,8 +139,6 @@ func (self *OidcAuthenticator) oauthOidcLogin(
 			}
 			oidcOauthConfig.Endpoint = provider.Endpoint()
 
-			utils.Debug(oidcOauthConfig)
-
 			// Create oauthState cookie
 			oauthState, err := r.Cookie("oauthstate")
 			if err != nil {

api/download.go

@@ -404,9 +404,16 @@ func getRows(
 			return nil, nil, nil, err
 		}
 
-		rs_reader, err := result_sets.NewResultSetReader(
-			file_store_factory, log_path)
+		options, err := tables.GetTableOptions(request)
+		if err != nil {
+			return nil, nil, nil, err
+		}
 
+		rs_reader, err := result_sets.NewResultSetReaderWithOptions(
+			ctx, config_obj, file_store_factory, log_path, options)
+		if err != nil {
+			return nil, nil, nil, err
+		}
 		return rs_reader.Rows(ctx), rs_reader.Close, log_path, err
 	}
 }

artifacts/testdata/server/testcases/yara_lint.in.yaml

@@ -0,0 +1,19 @@
+Parameters:
+  YaraRules: |
+    import "pe"
+
+    /* We dont support import "hash" so this import should be removed. */
+    import "hash"
+
+    rule Hive {
+       condition: pe.imphash() == "f34d5f2d4577ed6d9ceec516c1f5a744"
+    }
+
+    rule BadRule {
+       condition: hash.sha256(0, filesize) == "ab05"
+    }
+
+Queries:
+  - LET Cleaned <= yara_lint(clean=TRUE, rules=YaraRules)
+  - SELECT Cleaned FROM scope()
+  - SELECT * FROM yara(rules=Cleaned, accessor="data", files="Hello")

artifacts/testdata/server/testcases/yara_lint.out.yaml

@@ -0,0 +1,5 @@
+LET Cleaned <= yara_lint(clean=TRUE, rules=YaraRules)[]SELECT Cleaned FROM scope()[
+ {
+  "Cleaned": "import \"pe\"\n\nrule Hive {\n  condition:\n    pe.imphash() == \"f34d5f2d4577ed6d9ceec516c1f5a744\"\n}\n"
+ }
+]SELECT * FROM yara(rules=Cleaned, accessor="data", files="Hello")[]

gui/velociraptor/src/components/core/paged-table.jsx

@@ -667,7 +667,10 @@ class VeloPagedTable extends Component {
         if(_.isEmpty(transform) && !_.isEmpty(this.props.transform)) {
             Object.assign(transform, this.props.transform);
         }
+        return transform;
+    }
 
+    getTransformedRenderer = (transform)=>{
         if(_.isEmpty(transform)) {
             return <></>;
         }
@@ -851,6 +854,16 @@ class VeloPagedTable extends Component {
             downloads.columns = active_columns;
         }
 
+        if(transformed.filter_column) {
+            downloads.filter_column = transformed.filter_column;
+            downloads.filter_regex = transformed.filter_regex;
+        }
+
+        if (transformed.sort_column) {
+            downloads.sort_column = transformed.sort_column;
+            downloads.sort_direction = transformed.sort_direction;
+        }
+
         let all_compacted = true;
         let none_compacted = true;
 
@@ -924,7 +937,7 @@ class VeloPagedTable extends Component {
 
               </ButtonGroup>
                 <ButtonGroup className="float-right">
-                  { transformed }
+                  { this.getTransformedRenderer(transformed) }
                 </ButtonGroup>
               { this.props.toolbar || <></> }
             </Navbar>

magefile.go

@@ -320,6 +320,15 @@ func Arm() error {
 	}.Run()
 }
 
+func Mips() error {
+	return Builder{
+		extra_tags:  " release yara ",
+		goos:        "linux",
+		disable_cgo: true,
+		arch:        "mips",
+	}.Run()
+}
+
 // Builds a Development binary. This does not embed things like GUI
 // resources to allow them to be loaded from the local directory.
 func Dev() error {

vql/common/yara.go

@@ -711,7 +711,7 @@ func RuleGenerator(scope vfilter.Scope, rule string) string {
 			method += " " + kw
 
 		default:
-			scope.Log("Unknown shorthand directive %v", kw)
+			scope.Log("yara: Warning unknown shorthand directive %v - treating as Yara Rule", kw)
 			return rule
 		}
 	}

vql/common/yara_rules.go

@@ -23,26 +23,13 @@ var (
 	includedFunctions = map[string][]string{
 		"pe": []string{
 			"calculate_checksum",
+			"imphash",
 			"section_index",
-			"section_index",
-			"exports",
 			"exports",
-			"exports",
-			"exports_index",
 			"exports_index",
-			"exports_index",
-			"imports",
-			"imports",
 			"imports",
-			"imports",
-			"imports",
-			"imports",
-			"imports",
-			"imports",
-			"import_rva",
 			"import_rva",
 			"delayed_import_rva",
-			"delayed_import_rva",
 			"locale",
 			"language",
 			"is_dll",
@@ -52,27 +39,18 @@ var (
 		"math": {
 			"in_range",
 			"deviation",
-			"deviation",
 			"mean",
-			"mean",
-			"serial_correlation",
 			"serial_correlation",
 			"monte_carlo_pi",
-			"monte_carlo_pi",
-			"entropy",
 			"entropy",
 			"min",
 			"max",
 			"to_number",
 			"abs",
 			"count",
-			"count",
 			"percentage",
-			"percentage",
-			"mode",
 			"mode",
 			"to_string",
-			"to_string",
 		},
 		"elf": {
 			"telfhash",
@@ -528,7 +506,15 @@ func (self *RuleLinter) Lint() (*RuleLinter, []error) {
 		}
 	}
 
-	result.ruleset.Imports = self.ruleset.Imports
+	result.ruleset.Imports = nil
+
+	// Only include valid imports
+	for _, imp := range self.ruleset.Imports {
+		_, pres := includedFunctions[imp]
+		if pres {
+			result.ruleset.Imports = append(result.ruleset.Imports, imp)
+		}
+	}
 	return result, errors
 }
 
@@ -618,7 +604,7 @@ func (self *YaraLintFunction) Call(ctx context.Context,
 func (self *YaraLintFunction) Info(scope vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.FunctionInfo {
 	return &vfilter.FunctionInfo{
 		Name:     "yara_lint",
-		Doc:      "Clean a set of yara rules. This removed invalid or unsupported rules.",
+		Doc:      "Clean a set of yara rules. This removes invalid or unsupported rules.",
 		ArgType:  type_map.AddType(scope, &YaraLintFunctionArgs{}),
 		Metadata: vql.VQLMetadata().Build(),
 	}

vql/server/flows/uploads.go

@@ -93,6 +93,10 @@ func (self UploadsPlugins) Call(
 					components = upload.Stats.Components
 				}
 
+				if len(components) > 0 {
+					components[len(components)-1] += upload.Type
+				}
+
 				vfs_path := path_specs.NewUnsafeFilestorePath(components...).
 					SetType(api.PATH_TYPE_FILESTORE_ANY)