Update plyara structure and tests

YaraGuardian/__init__.py

@@ -1,2 +1,2 @@
-__version__ = '1.1.2'
+__version__ = '1.1.3'
 VERSION = __version__

core/services.py

@@ -11,7 +11,11 @@
 import datetime
 
 import chardet
+
 from plyara import YaraParser
+from plyara import ParserInterpreter
+
+interp = ParserInterpreter()
 
 
 def get_group_or_404(group_name):
@@ -121,7 +125,7 @@ def parse_rule_submission(raw_submission):
 
 
 def check_lexical_convention(entry):
-    return YaraParser.parserInterpreter.isValidRuleName(entry)
+    return interp.isValidRuleName(entry)
 
 
 def generate_kwargs_from_parsed_rule(parsed_rule):
@@ -134,13 +138,13 @@ def generate_kwargs_from_parsed_rule(parsed_rule):
     condition = parsed_rule['condition_terms']
     imports = parsed_rule.get('imports', [])
     comments = parsed_rule.get('comments', [])
-    dependencies = YaraParser.parserInterpreter.detectDependencies(parsed_rule)
+    dependencies = interp.detectDependencies(parsed_rule)
 
     # Calculate hash value of rule strings and condition
-    logic_hash = YaraParser.parserInterpreter.generateLogicHash(parsed_rule)
+    logic_hash = interp.generateLogicHash(parsed_rule)
 
     # Ensure that the proper imports are added based on condition
-    detected_imports = YaraParser.parserInterpreter.detectImports(parsed_rule)
+    detected_imports = interp.detectImports(parsed_rule)
     imports.extend(detected_imports)
 
     # TEMP FIX - Use only a single instance of a metakey

plyara/plyara/plyara.py

@@ -32,30 +32,16 @@ class ParserInterpreter:
     alternative representation of Yara rules
     """
 
-    rules = deque()
-
-    currentRule = {}
-
-    stringModifiersAccumulator = []
-    importsAccumulator = []
-    includesAccumulator = []
-    termAccumulator = []
-    scopeAccumulator = []
-    tagAccumulator = []
-    commentAccumulator = []
-
-    isPrintDebug = False
-
     comparison_operators = ('==', '!=', '>', '<', '>=', '<=')
 
-    import_options = ('pe',
-                      'elf',
-                      'cuckoo',
-                      'magic',
-                      'hash',
-                      'math',
-                      'dotnet',
-                      'androguard')
+    modules = ('pe',
+               'elf',
+               'cuckoo',
+               'magic',
+               'hash',
+               'math',
+               'dotnet',
+               'androguard')
 
     keywords = ('all', 'and', 'any', 'ascii', 'at', 'condition',
                 'contains', 'entrypoint', 'false', 'filesize',
@@ -68,6 +54,25 @@ class ParserInterpreter:
 
     function_keywords = ('uint8', 'uint16', 'uint32', 'uint8be', 'uint16be', 'uint32be')
 
+    def __init__(self, debug=False, additional_modules=None):
+        self.isPrintDebug = debug
+
+        self.rules = deque()
+        self.currentRule = {}
+
+        self.stringModifiersAccumulator = []
+        self.importsAccumulator = []
+        self.includesAccumulator = []
+        self.termAccumulator = []
+        self.scopeAccumulator = []
+        self.tagAccumulator = []
+        self.commentAccumulator = []
+
+        self.import_options = list(self.modules)
+
+        if additional_modules and isinstance(additional_modules, list):
+            self.import_options.extend(additional_modules)
+
     def reset(self):
         self.rules.clear()
 
@@ -81,8 +86,6 @@ def reset(self):
         self.tagAccumulator = []
         self.commentAccumulator = []
 
-        self.isPrintDebug = False
-
     def addElement(self, elementType, elementValue):
         """
         Accepts elements from the parser and uses them to
@@ -91,7 +94,10 @@ def addElement(self, elementType, elementValue):
 
         if elementType == ElementTypes.RULE_NAME:
             self.currentRule["rule_name"] = elementValue
+
             self.readAndResetAccumulators()
+            self.currentRule['imports'] = self.detectImports(self.currentRule)
+
             self.rules.append(self.currentRule)
 
             if self.isPrintDebug:
@@ -729,15 +735,19 @@ def yara_token_generator(data):
 
 class YaraParser(object):
 
-    parserInterpreter = ParserInterpreter()
+    def __init__(self, debug=False, additional_modules=None):
 
-    def __init__(self):
         self.tokens = YaraLexerModule.tokens
         self.lexer = lex.lex(module=YaraLexerModule())
 
-        self.parser = None  # attribute placeholder until build funct is called
-        self.parser_error = {} # attribute placeholder in case error encountered
+        self.parser = yacc.yacc(module=self, debug=debug)
+        self.parserInterpreter = ParserInterpreter(debug=debug,
+                                                   additional_modules=additional_modules)
+
+        # attribute placeholder in case error encountered
+        self.parser_error = {}
 
+        # Comments queue
         self.rule_comments = deque()
 
     def p_rules(self, p):
@@ -972,15 +982,7 @@ def p_error(self, p):
 
             raise TypeError(error_message)
 
-    # Build the parser
-    def build(self):
-        self.parser = yacc.yacc(module=self, debug=False)
-
     def run(self, raw_rules):
-        # Build parser if not done already
-        if not self.parser:
-            self.build()
-
         # Reset parser interpreter state
         self.parserInterpreter.reset()
 

plyara/plyara/tests/data/import_ruleset_androguard.yar

@@ -0,0 +1,16 @@
+// This ruleset is used for unit tests - Modification will require test updates
+
+import "androguard"
+
+rule androguard_001
+{
+    condition:
+        androguard.package_name(/videogame/)
+}
+
+rule androguard_002
+{
+    condition:
+        androguard.activity(/\.sms\./) or
+        androguard.activity("com.package.name.sendSMS")
+}

plyara/plyara/tests/data/import_ruleset_cuckoo.yar

@@ -0,0 +1,9 @@
+// This ruleset is used for unit tests - Modification will require test updates
+
+import "cuckoo"
+
+rule cuckoo_001
+{
+    condition:
+        cuckoo.network.http_request(/http:\/\/someone\.doingevil\.com/)
+}

plyara/plyara/tests/data/import_ruleset_dotnet.yar

@@ -0,0 +1,16 @@
+// This ruleset is used for unit tests - Modification will require test updates
+
+import "dotnet"
+
+rule dotnet_001
+{
+    condition:
+        dotnet.number_of_streams != 5
+}
+
+rule dotnet_002
+{
+    condition:
+        for any i in (0..dotnet.number_of_streams - 1):
+            (dotnet.streams[i].name == "#Blop")
+}

plyara/plyara/tests/data/import_ruleset_elf.yar

@@ -0,0 +1,15 @@
+// This ruleset is used for unit tests - Modification will require test updates
+
+import "elf"
+
+rule elf_001
+{
+    condition:
+        elf.number_of_sections == 1
+}
+
+rule elf_002
+{
+    condition:
+        elf.machine == elf.EM_X86_64
+}

plyara/plyara/tests/data/import_ruleset_hash.yar

@@ -0,0 +1,15 @@
+// This ruleset is used for unit tests - Modification will require test updates
+
+import "hash"
+
+rule hash_001
+{
+    condition:
+        hash.md5("dummy") == "275876e34cf609db118f3d84b799a790"
+}
+
+rule hash_002
+{
+    condition:
+        hash.md5(0, filesize) == "feba6c919e3797e7778e8f2e85fa033d"
+}

plyara/plyara/tests/data/import_ruleset_magic.yar

@@ -0,0 +1,15 @@
+// This ruleset is used for unit tests - Modification will require test updates
+
+import "magic"
+
+rule magic_001
+{
+    condition:
+        magic.type() contains "PDF"
+}
+
+rule magic_002
+{
+    condition:
+        magic.mime_type() == "application/pdf"
+}

plyara/plyara/tests/data/import_ruleset_math.yar

@@ -0,0 +1,9 @@
+// This ruleset is used for unit tests - Modification will require test updates
+
+import "math"
+
+rule math_001
+{
+    condition:
+        uint16(0) == 0x5A4D and math.entropy(0, filesize) > 7.0
+}

plyara/plyara/tests/data/import_ruleset_pe.yar

@@ -0,0 +1,21 @@
+// This ruleset is used for unit tests - Modification will require test updates
+
+import "pe"
+
+rule pe_001
+{
+    condition:
+        pe.number_of_sections == 1
+}
+
+rule pe_002
+{
+    condition:
+        pe.exports("CPlApplet")
+}
+
+rule pe_003
+{
+    condition:
+        pe.characteristics & pe.DLL
+}

plyara/plyara/tests/data/metadata_ruleset.yar

@@ -0,0 +1,35 @@
+// This ruleset is used for unit tests - Modification will require test updates
+
+rule StringTypeMetadata
+{
+    meta:
+        string_value = "String Metadata"
+
+    condition: false
+}
+
+rule IntegerTypeMetadata
+{
+    meta:
+        integer_value = 100
+
+    condition: false
+}
+
+rule BooleanTypeMetadata
+{
+    meta:
+        boolean_value = true
+
+    condition: false
+}
+
+rule AllTypesMetadata
+{
+    meta:
+        string_value = "Different String Metadata"
+        integer_value = 33
+        boolean_value = false
+
+    condition: false
+}

plyara/plyara/tests/data/mixed_ruleset.yar

@@ -0,0 +1,13 @@
+// This ruleset is used for unit tests - Modification will require test updates
+
+rule FirstRule
+{
+    meta:
+        author = "Andrés Iniesta"
+        date = "2015-01-01"
+    strings:
+        $a = "hark, a \"string\" here" fullword ascii
+        $b = { 00 22 44 66 88 aa cc ee }
+    condition:
+        all of them
+}

plyara/plyara/tests/data/scope_ruleset.yar

@@ -0,0 +1,16 @@
+// This ruleset is used for unit tests - Modification will require test updates
+
+global rule GlobalScope
+{
+    condition: false
+}
+
+private rule PrivateScope
+{
+    condition: false
+}
+
+global private rule PrivateGlobalScope
+{
+    condition: false
+}