Show prevalence of rules in the output

CHANGELOG.md

@@ -33,6 +33,7 @@ Speaking of new rules, we have eight additions, coming from Ronnie, Jakub, Morit
 - ELF: implement import and export name extractor #1607 #1608 @Aayush-Goel-04
 - bump pydantic from 1.10.9 to 2.1.1 #1582 @Aayush-Goel-04
 - develop script to highlight features not used during matching #331 @Aayush-Goel-04
+- Show prevalence of rules in the output #520 @Aayush-Goel-04
 
 ### New Rules (8)
 

capa/render/default.py

@@ -6,7 +6,11 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
+import gzip
+import json
 import collections
+from typing import Dict
+from pathlib import Path
 
 import tabulate
 
@@ -82,26 +86,66 @@ def render_capabilities(doc: rd.ResultDocument, ostream: StringIO):
         | ...                                                   | ...                                             |
         +-------------------------------------------------------+-------------------------------------------------+
     """
+
+    def load_rules_prevalence(file: Path) -> Dict[str, str]:
+        if not file.exists():
+            raise FileNotFoundError(f"File '{file}' not found.")
+        try:
+            with gzip.open(file, "rb") as gzfile:
+                return json.loads(gzfile.read().decode("utf-8"))
+        except Exception as e:
+            raise RuntimeError(f"An error occurred while loading '{file}': {e}")
+
     subrule_matches = find_subrule_matches(doc)
+    CD = Path(__file__).resolve().parent.parent.parent
+    rules_prevalence = load_rules_prevalence(CD / "assets" / "rules_prevalence.json.gz")
 
-    rows = []
+    # seperate rules based on their prevalence
+    common = []
+    rare = []
     for rule in rutils.capability_rules(doc):
         if rule.meta.name in subrule_matches:
-            # rules that are also matched by other rules should not get rendered by default.
-            # this cuts down on the amount of output while giving approx the same detail.
-            # see #224
             continue
 
         count = len(rule.matches)
-        if count == 1:
-            capability = rutils.bold(rule.meta.name)
+        matches = f"({count} matches)" if count > 1 else ""
+
+        prevalence = rules_prevalence.get(rule.meta.name, None)
+
+        if prevalence == "rare":
+            rare.append((rule.meta.namespace, rule.meta.name, matches, rutils.bold(prevalence)))
+        elif prevalence == "common":
+            common.append((rule.meta.namespace, rule.meta.name, matches, rutils.bold(prevalence)))
         else:
-            capability = f"{rutils.bold(rule.meta.name)} ({count} matches)"
-        rows.append((capability, rule.meta.namespace))
+            common.append((rule.meta.namespace, rule.meta.name, matches, "unknown"))
+
+    rows = []
+
+    def process_data(data):
+        "joins combine similar rules into a single section"
+        if len(data) == 0:
+            return
+        capabilities = ""
+        namespaces = ""
+        prevalences = ""
+
+        for ns, c, i, p in sorted(data, key=lambda x: (x[0], x[1])):
+            capabilities += f"{rutils.bold(c)} {i}\n"
+            namespaces += ns + "\n"
+            prevalences += p + "\n"
+
+        rows.append((capabilities.strip(), namespaces.strip(), prevalences.strip()))
+
+    process_data(rare)
+    process_data(common)
 
     if rows:
         ostream.write(
-            tabulate.tabulate(rows, headers=[width("Capability", 50), width("Namespace", 50)], tablefmt="mixed_outline")
+            tabulate.tabulate(
+                rows,
+                headers=[width("Capability", 50), width("Namespace", 50), width("Prevalence", 10)],
+                tablefmt="mixed_grid",
+            )
         )
         ostream.write("\n")
     else: