analysis: add some misc analyses/analysis work

src/analysis/pass/filter-xed-nop/filter-xed-nop

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# filter-xed-nop: find and remove cohorts that XED thinks are NOPs.
+
+require "json"
+
+# TODO(ww): Remove this.
+XED_SO = "./src/worker/xed/xed.so"
+
+warn "[+] pass: filter-xed-nop"
+
+count = 0
+$stdin.each_line do |line|
+  result = JSON.parse line, symbolize_names: true
+
+  xed = result[:outputs].find { |o| o[:worker_so] == XED_SO }
+
+  # `addr32 nop ...` and `nop ...` both occur
+  if xed[:result].include?("nop ")
+    count += 1
+    next
+  end
+
+  $stdout.puts result.to_json
+end
+
+warn "[+] pass: filter-xed-nop done: #{count} filtered"

src/analysis/pass/filter-zydis-find-overaccept/filter-zydis-find-overaccept

@@ -0,0 +1,48 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# filter-zydis-find-overaccept: find inputs that Zydis potentially overaccepts
+# (i.e., inputs the other high-quality decoders think are invalid)
+
+require "json"
+
+# TODO(ww): Remove this.
+BDDISASM_SO = "./src/worker/bddisasm/bddisasm.so"
+XED_SO = "./src/worker/xed/xed.so"
+ZYDIS_SO = "./src/worker/zydis/zydis.so"
+ICED_SO = "./src/worker/iced/iced.so"
+
+def success?(decoder)
+  decoder[:status][:value] == 1
+end
+
+def failure?(decoder)
+  !success?(decoder)
+end
+
+def failure_by_consensus?(*decoders)
+  nfailures = decoders.select { |d| failure?(d) }.size
+
+  (nfailures / decoders.size.to_f) > 0.50
+end
+
+warn "[+] pass: filter-zydis-find-overaccept"
+
+count = 0
+$stdin.each_line do |line|
+  result = JSON.parse line, symbolize_names: true
+
+  bddisasm = result[:outputs].find { |o| o[:worker_so] == BDDISASM_SO }
+  xed = result[:outputs].find { |o| o[:worker_so] == XED_SO }
+  zydis = result[:outputs].find { |o| o[:worker_so] == ZYDIS_SO }
+  iced = result[:outputs].find { |o| o[:worker_so] == ICED_SO }
+
+  # If Zydis reports success when other high-quality decoders don't, keep it.
+  if success?(zydis) && failure_by_consensus?(bddisasm, xed, iced)
+    $stdout.puts result.to_json
+  end
+
+  count += 1
+end
+
+warn "[+] pass: filter-zydis-find-overaccept done: #{count} filtered"

src/analysis/pass/filter-zydis-find-overaccept/spec.yml

@@ -0,0 +1,3 @@
+name: filter-zydis-find-overaccept
+desc: Find results that Zydis potentially overaccepts
+run: filter-zydis-find-overaccept

src/analysis/pass/find-small-inputs/find-small-inputs

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# find-small-inputs: find cohorts whose inputs are relatively small.
+
+require "json"
+
+warn "[+] pass: find-small-inputs"
+
+count = 0
+$stdin.each_line do |line|
+  result = JSON.parse line, symbolize_names: true
+
+  # input is a hex string, so 8 bytes * 2 bytes per hex char
+  if result[:input].size <= 16
+    count += 1
+    next
+  end
+
+  $stdout.puts result.to_json
+end
+
+warn "[+] pass: find-small-inputs done: #{count} filtered"

src/analysis/pass/find-small-inputs/spec.yml

@@ -0,0 +1,3 @@
+name: find-small-inputs
+desc: Find results whose inputs are relatively small (<= 8 bytes)
+run: find-small-inputs

src/analysis/pass/prune-capstone/prune-capstone

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# prune-capstone: remove Capstone results from each cohort, if present
+
+require "json"
+
+# TODO(ww): Remove this.
+CAPSTONE_SO = "./src/worker/capstone/capstone.so"
+
+warn "[+] pass: prune-capstone"
+
+$stdin.each_line do |line|
+  result = JSON.parse line, symbolize_names: true
+  result[:outputs].reject! { |o| o[:worker_so] == CAPSTONE_SO }
+
+  $stdout.puts result.to_json
+end
+
+warn "[+] pass: prune-capstone done"

src/analysis/pass/prune-capstone/spec.yml

@@ -0,0 +1,3 @@
+name: prune-capstone
+desc: Remove all Capstone results
+run: prune-capstone

src/analysis/pass/prune-libopcodes/prune-libopcodes

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# prune-libopcodes: remove libopcodes results from each cohort, if present
+
+require "json"
+
+# TODO(ww): Remove this.
+LIBOPCODES_SO = "./src/worker/bfd/bfd.so"
+
+warn "[+] pass: prune-libopcodes"
+
+$stdin.each_line do |line|
+  result = JSON.parse line, symbolize_names: true
+  result[:outputs].reject! { |o| o[:worker_so] == LIBOPCODES_SO }
+
+  $stdout.puts result.to_json
+end
+
+warn "[+] pass: prune-libopcodes done"

src/analysis/pass/prune-libopcodes/spec.yml

@@ -0,0 +1,3 @@
+name: prune-libopcodes
+desc: Remove all libopcodes (BFD) results
+run: prune-libopcodes

src/analysis/passes.yml

@@ -68,3 +68,16 @@ xed-underaccept:
   - filter-xed-find-underaccept
   - minimize-input
   - normalize
+
+# Find relatively small (<= 8 byte) inputs.
+# This should be run after an analysis that does `minimize-input`
+small-inputs:
+  - find-small-inputs
+
+# Remove all libopcodes results.
+remove-libopcodes:
+  - prune-libopcodes
+
+# Remove all capstone results.
+remove-capstone:
+  - prune-capstone