Pharos batch commit January 7, 2022

Backward incompatible changes:

* Adjustments to work with newer versions of ROSE.  The new required
  minimum version of ROSE is now 0.11.39.6.

* Facts produced by OOAnalyzer related to function calls and
  parameters have been changed to be more general purpose.  As a
  consequence fact files generated with earlier versions of OOAnalyzer
  will need to be regenerated.

  These fact changes facilitated a wide variety of small fixes in
  OOAnalyzer that improved accuracy, and previously generated results
  and JSON files should probably be regenerated as well.

Other changes:

* Documentation updates.

* Very untested support for ARM32 in dumpmasm and fn2hash.  (This does
  not affect other tools like OOAnalyzer.)  This optional feature
  requires that ROSE (and pharos) be compiled with Capstone support
  (see https://www.capstone-engine.org/ ).

* Performance improvements.

* Many small bug fixes.

Fixes: #199
Fixes: #198
Fixes: #196
Fixes: #192
Fixes: #191

CMakeLists.txt

@@ -27,13 +27,10 @@ set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS
 
 include(AddCXXCompilerFlags)
 add_cxx_compiler_flags(-pedantic -pedantic-errors -Wall -Wextra -Wshadow -Wstrict-aliasing -Wno-misleading-indentation "-ftemplate-depth=1024" "-ftrack-macro-expansion=0")
-option(PHAROS_FAIL_ON_WARNING "Cause compilation to fail if a warning is generated" OFF)
-if(PHAROS_FAIL_ON_WARNING)
-  add_cxx_compiler_flags(-Werror)
-endif()
 
 set(CMAKE_INSTALL_RPATH_USE_LINK_PATH true)
 set(CMAKE_INSTALL_RPATH "$ORIGIN/../lib")
+set(BUILD_SHARED_LIBS true)
 
 # Load the user's customizations, then the site customization, then
 # the local customization.  This lets the local customization take
@@ -66,7 +63,11 @@ if(CMAKE_BUILD_TYPE STREQUAL "Release")
 endif()
 
 # Rose
-find_package(Rose 0.11.11.2 REQUIRED)
+
+# Anyone modifying this version should also manually modify libpharos/version.hpp.in.  They should set the PHAORS_ROSE_MINI
+set(PHAROS_ROSE_MINIMUM_VERSION 0.11.39.6)
+
+find_package(Rose ${PHAROS_ROSE_MINIMUM_VERSION} REQUIRED)
 
 # Find git
 find_package(Git REQUIRED)
@@ -97,10 +98,17 @@ if(PHAROS_BROKEN_THREADS)
   add_definitions(-DPHAROS_BROKEN_THREADS)
 endif()
 
+# Add gtests
+add_subdirectory(gtest)
+
+option(PHAROS_FAIL_ON_WARNING "Cause compilation to fail if a warning is generated" OFF)
+if(PHAROS_FAIL_ON_WARNING)
+  add_cxx_compiler_flags(-Werror)
+endif()
+
 # Add directories
 add_subdirectory(libpharos)
 add_subdirectory(tools)
-add_subdirectory(gtest)
 add_subdirectory(share)
 
 install(FILES README.md CONTRIBUTING.md LICENSE.md INSTALL.md DESTINATION share/doc/pharos

INSTALL.md

@@ -45,7 +45,7 @@ $ sudo apt update
 $ sudo apt install build-essential wget flex ghostscript bzip2 \
   git subversion automake libtool bison python libncurses-dev \
   vim-common sqlite3 libsqlite3-0 libsqlite3-dev zlib1g-dev cmake \
-  libyaml-cpp-dev libboost-all-dev libboost-dev libxml2-dev
+  libyaml-cpp-dev libboost-all-dev libboost-dev libxml2-dev ninja-build
 ```
 
 Then install all dependencies by running:
@@ -223,10 +223,10 @@ $ cd rose
 This version has a reasonable chance of working or only having minor
 issues.  If you want to be conservative and use the version of ROSE
 that was known to compile with the latest major commit to the Pharos
-repository, you can checkout this commit (ROSE version v0.11.11.2):
+repository, you can checkout this commit (ROSE version v0.11.39.6):
 
 ```
-$ git checkout v0.11.11.2
+$ git checkout v0.11.39.6
 ```
 
 ROSE can be configured in a multitude of ways, and some attention to

README.md

@@ -38,17 +38,15 @@ like opening a file, writing to it, and the closing it.
 
 ## [OOAnalyzer](tools/ooanalyzer/ooanalyzer.pod)
 
-OOAnalyzer is a tool for the analysis and recovery of object oriented
-constructs. This tool was the subject of a paper titled ["Using Logic
-Programming to Recover C++ Classes and Methods from Compiled
-Executables"](https://edmcman.github.io/papers/ccs18.pdf) which was
-published at the ACM Conference on Computer and Communications
-Security in 2018. The tool identifies object members and methods by
-tracking object pointers between functions in the program.  A previous
-implementation of this tool was named "Objdigger", but it was renamed
-to reflect a substantial redesign using Prolog rules to recover the
-object attributes.  For more detailed instructons on how to run
-OOAnalyzer on very large executables, see these
+OOAnalyzer is a tool for the analysis and recovery of object oriented constructs. This tool was
+the subject of a paper titled ["Using Logic Programming to Recover C++ Classes and Methods from
+Compiled Executables"](https://edmcman.github.io/papers/ccs18.pdf) which was published at the
+ACM Conference on Computer and Communications Security in 2018. The tool identifies object
+members and methods by tracking object pointers between functions in the program.  A previous
+implementation of this tool was named "Objdigger", but it was renamed to reflect a substantial
+redesign using Prolog rules to recover the object attributes.  The current version of the tool
+only supports analysis of 32-bit x86 executables compiled by Microsoft Visual C++.  For more
+detailed instructons on how to run OOAnalyzer on very large executables, see these
 [notes](share/prolog/oorules/README.md).
 
 The Pharos distribution used to include a plugin that imported OO

cmake/FindCapstone.cmake

@@ -0,0 +1,2 @@
+include(PharosFindLibrary)
+pharos_find_library(Capstone Capstone::Capstone DYNAMIC LIBRARY capstone INCLUDE capstone/capstone.h)

cmake/FindRose.cmake

@@ -37,11 +37,20 @@ endforeach()
 mark_as_advanced(ROSE_LIBRARY ROSE_INCLUDE_DIR SAWYER_INCLUDE_DIR)
 
 if(ROSE_INCLUDE_DIR)
+
   file(STRINGS "${ROSE_INCLUDE_DIR}/rosePublicConfig.h" _ver_line
     REGEX "^#define ROSE_PACKAGE_VERSION  *\"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\""
     LIMIT_COUNT 1)
   string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+" ROSE_VERSION "${_ver_line}")
   unset(_ver_line)
+
+  file(STRINGS "${ROSE_INCLUDE_DIR}/rosePublicConfig.h" needs_capstone
+    REGEX "^#define ROSE_HAVE_CAPSTONE"
+    LIMIT_COUNT 1)
+  if(needs_capstone)
+    find_package(Capstone REQUIRED)
+  endif()
+
 endif()
 
 include(FindPackageHandleStandardArgs)
@@ -61,13 +70,33 @@ if(ROSE_FOUND)
     add_library(Rose::Rose UNKNOWN IMPORTED)
     set_property(TARGET Rose::Rose PROPERTY INTERFACE_INCLUDE_DIRECTORIES
       ${ROSE_INCLUDE_DIR} ${Z3_INCLUDE_DIRS} ${Boost_INCLUDE_DIRS} ${YamlCpp_INCLUDE_DIR}
-      ${SAWYER_INCLUDE_DIR})
+      ${SAWYER_INCLUDE_DIR} ${Capstone_INCLUDE_DIR})
     set_property(TARGET Rose::Rose PROPERTY INTERFACE_SYSTEM_INCLUDE_DIRECTORIES
       ${ROSE_INCLUDE_DIR} ${Z3_INCLUDE_DIRS} ${Boost_INCLUDE_DIRS} ${YamlCpp_INCLUDE_DIR}
-      ${SAWYER_INCLUDE_DIR})
+      ${SAWYER_INCLUDE_DIR} ${Capstone_INCLUDE_DIR})
     set_property(TARGET Rose::Rose PROPERTY IMPORTED_LOCATION ${ROSE_LIBRARY})
     set_property(TARGET Rose::Rose PROPERTY INTERFACE_LINK_LIBRARIES
       ${Boost_LIBRARIES} ${Z3_LIBRARIES}
       ${YamlCpp_LIBRARY} ${CMAKE_DL_LIBS} Threads::Threads)
   endif()
 endif()
+
+function(rose_version_from_string version_string version_number)
+  string(REGEX MATCHALL "[0-9]+" parts "${version_string}")
+  list(APPEND parts 0 0 0 0)
+  list(GET parts 0 1 2 3 parts)
+  set(digits 3 3 3 4)
+  set(result)
+  foreach(idx RANGE 3)
+    list(GET parts ${idx} part)
+    list(GET digits ${idx} tgt)
+    string(LENGTH ${part} len)
+    while(len LESS tgt)
+      set(part "0${part}")
+      string(LENGTH ${part} len)
+    endwhile()
+    set(result "${result}${part}")
+  endforeach(idx)
+  string(REGEX REPLACE "^0+" "" result "${result}")
+  set("${version_number}" "${result}ul" PARENT_SCOPE)
+endfunction()

libpharos/CMakeLists.txt

@@ -124,6 +124,11 @@ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/config.yaml"
   DESTINATION etc
   RENAME pharos.yaml)
 
+# Create version.hpp
+rose_version_from_string(${ROSE_VERSION} ROSE_NUMERIC_VERSION)
+rose_version_from_string(${PHAROS_ROSE_MINIMUM_VERSION} PHAROS_ROSE_MINIMUM_NUMERIC_VERSION)
+configure_file(version.hpp.in version.hpp)
+
 # Generate PHAROS_REVISION in every build
 add_custom_target(generate_pharos_revision ALL)
 add_custom_command(

libpharos/apigraph.cpp

@@ -1,4 +1,4 @@
-// Copyright 2015-2019 Carnegie Mellon University.  See LICENSE file for terms.
+// Copyright 2015-2021 Carnegie Mellon University.  See LICENSE file for terms.
 
 // Author: Jeff Gennari
 // Date: 2015-06-22
@@ -15,7 +15,6 @@
 #include <boost/algorithm/string.hpp>
 #include <boost/range/adaptors.hpp>
 
-#include <rose.h>
 #include <Sawyer/GraphBoost.h>
 
 #include "pdg.hpp"
@@ -75,7 +74,7 @@ void debug_print_match_table(const ApiParamMatchTable& match_table) {
     assert(plist);
 
     std::ostringstream pstr;;
-    for (const ApiParameterPtr apip : *plist) {
+    for (const ApiParameterPtr & apip : *plist) {
       assert(apip);
       if (apip->value) {
         pstr << *(apip->value->get_expression()) << " | ";
@@ -940,7 +939,7 @@ bool ApiSearchExecutor::EvaluateApiMatchTable(const ApiSigFunc& sig_func,
             assert(sig_list);
 
             bool found_match = false;
-            for (const ApiParameterPtr sig_pd : *sig_list) {
+            for (const ApiParameterPtr & sig_pd : *sig_list) {
               assert(sig_pd);
               if (call_pd.get_value()->can_be_equal(sig_pd->value)) {
                 found_match = true;
@@ -1001,7 +1000,7 @@ bool ApiSearchExecutor::EvaluateApiMatchTable(const ApiSigFunc& sig_func,
           bool rv_found_match = false;
           ApiParamPtrListPtr rlist = ret_entry->second;
           assert(rlist);
-          for (const ApiParameterPtr sig_rv : *rlist) {
+          for (const ApiParameterPtr & sig_rv : *rlist) {
             assert(sig_rv);
 
             if (sig_rv->value && rv.get_value()) {
@@ -2851,7 +2850,7 @@ bool ApiSearchManager::Search(const ApiSigVector &sigs, ApiSearchResultVector &r
   sig_count_ = sigs.size();
   sig_progress_ = 0;
 
-  for (const ApiSig sig : sigs) {
+  for (const ApiSig & sig : sigs) {
     GDEBUG << "Processing signature: " << sig.name << LEND;
 
     UpdateProgress(sig);

libpharos/apigraph.hpp

@@ -1,4 +1,4 @@
-// Copyright 2015-2019 Carnegie Mellon University.  See LICENSE file for terms.
+// Copyright 2015-2021 Carnegie Mellon University.  See LICENSE file for terms.
 
 // Author: Jeff Gennari
 // Date: 2015-06-22
@@ -7,13 +7,15 @@
 #ifndef Pharos_APIGRAPH_H_
 #define Pharos_APIGRAPH_H_
 
-#include <rose.h>
-#include <boost/graph/depth_first_search.hpp>
+#include "rose.hpp"
+#include <Rose/BinaryAnalysis/ControlFlow.h>
+
 #include <Sawyer/Message.h>
 #include <Sawyer/ProgressBar.h>
+
+#include <boost/graph/depth_first_search.hpp>
 #include <boost/property_map/property_map.hpp>
 #include <boost/ptr_container/ptr_vector.hpp>
-#include <BinaryControlFlow.h>
 
 #include "descriptors.hpp"
 #include "apisig.hpp"
@@ -493,7 +495,7 @@ class ApiResultJsonFormatter : public ApiResultFormatter {
 
  public:
 
-  ApiResultJsonFormatter() { 
+  ApiResultJsonFormatter() {
       out_json_ = json::simple_builder()->object();
   }
 

libpharos/apisig.cpp

@@ -1,4 +1,4 @@
-// Copyright 2015, 2016 Carnegie Mellon University.  See LICENSE file for terms.
+// Copyright 2015-2016 Carnegie Mellon University.  See LICENSE file for terms.
 
 #include <algorithm>    // std::copy
 #include <iterator>

libpharos/apisig.hpp

@@ -1,10 +1,15 @@
-// Copyright 2015-2017, 2019 Carnegie Mellon University.  See LICENSE file for terms.
+// Copyright 2015-2021 Carnegie Mellon University.  See LICENSE file for terms.
 
 #ifndef Pharos_APISIG_H_
 #define Pharos_APISIG_H_
 
 #include <iostream>
 #include <fstream>
+
+// Define to an annoying warning in later versions of Boost that results from including the
+// property_tree stuff.
+#define BOOST_BIND_GLOBAL_PLACEHOLDERS 1
+
 #include <boost/algorithm/string.hpp>
 #include <boost/property_tree/json_parser.hpp>
 #include <boost/property_map/property_map.hpp>

libpharos/badcode.cpp

@@ -1,6 +1,4 @@
-// Copyright 2015-2019 Carnegie Mellon University.  See LICENSE file for terms.
-
-#include <rose.h>
+// Copyright 2015-2021 Carnegie Mellon University.  See LICENSE file for terms.
 
 #include "badcode.hpp"
 #include "descriptors.hpp"

libpharos/bua.hpp

@@ -1,9 +1,8 @@
-// Copyright 2018-2019 Carnegie Mellon University.  See LICENSE file for terms.
+// Copyright 2018-2021 Carnegie Mellon University.  See LICENSE file for terms.
 
 #ifndef Pharos_Bua_H
 #define Pharos_Bua_H
 
-#include <rose.h>
 #include "options.hpp"
 #include <atomic>
 

libpharos/calls.cpp

@@ -1,9 +1,9 @@
-// Copyright 2015-2020 Carnegie Mellon University.  See LICENSE file for terms.
+// Copyright 2015-2021 Carnegie Mellon University.  See LICENSE file for terms.
 
 #include <boost/optional.hpp>
 #include <boost/property_map/property_map.hpp>
 
-#include <rose.h>
+#include "rose.hpp"
 #include <AstTraversal.h>
 
 #include "calls.hpp"

libpharos/calls.hpp

@@ -1,12 +1,11 @@
-// Copyright 2015-2020 Carnegie Mellon University.  See LICENSE file for terms.
+// Copyright 2015-2021 Carnegie Mellon University.  See LICENSE file for terms.
 
 #ifndef Pharos_Calls_H
 #define Pharos_Calls_H
 
 #include <boost/format.hpp>
 #include <boost/property_map/property_map.hpp>
 #include <boost/iterator/filter_iterator.hpp> // for filter_iterator
-#include <rose.h>
 
 #include "funcs.hpp"
 #include "imports.hpp"

libpharos/cdg.hpp

@@ -1,10 +1,10 @@
-// Copyright 2015-2019 Carnegie Mellon University.  See LICENSE file for terms.
+// Copyright 2015-2021 Carnegie Mellon University.  See LICENSE file for terms.
 
 #ifndef Pharos_CDG_H
 #define Pharos_CDG_H
 
-#include <rose.h>
-#include <Partitioner2/Partitioner.h>
+#include "rose.hpp"
+#include <Rose/BinaryAnalysis/Partitioner2/Partitioner.h>
 
 #include "misc.hpp"
 #include <boost/range/adaptor/transformed.hpp>

libpharos/convention.cpp

@@ -1,10 +1,9 @@
-// Copyright 2015-2020 Carnegie Mellon University.  See LICENSE file for terms.
+// Copyright 2015-2021 Carnegie Mellon University.  See LICENSE file for terms.
 
 #include <boost/optional.hpp>
 
-#include <rose.h>
-// For isNOP().
-#include <sageInterfaceAsm.h>
+#include "rose.hpp"
+#include <sageInterfaceAsm.h> // For isNOP().
 
 #include "misc.hpp"
 #include "funcs.hpp"
@@ -374,7 +373,7 @@ void RegisterUsage::analyze_parameters() {
         // exclude the xor in the special cases above, but we end up marking the move as a
         // parameter use, which is compeletely incorrect.  At least one way to have detected
         // this is that the value under consideration is not the "starting" value of any register.
-        const std::string& cmt = aa.value->get_comment();
+        const std::string& cmt = aa.value->comment();
         size_t clen = cmt.size();
         // If there's no comment at all, we're not a starting value, and thus not a parameter.
         // Also, if the comment doesn't look like a starting value comment, then we're also not

libpharos/datatypes.hpp

@@ -1,12 +1,10 @@
-// Copyright 2015-2018 Carnegie Mellon University.  See LICENSE file for terms.
+// Copyright 2015-2021 Carnegie Mellon University.  See LICENSE file for terms.
 
 #ifndef Pharos_DataTypes_H
 #define Pharos_DataTypes_H
 
 #include <boost/format.hpp>
 
-#include <rose.h>
-
 #include "globals.hpp"
 
 namespace pharos {