Merge branch 'dev'

DFIR-ORC · Jun 10, 2020 · 4c16164 · 4c16164
2 parents e50af36 + 1ea5b3d
commit 4c16164
Show file tree

Hide file tree

Showing 44 changed files with 3,240 additions and 1,405 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -13,6 +13,7 @@ option(ORC_BUILD_ORC        "Build Orc binary" ON)
 option(ORC_BUILD_PARQUET    "Build Parquet module" OFF)
 option(ORC_BUILD_SQL        "Build SQL module" OFF)
 option(ORC_BUILD_SSDEEP     "Build with ssdeep support" OFF)
+option(ORC_BUILD_JSON       "Build with JSON StructuredOutput enabled" ON)
 
 if(NOT ORC_BUILD_COMMAND)
     set(ORC_BUILD_FASTFIND OFF)
@@ -91,6 +92,10 @@ if(ORC_VCPKG_ROOT)
         list(APPEND _PACKAGES arrow)
     endif()
 
+    if(ORC_BUILD_JSON)
+        list(APPEND _PACKAGES rapidjson)
+    endif()
+
     if(ORC_BUILD_CHAKRACORE)
         list(APPEND _PACKAGES
             chakracore:x86-windows

diff --git a/README.md b/README.md
@@ -59,6 +59,7 @@ Using default options is recommended with the exception of `ORC_BUILD_VCPKG` whi
 | ORC_BUILD_PARQUET    | OFF                   | Build Parquet module (x64)    |
 | ORC_BUILD_SQL        | OFF                   | Build SQL module [1]          |
 | ORC_BUILD_SSDEEP     | OFF                   | Build with ssdeep support     |
+| ORC_BUILD_JSON       | ON                    | Build with JSON enabled       |
 | ORC_USE_STATIC_CRT   | ON                    | Use static runtime            |
 | ORC_VCPKG_ROOT       | ${ORC}/external/vcpkg | VCPKG root directory          |
 | ORC_XMLLITE_PATH     |                       | XmlLite.dll path (xp sp2)     |

diff --git a/azure-pipelines-template.yml b/azure-pipelines-template.yml
@@ -3,29 +3,23 @@ parameters:
 
 jobs:
 - job: vs2017_${{ parameters.arch }}
-  variables:
-    ${{ if eq( parameters.arch, 'x64') }}:
-      winArch: " Win64"
-    ${{ if eq( parameters.arch, 'x86') }}:
-      winArch: ""
   pool:
     vmImage: 'vs2017-win2016'
   steps:
   - checkout: self
     clean: true
     submodules: true
-
-  - task: CMake@1
-    displayName: CMake configuration
-    inputs:
-      workingDirectory: 'build-${{ parameters.arch }}'
-      cmakeArgs: '-G "Visual Studio 15 2017${{ variables.winArch }}" -T v141_xp -DORC_BUILD_VCPKG=ON ..'
-
-  - task: CMake@1
-    displayName: CMake build
-    inputs:
-      workingDirectory: 'build-${{ parameters.arch }}'
-      cmakeArgs: '--build . --config MinSizeRel -- -maxcpucount'
+  - task: PowerShell@2
+    inputs:
+      targetType: 'inline'
+      script: |
+        . $(Build.SourcesDirectory)/tools/ci/build.ps1
+        Build-ORC `
+          -Source $(Build.SourcesDirectory) `
+          -Output $(Build.SourcesDirectory)/artifacts `
+          -Toolchain vs2017 `
+          -Architecture ${{ parameters.arch }} `
+          -Configuration Debug,MinSizeRel
 
   - task: VSTest@2
     inputs:
@@ -36,28 +30,10 @@ jobs:
 
   - task: CopyFiles@2
     inputs:
-      sourceFolder: '$(Build.SourcesDirectory)'
-      contents: 'build-${{ parameters.arch }}/MinSizeRel/dfir-orc_${{ parameters.arch }}.exe'
-      TargetFolder: '$(Build.ArtifactStagingDirectory)'
-
-  - task: CopyFiles@2
-    inputs:
-      sourceFolder: '$(Build.SourcesDirectory)'
-      contents: 'build-${{ parameters.arch }}/MinSizeRel/dfir-orc_${{ parameters.arch }}.pdb'
+      sourceFolder: '$(Build.SourcesDirectory)/artifacts'
+      contents: '**'
       TargetFolder: '$(Build.ArtifactStagingDirectory)'
 
-  - task: CopyFiles@2
-    inputs:
-      sourceFolder: $(Build.SourcesDirectory)
-      contents: 'build-${{ parameters.arch }}/MinSizeRel/fastfind_${{ parameters.arch }}.exe'
-      TargetFolder: $(Build.ArtifactStagingDirectory)
-
-  - task: CopyFiles@2
-    inputs:
-      sourceFolder: $(Build.SourcesDirectory)
-      contents: 'build-${{ parameters.arch }}/MinSizeRel/fastfind_${{ parameters.arch }}.pdb'
-      TargetFolder: $(Build.ArtifactStagingDirectory)
-
   - task: PublishBuildArtifacts@1
     inputs:
       pathtoPublish: $(Build.ArtifactStagingDirectory)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -1,6 +1,11 @@
 trigger:
   - master
 
+parameters:
+- name: PublishRelease
+  type: boolean
+  default: true
+
 stages:
   - stage: Build
     jobs:
@@ -13,6 +18,7 @@ stages:
         arch: "x86"
 
   - stage: Publish
+    condition: ${{parameters.PublishRelease}}
     jobs:
     - job: RetrieveAndPublish
       steps:
@@ -43,13 +49,13 @@ stages:
           tagSource: 'auto'
           tagPattern: '^v?[0-9]+\.[0-9]+\.[0-9]+'
           assets: |
-            $(Build.ArtifactStagingDirectory)/drop_x64/build-x64/MinSizeRel/DFIR-Orc_x64.exe
-            $(Build.ArtifactStagingDirectory)/drop_x64/build-x64/MinSizeRel/DFIR-Orc_x64.pdb
-            $(Build.ArtifactStagingDirectory)/drop_x64/build-x64/MinSizeRel/FastFind_x64.exe
-            $(Build.ArtifactStagingDirectory)/drop_x64/build-x64/MinSizeRel/FastFind_x64.pdb
-            $(Build.ArtifactStagingDirectory)/drop_x86/build-x86/MinSizeRel/DFIR-Orc_x86.exe
-            $(Build.ArtifactStagingDirectory)/drop_x86/build-x86/MinSizeRel/DFIR-Orc_x86.pdb
-            $(Build.ArtifactStagingDirectory)/drop_x86/build-x86/MinSizeRel/FastFind_x86.exe
-            $(Build.ArtifactStagingDirectory)/drop_x86/build-x86/MinSizeRel/FastFind_x86.pdb
+            $(Build.ArtifactStagingDirectory)/drop_x64/bin/MinSizeRel/DFIR-Orc_x64.exe
+            $(Build.ArtifactStagingDirectory)/drop_x64/pdb/MinSizeRel/DFIR-Orc_x64.pdb
+            $(Build.ArtifactStagingDirectory)/drop_x64/bin/MinSizeRel/FastFind_x64.exe
+            $(Build.ArtifactStagingDirectory)/drop_x64/pdb/MinSizeRel/FastFind_x64.pdb
+            $(Build.ArtifactStagingDirectory)/drop_x86/bin/MinSizeRel/DFIR-Orc_x86.exe
+            $(Build.ArtifactStagingDirectory)/drop_x86/pdb/MinSizeRel/DFIR-Orc_x86.pdb
+            $(Build.ArtifactStagingDirectory)/drop_x86/bin/MinSizeRel/FastFind_x86.exe
+            $(Build.ArtifactStagingDirectory)/drop_x86/pdb/MinSizeRel/FastFind_x86.pdb
           changeLogCompareToRelease: 'lastFullRelease'
           changeLogType: 'commitBased'
diff --git a/src/FastFind/FastFindSample.xml b/src/FastFind/FastFindSample.xml
@@ -4,9 +4,9 @@
 
   <filesystem>
 
-    <location shadows="no">T:\</location>
+    <location shadows="no">D:\</location>
 
-    <ntfs_find header="MZ"  />
+    <ntfs_find name="procexp.exe" />
 
   </filesystem>
 

diff --git a/src/OrcCommand/ConfigFile_WOLFLauncher.cpp b/src/OrcCommand/ConfigFile_WOLFLauncher.cpp
@@ -177,6 +177,8 @@ HRESULT Orc::Config::Wolf::root(ConfigItem& item)
         return hr;
     if (FAILED(hr = item.AddChild(L"log", Orc::Config::Common::output, WOLFLAUNCHER_LOG)))
         return hr;
+    if (FAILED(hr = item.AddChild(L"outline", Orc::Config::Common::output, WOLFLAUNCHER_OUTLINE)))
+        return hr;
     if (FAILED(hr = item.AddAttribute(L"childdebug", WOLFLAUNCHER_CHILDDEBUG, ConfigItem::OPTION)))
         return hr;
     if (FAILED(hr = item.AddAttribute(L"command_timeout", WOLFLAUNCHER_GLOBAL_CMD_TIMEOUT, ConfigItem::OPTION)))

diff --git a/src/OrcCommand/ConfigFile_WOLFLauncher.h b/src/OrcCommand/ConfigFile_WOLFLauncher.h
@@ -67,11 +67,12 @@ constexpr auto WOLFLAUNCHER_RECIPIENT_ARCHIVE = 1L;
 constexpr auto WOLFLAUNCHER_ARCHIVE = 0L;
 constexpr auto WOLFLAUNCHER_RECIPIENT = 1L;
 constexpr auto WOLFLAUNCHER_LOG = 2L;
-constexpr auto WOLFLAUNCHER_CHILDDEBUG = 3L;
-constexpr auto WOLFLAUNCHER_GLOBAL_CMD_TIMEOUT = 4L;
-constexpr auto WOLFLAUNCHER_GLOBAL_ARCHIVE_TIMEOUT = 5L;
-constexpr auto WOLFLAUNCHER_WERDONTSHOWUI = 6L;
-constexpr auto WOLFLAUNCHER_PRIORITY = 7L;
+constexpr auto WOLFLAUNCHER_OUTLINE = 3L;
+constexpr auto WOLFLAUNCHER_CHILDDEBUG = 4L;
+constexpr auto WOLFLAUNCHER_GLOBAL_CMD_TIMEOUT = 5L;
+constexpr auto WOLFLAUNCHER_GLOBAL_ARCHIVE_TIMEOUT = 6L;
+constexpr auto WOLFLAUNCHER_WERDONTSHOWUI = 7L;
+constexpr auto WOLFLAUNCHER_PRIORITY = 8L;
 
 constexpr auto WOLFLAUNCHER_WOLF = 0L;
 

diff --git a/src/OrcCommand/FastFind_Run.cpp b/src/OrcCommand/FastFind_Run.cpp
@@ -446,12 +446,14 @@ HRESULT Main::Run()
 
     if (config.outStructured.Type & OutputSpec::Kind::StructuredFile)
     {
-        pWriterOutput = StructuredOutputWriter::GetWriter(_L_, config.outStructured);
+        auto writer = StructuredOutputWriter::GetWriter(_L_, config.outStructured, nullptr);
+        pWriterOutput = std::dynamic_pointer_cast<StructuredOutputWriter>(writer);
     }
     else if (config.outStructured.Type == OutputSpec::Kind::Directory)
     {
-        pWriterOutput = StructuredOutputWriter::GetWriter(
-            _L_, config.outStructured, L"{Name}_{SystemType}_{ComputerName}.xml", L"FastFind");
+        auto writer = StructuredOutputWriter::GetWriter(
+            _L_, config.outStructured, L"{Name}_{SystemType}_{ComputerName}.xml", L"FastFind", nullptr);
+        pWriterOutput = std::dynamic_pointer_cast<StructuredOutputWriter>(writer);
     }
 
     if (pWriterOutput != nullptr)

diff --git a/src/OrcCommand/ToolEmbed_Run.cpp b/src/OrcCommand/ToolEmbed_Run.cpp
@@ -37,7 +37,7 @@ HRESULT Main::WriteEmbedConfig(
     outputEmbedFile.Path = strOutputFile;
     outputEmbedFile.OutputEncoding = OutputSpec::Encoding::UTF8;
 
-    auto writer = StructuredOutputWriter::GetWriter(_L_, outputEmbedFile);
+    auto writer = StructuredOutputWriter::GetWriter(_L_, outputEmbedFile, nullptr);
 
     if (writer == nullptr)
     {
@@ -48,7 +48,7 @@ HRESULT Main::WriteEmbedConfig(
     writer->BeginElement(L"toolembed");
 
     writer->BeginElement(L"input");
-    writer->WriteString(strMothership.c_str());
+    writer->Write(strMothership.c_str());
     writer->EndElement(L"input");
 
     for (const auto& item : values)
@@ -57,33 +57,32 @@ HRESULT Main::WriteEmbedConfig(
         {
             case EmbeddedResource::EmbedSpec::File:
                 writer->BeginElement(L"file");
-                writer->WriteNameValuePair(L"name", item.Name.c_str());
-                writer->WriteNameValuePair(L"path", item.Value.c_str());
+                writer->WriteNamed(L"name", item.Name.c_str());
+                writer->WriteNamed(L"path", item.Value.c_str());
                 writer->EndElement(L"file");
                 break;
             case EmbeddedResource::EmbedSpec::NameValuePair:
                 writer->BeginElement(L"pair");
-                writer->WriteNameValuePair(L"name", item.Name.c_str());
-                writer->WriteNameValuePair(L"value", item.Value.c_str());
+                writer->WriteNamed(L"name", item.Name.c_str());
+                writer->WriteNamed(L"value", item.Value.c_str());
                 writer->EndElement(L"pair");
                 break;
             case EmbeddedResource::EmbedSpec::Archive:
                 writer->BeginElement(L"archive");
-                writer->WriteNameValuePair(L"name", item.Name.c_str());
-                writer->WriteNameValuePair(L"format", item.ArchiveFormat.c_str());
+                writer->WriteNamed(L"name", item.Name.c_str());
+                writer->WriteNamed(L"format", item.ArchiveFormat.c_str());
 
                 for (const auto& arch_item : item.ArchiveItems)
                 {
                     writer->BeginElement(L"file");
-                    writer->WriteNameValuePair(L"name", arch_item.Name.c_str());
-                    writer->WriteNameValuePair(L"path", arch_item.Path.c_str());
+                    writer->WriteNamed(L"name", arch_item.Name.c_str());
+                    writer->WriteNamed(L"path", arch_item.Path.c_str());
                     writer->EndElement(L"file");
                 }
 
                 writer->EndElement(L"archive");
         }
     }
-
     writer->EndElement(L"toolembed");
 
     return S_OK;

diff --git a/src/OrcCommand/UtilitiesMain.h b/src/OrcCommand/UtilitiesMain.h
@@ -674,6 +674,12 @@ class ORCLIB_API UtilitiesMain
             return E_ABORT;
         }
 
+        WSADATA wsa_data;
+        if(WSAStartup(MAKEWORD(2, 2), &wsa_data))
+        {
+            log::Error(_L_,HRESULT_FROM_WIN32(WSAGetLastError()), L"Failed to initialize WinSock 2.2\r\n");
+        }
+
         if (FAILED(hr = CoInitializeEx(0, COINIT_MULTITHREADED)))
         {
             log::Error(_L_, hr, L"Failed to initialize COM library\r\n");
@@ -824,6 +830,12 @@ class ORCLIB_API UtilitiesMain
         }
 
         _L_->Close();
+
+        if(WSACleanup())
+        {
+            log::Error(_L_, HRESULT_FROM_WIN32(WSAGetLastError()), L"Failed to cleanup WinSock 2.2\r\n");
+        }
+
         Robustness::UnInitialize(INFINITE);
         return dwErrorCount;
     }

diff --git a/src/OrcCommand/WolfExecution.h b/src/OrcCommand/WolfExecution.h
@@ -120,6 +120,7 @@ class WolfExecution
 
     FILETIME m_StartTime;
     FILETIME m_FinishTime;
+    FILETIME m_ArchiveFinishTime;
 
     std::shared_ptr<TableOutput::IWriter> m_JobStatisticsWriter;
     OutputSpec m_JobStatisticsOutput;

diff --git a/src/OrcCommand/WolfExecution_Config.cpp b/src/OrcCommand/WolfExecution_Config.cpp
@@ -545,7 +545,7 @@ HRESULT WolfExecution::SetRestrictionsFromConfig(const ConfigItem& item)
     WORD arch = 0;
     if (FAILED(hr = SystemDetails::GetArchitecture(arch)))
     {
-        log::Warning(_L_, hr, L"Failed to retrieve architecture");
+        log::Warning(_L_, hr, L"Failed to retrieve architecture\r\n");
         return hr;
     }
 
@@ -560,17 +560,19 @@ HRESULT WolfExecution::SetRestrictionsFromConfig(const ConfigItem& item)
             log::Warning(
                 _L_,
                 E_INVALIDARG,
-                L"Specified size is too big for JOB MEMORY restriction (%s)",
+                L"Specified size is too big for JOB MEMORY restriction (%s), limit ignored\r\n",
                 item[WOLFLAUNCHER_JOBMEMORY].c_str());
         }
-
-        if (!m_Restrictions.ExtendedLimits)
+        else
         {
-            m_Restrictions.ExtendedLimits.emplace();
-            ZeroMemory(&m_Restrictions.ExtendedLimits.value(), sizeof(JOBOBJECT_EXTENDED_LIMIT_INFORMATION));
+            if (!m_Restrictions.ExtendedLimits)
+            {
+                m_Restrictions.ExtendedLimits.emplace();
+                ZeroMemory(&m_Restrictions.ExtendedLimits.value(), sizeof(JOBOBJECT_EXTENDED_LIMIT_INFORMATION));
+            }
+            m_Restrictions.ExtendedLimits->JobMemoryLimit = msl::utilities::SafeInt<SIZE_T>(li.QuadPart);
+            m_Restrictions.ExtendedLimits->BasicLimitInformation.LimitFlags |= JOB_OBJECT_LIMIT_JOB_MEMORY;
         }
-        m_Restrictions.ExtendedLimits->JobMemoryLimit = msl::utilities::SafeInt<SIZE_T>(li.QuadPart);
-        m_Restrictions.ExtendedLimits->BasicLimitInformation.LimitFlags |= JOB_OBJECT_LIMIT_JOB_MEMORY;
     }
 
     if (item[WOLFLAUNCHER_PROCESSMEMORY])
@@ -584,16 +586,19 @@ HRESULT WolfExecution::SetRestrictionsFromConfig(const ConfigItem& item)
             log::Warning(
                 _L_,
                 E_INVALIDARG,
-                L"Specified size is too big for PROCESS memory restriction (%s)",
+                L"Specified size is too big for PROCESS memory restriction (%s), limit ignored\r\n",
                 item[WOLFLAUNCHER_PROCESSMEMORY].c_str());
         }
-        if (!m_Restrictions.ExtendedLimits)
+        else
         {
-            m_Restrictions.ExtendedLimits.emplace();
-            ZeroMemory(&m_Restrictions.ExtendedLimits.value(), sizeof(JOBOBJECT_EXTENDED_LIMIT_INFORMATION));
+            if (!m_Restrictions.ExtendedLimits)
+            {
+                m_Restrictions.ExtendedLimits.emplace();
+                ZeroMemory(&m_Restrictions.ExtendedLimits.value(), sizeof(JOBOBJECT_EXTENDED_LIMIT_INFORMATION));
+            }
+            m_Restrictions.ExtendedLimits->ProcessMemoryLimit = msl::utilities::SafeInt<SIZE_T>(li.QuadPart);
+            m_Restrictions.ExtendedLimits->BasicLimitInformation.LimitFlags |= JOB_OBJECT_LIMIT_PROCESS_MEMORY;
         }
-        m_Restrictions.ExtendedLimits->ProcessMemoryLimit = msl::utilities::SafeInt<SIZE_T>(li.QuadPart);
-        m_Restrictions.ExtendedLimits->BasicLimitInformation.LimitFlags |= JOB_OBJECT_LIMIT_PROCESS_MEMORY;
     }
 
     if (item[WOLFLAUNCHER_ELAPSEDTIME])