Report Program File contents as packages #238

Signed-off-by: Jono Yang <[email protected]>

scanpipe/pipelines/windows_docker.py

@@ -41,11 +41,12 @@ def steps(cls):
             cls.collect_and_create_system_packages,
             cls.tag_known_software_packages,
             cls.tag_uninteresting_codebase_resources,
-            # cls.tag_empty_files,
-            # cls.scan_for_application_packages,
-            # cls.scan_for_files,
-            # cls.analyze_scanned_files,
-            # cls.tag_not_analyzed_codebase_resources,
+            cls.tag_program_files_dirs_as_packages,
+            cls.tag_empty_files,
+            cls.scan_for_application_packages,
+            cls.scan_for_files,
+            cls.analyze_scanned_files,
+            cls.tag_not_analyzed_codebase_resources,
         )
 
     def tag_known_software_packages(self):
@@ -61,3 +62,10 @@ def tag_uninteresting_codebase_resources(self):
         """
         docker.tag_whiteout_codebase_resources(self.project)
         windows.tag_uninteresting_windows_codebase_resources(self.project)
+
+    def tag_program_files_dirs_as_packages(self):
+        """
+        Report the immediate subdirectories of `Program Files` and `Program
+        Files (x86)` as packages
+        """
+        windows.tag_program_files(self.project)

scanpipe/pipes/windows.py

@@ -77,15 +77,15 @@ def tag_installed_package_files(project, root_dir_pattern, package):
     CodebaseResource and set the status
     """
     qs = project.codebaseresources.no_status()
-    installed_package_files = qs.filter(rootfs_path__regex=root_dir_pattern)
+    installed_package_files = qs.filter(rootfs_path__startswith=root_dir_pattern)
     # If we find files whose names start with `root_dir_pattern`, we consider
     # these files to be part of the Package `package` and tag these files as
     # such
     if installed_package_files:
         created_package = pipes.update_or_create_package(project=project, package_data=package.to_dict())
         for installed_package_file in installed_package_files:
             installed_package_file.discovered_packages.add(created_package)
-            installed_package_file.status = "system-package"
+            installed_package_file.status = "installed-package"
             installed_package_file.save()
         created_package.save()
 
@@ -171,3 +171,45 @@ def tag_known_software(project):
             root_dir_pattern=openjdk_path,
             package=openjdk_package
         )
+
+
+PROGRAM_FILES_DIRS_TO_IGNORE = (
+    "Common Files",
+    "Common_Files",
+    "common_files",
+    "Microsoft",
+)
+
+
+def tag_program_files(project):
+    """
+    Report all subdirectories of Program Files and Program Files (x86) as
+    packages
+    """
+    qs = project.codebaseresources.no_status()
+    # Get all files from Program_Files and Program_Files_(x86)
+    program_files_one_directory_below_pattern = r"(/Files/Program_Files(_\(x86\))?/([^/]+))"
+    program_files_one_directory_below_pattern_compiled = re.compile(program_files_one_directory_below_pattern)
+    program_files_dirname_by_path = {}
+    lookup = Q(rootfs_path__startswith="/Files/Program_Files") | Q(rootfs_path__startswith="/Files/Program_Files_(x86)")
+    for program_file in qs.filter(lookup):
+        _, program_files_subdir, _, dirname, _ = re.split(
+            program_files_one_directory_below_pattern_compiled,
+            program_file.rootfs_path
+        )
+        if (program_files_subdir in program_files_dirname_by_path
+                or dirname in PROGRAM_FILES_DIRS_TO_IGNORE):
+            continue
+        program_files_dirname_by_path[program_files_subdir] = dirname
+
+    for program_root_dir, program_root_dir_name in program_files_dirname_by_path.items():
+        package = Package(
+            type="windows-program",
+            name=program_root_dir_name,
+            version="nv",
+        )
+        tag_installed_package_files(
+            project=project,
+            root_dir_pattern=program_root_dir,
+            package=package
+        )

scanpipe/tests/test_pipes.py

@@ -767,6 +767,52 @@ def test_scanpipe_pipes_windows_tag_known_software(self):
         self.assertEqual("system-package", resource3.status)
         self.assertEqual("system-package", resource4.status)
 
+    def test_scanpipe_pipes_windows_tag_program_files(self):
+        p1 = Project.objects.create(name="Analysis")
+        resource1 = CodebaseResource.objects.create(
+            project=p1,
+            path="root/Files/Program_Files_(x86)/Microsoft/example.exe",
+            rootfs_path="/Files/Program_Files_(x86)/Microsoft/example.exe"
+        )
+        resource2 = CodebaseResource.objects.create(
+            project=p1,
+            path="root/Files/Program_Files/Microsoft/example.exe",
+            rootfs_path="/Files/Program_Files/Microsoft/example.exe"
+        )
+        resource3 = CodebaseResource.objects.create(
+            project=p1,
+            path="root/Files/Program_Files_(x86)/7Zip/7z.exe",
+            rootfs_path="/Files/Program_Files_(x86)/7Zip/7z.exe"
+        )
+        resource4 = CodebaseResource.objects.create(
+            project=p1,
+            path="root/Files/Program_Files/7Zip/7z.exe",
+            rootfs_path="/Files/Program_Files/7Zip/7z.exe"
+        )
+        resource5 = CodebaseResource.objects.create(
+            project=p1,
+            path="root/Files/Program_Files_(x86)/common_files/sample.dat",
+            rootfs_path="/Files/Program_Files_(x86)/common_files/sample.dat"
+        )
+        resource6 = CodebaseResource.objects.create(
+            project=p1,
+            path="root/Files/Program_Files/common_files/sample.dat",
+            rootfs_path="/Files/Program_Files/common_files/sample.dat"
+        )
+        windows.tag_program_files(p1)
+        resource1.refresh_from_db()
+        resource2.refresh_from_db()
+        resource3.refresh_from_db()
+        resource4.refresh_from_db()
+        resource5.refresh_from_db()
+        resource6.refresh_from_db()
+        self.assertEqual("", resource1.status)
+        self.assertEqual("", resource2.status)
+        self.assertEqual("installed-package", resource3.status)
+        self.assertEqual("installed-package", resource4.status)
+        self.assertEqual("", resource5.status)
+        self.assertEqual("", resource6.status)
+
 
 class ScanPipePipesTransactionTest(TransactionTestCase):
     """