diff --git a/scanpipe/pipes/rootfs.py b/scanpipe/pipes/rootfs.py index 4d4054572..f38bef46b 100644 --- a/scanpipe/pipes/rootfs.py +++ b/scanpipe/pipes/rootfs.py @@ -357,4 +357,4 @@ def tag_ignorable_codebase_resources(project): lookups |= Q(rootfs_path__icontains=pattern) lookups |= Q(rootfs_path__iregex=translated_pattern) qs = project.codebaseresources.no_status() - qs.filter(lookups).update(status='ignored-default-ignores') + qs.filter(lookups).update(status="ignored-default-ignores") diff --git a/scanpipe/pipes/windows.py b/scanpipe/pipes/windows.py index fa78d9ab7..efc400d76 100644 --- a/scanpipe/pipes/windows.py +++ b/scanpipe/pipes/windows.py @@ -24,11 +24,11 @@ from django.db.models import Q -from scanpipe import pipes - from packagedcode import win_reg from packagedcode.models import Package +from scanpipe import pipes + def package_getter(root_dir, **kwargs): """ @@ -44,30 +44,30 @@ def tag_uninteresting_windows_codebase_resources(project): Tag known uninteresting files as uninteresting """ uninteresting_files = ( - 'DefaultUser_Delta', - 'Sam_Delta', - 'Security_Delta', - 'Software_Delta', - 'System_Delta', - 'NTUSER.DAT', - 'desktop.ini', - 'BBI', - 'BCD-Template', - 'DEFAULT', - 'DRIVERS', - 'ELAM', - 'SAM', - 'SECURITY', - 'SOFTWARE', - 'SYSTEM', + "DefaultUser_Delta", + "Sam_Delta", + "Security_Delta", + "Software_Delta", + "System_Delta", + "NTUSER.DAT", + "desktop.ini", + "BBI", + "BCD-Template", + "DEFAULT", + "DRIVERS", + "ELAM", + "SAM", + "SECURITY", + "SOFTWARE", + "SYSTEM", ) uninteresting_file_extensions = ( - '.lnk', - '.library-ms', - '.LOG', - '.inf_loc', - '.NLS', + ".lnk", + ".library-ms", + ".LOG", + ".inf_loc", + ".NLS", ) lookups = Q() @@ -92,7 +92,9 @@ def tag_installed_package_files(project, root_dir_pattern, package): # these files to be part of the Package `package` and tag these files as # such if installed_package_files: - created_package = pipes.update_or_create_package(project=project, package_data=package.to_dict()) + created_package = pipes.update_or_create_package( + project=project, package_data=package.to_dict() + ) for installed_package_file in installed_package_files: installed_package_file.discovered_packages.add(created_package) installed_package_file.status = "installed-package" @@ -114,57 +116,63 @@ def tag_known_software(project): """ qs = project.codebaseresources.no_status() python_root_directory_name_pattern = r"/Files/Python(\d*)" - python_root_directory_name_pattern_compiled = re.compile(python_root_directory_name_pattern) + python_root_directory_name_pattern_compiled = re.compile( + python_root_directory_name_pattern + ) python_paths_by_versions = {} - for python_codebase_resource in qs.filter(rootfs_path__regex=python_root_directory_name_pattern): + for python_codebase_resource in qs.filter( + rootfs_path__regex=python_root_directory_name_pattern + ): _, version, _ = re.split( python_root_directory_name_pattern_compiled, - python_codebase_resource.rootfs_path + python_codebase_resource.rootfs_path, ) if not version: - version = 'nv' + version = "nv" if version in python_paths_by_versions: continue - if version != 'nv': - version_with_dots = '.'.join(digit for digit in version) - python_paths_by_versions[version_with_dots] = f'/Files/Python{version}' + if version != "nv": + version_with_dots = ".".join(digit for digit in version) + python_paths_by_versions[version_with_dots] = f"/Files/Python{version}" else: - python_paths_by_versions[version] = '/Files/Python' + python_paths_by_versions[version] = "/Files/Python" for python_version, python_path in python_paths_by_versions.items(): python_package = Package( - type="windows-program", - name="Python", - version=python_version, - license_expression="python", - copyright="Copyright (c) Python Software Foundation", - homepage_url="https://www.python.org/" + type="windows-program", + name="Python", + version=python_version, + license_expression="python", + copyright="Copyright (c) Python Software Foundation", + homepage_url="https://www.python.org/", ) tag_installed_package_files( - project=project, - root_dir_pattern=python_path, - package=python_package + project=project, root_dir_pattern=python_path, package=python_package ) qs = project.codebaseresources.no_status() openjdk_root_directory_name_pattern = r"/Files/(open)?jdk(-((\d*)(\.\d+)*))*" - openjdk_root_directory_name_pattern_compiled = re.compile(openjdk_root_directory_name_pattern) + openjdk_root_directory_name_pattern_compiled = re.compile( + openjdk_root_directory_name_pattern + ) openjdk_paths_by_versions = {} - for openjdk_codebase_resource in qs.filter(rootfs_path__regex=openjdk_root_directory_name_pattern): + for openjdk_codebase_resource in qs.filter( + rootfs_path__regex=openjdk_root_directory_name_pattern + ): _, open_prefix, _, openjdk_version, _, _, _ = re.split( openjdk_root_directory_name_pattern_compiled, - openjdk_codebase_resource.rootfs_path + openjdk_codebase_resource.rootfs_path, ) if not openjdk_version: - openjdk_version = 'nv' + openjdk_version = "nv" if not open_prefix: - open_prefix = '' + open_prefix = "" if openjdk_version in openjdk_paths_by_versions: continue - if openjdk_version != 'nv': - openjdk_path = f'/Files/{open_prefix}jdk-{openjdk_version}' + if openjdk_version != "nv": + openjdk_path = f"/Files/{open_prefix}jdk-{openjdk_version}" else: - openjdk_path = f'/Files/{open_prefix}jdk' + openjdk_path = f"/Files/{open_prefix}jdk" openjdk_paths_by_versions[openjdk_version] = openjdk_path for openjdk_version, openjdk_path in openjdk_paths_by_versions.items(): @@ -177,9 +185,7 @@ def tag_known_software(project): homepage_url="http://openjdk.java.net/", ) tag_installed_package_files( - project=project, - root_dir_pattern=openjdk_path, - package=openjdk_package + project=project, root_dir_pattern=openjdk_path, package=openjdk_package ) @@ -196,28 +202,36 @@ def tag_program_files(project): """ qs = project.codebaseresources.no_status() # Get all files from Program_Files and Program_Files_(x86) - program_files_one_directory_below_pattern = r"(/Files/Program Files( \(x86\))?/([^/]+))" - program_files_one_directory_below_pattern_compiled = re.compile(program_files_one_directory_below_pattern) + program_files_one_directory_below_pattern = ( + r"(/Files/Program Files( \(x86\))?/([^/]+))" + ) + program_files_one_directory_below_pattern_compiled = re.compile( + program_files_one_directory_below_pattern + ) program_files_dirname_by_path = {} - lookup = Q(rootfs_path__startswith="/Files/Program Files") | Q(rootfs_path__startswith="/Files/Program Files (x86)") + lookup = Q(rootfs_path__startswith="/Files/Program Files") | Q( + rootfs_path__startswith="/Files/Program Files (x86)" + ) for program_file in qs.filter(lookup): _, program_files_subdir, _, dirname, _ = re.split( - program_files_one_directory_below_pattern_compiled, - program_file.rootfs_path + program_files_one_directory_below_pattern_compiled, program_file.rootfs_path ) - if (program_files_subdir in program_files_dirname_by_path - or dirname.lower() in map(str.lower, PROGRAM_FILES_DIRS_TO_IGNORE)): + if ( + program_files_subdir in program_files_dirname_by_path + or dirname.lower() in map(str.lower, PROGRAM_FILES_DIRS_TO_IGNORE) + ): continue program_files_dirname_by_path[program_files_subdir] = dirname - for program_root_dir, program_root_dir_name in program_files_dirname_by_path.items(): + for ( + program_root_dir, + program_root_dir_name, + ) in program_files_dirname_by_path.items(): package = Package( type="windows-program", name=program_root_dir_name, version="nv", ) tag_installed_package_files( - project=project, - root_dir_pattern=program_root_dir, - package=package + project=project, root_dir_pattern=program_root_dir, package=package ) diff --git a/scanpipe/tests/test_pipes.py b/scanpipe/tests/test_pipes.py index 55e5bfdcc..1a32af2cd 100644 --- a/scanpipe/tests/test_pipes.py +++ b/scanpipe/tests/test_pipes.py @@ -763,24 +763,24 @@ def test_scanpipe_pipes_windows_tag_uninteresting_windows_codebase_resources(sel project=p1, path="root/Files/example.lnk", rootfs_path="/Files/example.lnk", - extension=".lnk" + extension=".lnk", ) resource2 = CodebaseResource.objects.create( project=p1, path="root/Hives/Software_Delta", - rootfs_path="/Hives/Software_Delta" + rootfs_path="/Hives/Software_Delta", ) resource3 = CodebaseResource.objects.create( project=p1, path="root/Files/example.dat", rootfs_path="/Files/example.dat", - extension=".dat" + extension=".dat", ) resource4 = CodebaseResource.objects.create( project=p1, path="root/Files/User/Test/foo.dat", rootfs_path="/Files/User/Test/foo.dat", - extension=".dat" + extension=".dat", ) windows.tag_uninteresting_windows_codebase_resources(p1) @@ -798,22 +798,22 @@ def test_scanpipe_pipes_windows_tag_known_software(self): resource1 = CodebaseResource.objects.create( project=p1, path="root/Files/Python/py.exe", - rootfs_path="/Files/Python/py.exe" + rootfs_path="/Files/Python/py.exe", ) resource2 = CodebaseResource.objects.create( project=p1, path="root/Files/Python27/python2.exe", - rootfs_path="/Files/Python27/python2.exe" + rootfs_path="/Files/Python27/python2.exe", ) resource3 = CodebaseResource.objects.create( project=p1, path="root/Files/Python3/python3.exe", - rootfs_path="/Files/Python3/python3.exe" + rootfs_path="/Files/Python3/python3.exe", ) resource4 = CodebaseResource.objects.create( project=p1, path="root/Files/Python39/python3.9", - rootfs_path="/Files/Python39/python3.9.exe" + rootfs_path="/Files/Python39/python3.9.exe", ) windows.tag_known_software(p1) @@ -831,32 +831,32 @@ def test_scanpipe_pipes_windows_tag_program_files(self): resource1 = CodebaseResource.objects.create( project=p1, path="root/Files/Program Files (x86)/Microsoft/example.exe", - rootfs_path="/Files/Program Files (x86)/Microsoft/example.exe" + rootfs_path="/Files/Program Files (x86)/Microsoft/example.exe", ) resource2 = CodebaseResource.objects.create( project=p1, path="root/Files/Program Files/Microsoft/example.exe", - rootfs_path="/Files/Program Files/Microsoft/example.exe" + rootfs_path="/Files/Program Files/Microsoft/example.exe", ) resource3 = CodebaseResource.objects.create( project=p1, path="root/Files/Program Files (x86)/7Zip/7z.exe", - rootfs_path="/Files/Program Files (x86)/7Zip/7z.exe" + rootfs_path="/Files/Program Files (x86)/7Zip/7z.exe", ) resource4 = CodebaseResource.objects.create( project=p1, path="root/Files/Program Files/7Zip/7z.exe", - rootfs_path="/Files/Program Files/7Zip/7z.exe" + rootfs_path="/Files/Program Files/7Zip/7z.exe", ) resource5 = CodebaseResource.objects.create( project=p1, path="root/Files/Program Files (x86)/common files/sample.dat", - rootfs_path="/Files/Program Files (x86)/common files/sample.dat" + rootfs_path="/Files/Program Files (x86)/common files/sample.dat", ) resource6 = CodebaseResource.objects.create( project=p1, path="root/Files/Program Files/common files/sample.dat", - rootfs_path="/Files/Program Files/common files/sample.dat" + rootfs_path="/Files/Program Files/common files/sample.dat", ) windows.tag_program_files(p1) resource1.refresh_from_db() @@ -877,27 +877,23 @@ def test_scanpipe_pipes_rootfs_tag_ignorable_codebase_resources(self): resource1 = CodebaseResource.objects.create( project=p1, path="root/user/cmake_install.cmake", - rootfs_path="/user/cmake_install.cmake" + rootfs_path="/user/cmake_install.cmake", ) resource2 = CodebaseResource.objects.create( - project=p1, - path="root/user/example.pot", - rootfs_path="/user/example.pot" + project=p1, path="root/user/example.pot", rootfs_path="/user/example.pot" ) resource3 = CodebaseResource.objects.create( project=p1, path="root/user/__pycache__/foo.pyc", - rootfs_path="/user/__pycache__/foo.pyc" + rootfs_path="/user/__pycache__/foo.pyc", ) resource4 = CodebaseResource.objects.create( - project=p1, - path="root/user/foo.css.map", - rootfs_path="/user/foo.css.map" + project=p1, path="root/user/foo.css.map", rootfs_path="/user/foo.css.map" ) resource5 = CodebaseResource.objects.create( project=p1, path="root/user/should-not-be-ignored.txt", - rootfs_path="/user/should-not-be-ignored.txt" + rootfs_path="/user/should-not-be-ignored.txt", ) rootfs.tag_ignorable_codebase_resources(p1) resource1.refresh_from_db()