Skip to content

Commit

Permalink
Update formatting #238
Browse files Browse the repository at this point in the history 
Signed-off-by: Jono Yang <[email protected]>
  • Loading branch information
@JonoYang
JonoYang committed Jul 28, 2021
1 parent 439a7fe commit 0dec201
Show file tree
Hide file tree
Showing 3 changed files with 97 additions and 87 deletions.
2 changes: 1 addition & 1 deletion scanpipe/pipes/rootfs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -357,4 +357,4 @@ def tag_ignorable_codebase_resources(project):
lookups |= Q(rootfs_path__icontains=pattern)
lookups |= Q(rootfs_path__iregex=translated_pattern)
qs = project.codebaseresources.no_status()
qs.filter(lookups).update(status='ignored-default-ignores')
qs.filter(lookups).update(status="ignored-default-ignores")
140 changes: 77 additions & 63 deletions scanpipe/pipes/windows.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,11 @@

from django.db.models import Q

from scanpipe import pipes

from packagedcode import win_reg
from packagedcode.models import Package

from scanpipe import pipes


def package_getter(root_dir, **kwargs):
"""
Expand All @@ -44,30 +44,30 @@ def tag_uninteresting_windows_codebase_resources(project):
Tag known uninteresting files as uninteresting
"""
uninteresting_files = (
'DefaultUser_Delta',
'Sam_Delta',
'Security_Delta',
'Software_Delta',
'System_Delta',
'NTUSER.DAT',
'desktop.ini',
'BBI',
'BCD-Template',
'DEFAULT',
'DRIVERS',
'ELAM',
'SAM',
'SECURITY',
'SOFTWARE',
'SYSTEM',
"DefaultUser_Delta",
"Sam_Delta",
"Security_Delta",
"Software_Delta",
"System_Delta",
"NTUSER.DAT",
"desktop.ini",
"BBI",
"BCD-Template",
"DEFAULT",
"DRIVERS",
"ELAM",
"SAM",
"SECURITY",
"SOFTWARE",
"SYSTEM",
)

uninteresting_file_extensions = (
'.lnk',
'.library-ms',
'.LOG',
'.inf_loc',
'.NLS',
".lnk",
".library-ms",
".LOG",
".inf_loc",
".NLS",
)

lookups = Q()
Expand All @@ -92,7 +92,9 @@ def tag_installed_package_files(project, root_dir_pattern, package):
# these files to be part of the Package `package` and tag these files as
# such
if installed_package_files:
created_package = pipes.update_or_create_package(project=project, package_data=package.to_dict())
created_package = pipes.update_or_create_package(
project=project, package_data=package.to_dict()
)
for installed_package_file in installed_package_files:
installed_package_file.discovered_packages.add(created_package)
installed_package_file.status = "installed-package"
Expand All @@ -114,57 +116,63 @@ def tag_known_software(project):
"""
qs = project.codebaseresources.no_status()
python_root_directory_name_pattern = r"/Files/Python(\d*)"
python_root_directory_name_pattern_compiled = re.compile(python_root_directory_name_pattern)
python_root_directory_name_pattern_compiled = re.compile(
python_root_directory_name_pattern
)
python_paths_by_versions = {}
for python_codebase_resource in qs.filter(rootfs_path__regex=python_root_directory_name_pattern):
for python_codebase_resource in qs.filter(
rootfs_path__regex=python_root_directory_name_pattern
):
_, version, _ = re.split(
python_root_directory_name_pattern_compiled,
python_codebase_resource.rootfs_path
python_codebase_resource.rootfs_path,
)
if not version:
version = 'nv'
version = "nv"
if version in python_paths_by_versions:
continue
if version != 'nv':
version_with_dots = '.'.join(digit for digit in version)
python_paths_by_versions[version_with_dots] = f'/Files/Python{version}'
if version != "nv":
version_with_dots = ".".join(digit for digit in version)
python_paths_by_versions[version_with_dots] = f"/Files/Python{version}"
else:
python_paths_by_versions[version] = '/Files/Python'
python_paths_by_versions[version] = "/Files/Python"

for python_version, python_path in python_paths_by_versions.items():
python_package = Package(
type="windows-program",
name="Python",
version=python_version,
license_expression="python",
copyright="Copyright (c) Python Software Foundation",
homepage_url="https://www.python.org/"
type="windows-program",
name="Python",
version=python_version,
license_expression="python",
copyright="Copyright (c) Python Software Foundation",
homepage_url="https://www.python.org/",
)
tag_installed_package_files(
project=project,
root_dir_pattern=python_path,
package=python_package
project=project, root_dir_pattern=python_path, package=python_package
)

qs = project.codebaseresources.no_status()
openjdk_root_directory_name_pattern = r"/Files/(open)?jdk(-((\d*)(\.\d+)*))*"
openjdk_root_directory_name_pattern_compiled = re.compile(openjdk_root_directory_name_pattern)
openjdk_root_directory_name_pattern_compiled = re.compile(
openjdk_root_directory_name_pattern
)
openjdk_paths_by_versions = {}
for openjdk_codebase_resource in qs.filter(rootfs_path__regex=openjdk_root_directory_name_pattern):
for openjdk_codebase_resource in qs.filter(
rootfs_path__regex=openjdk_root_directory_name_pattern
):
_, open_prefix, _, openjdk_version, _, _, _ = re.split(
openjdk_root_directory_name_pattern_compiled,
openjdk_codebase_resource.rootfs_path
openjdk_codebase_resource.rootfs_path,
)
if not openjdk_version:
openjdk_version = 'nv'
openjdk_version = "nv"
if not open_prefix:
open_prefix = ''
open_prefix = ""
if openjdk_version in openjdk_paths_by_versions:
continue
if openjdk_version != 'nv':
openjdk_path = f'/Files/{open_prefix}jdk-{openjdk_version}'
if openjdk_version != "nv":
openjdk_path = f"/Files/{open_prefix}jdk-{openjdk_version}"
else:
openjdk_path = f'/Files/{open_prefix}jdk'
openjdk_path = f"/Files/{open_prefix}jdk"
openjdk_paths_by_versions[openjdk_version] = openjdk_path

for openjdk_version, openjdk_path in openjdk_paths_by_versions.items():
Expand All @@ -177,9 +185,7 @@ def tag_known_software(project):
homepage_url="http://openjdk.java.net/",
)
tag_installed_package_files(
project=project,
root_dir_pattern=openjdk_path,
package=openjdk_package
project=project, root_dir_pattern=openjdk_path, package=openjdk_package
)


Expand All @@ -196,28 +202,36 @@ def tag_program_files(project):
"""
qs = project.codebaseresources.no_status()
# Get all files from Program_Files and Program_Files_(x86)
program_files_one_directory_below_pattern = r"(/Files/Program Files( \(x86\))?/([^/]+))"
program_files_one_directory_below_pattern_compiled = re.compile(program_files_one_directory_below_pattern)
program_files_one_directory_below_pattern = (
r"(/Files/Program Files( \(x86\))?/([^/]+))"
)
program_files_one_directory_below_pattern_compiled = re.compile(
program_files_one_directory_below_pattern
)
program_files_dirname_by_path = {}
lookup = Q(rootfs_path__startswith="/Files/Program Files") | Q(rootfs_path__startswith="/Files/Program Files (x86)")
lookup = Q(rootfs_path__startswith="/Files/Program Files") | Q(
rootfs_path__startswith="/Files/Program Files (x86)"
)
for program_file in qs.filter(lookup):
_, program_files_subdir, _, dirname, _ = re.split(
program_files_one_directory_below_pattern_compiled,
program_file.rootfs_path
program_files_one_directory_below_pattern_compiled, program_file.rootfs_path
)
if (program_files_subdir in program_files_dirname_by_path
or dirname.lower() in map(str.lower, PROGRAM_FILES_DIRS_TO_IGNORE)):
if (
program_files_subdir in program_files_dirname_by_path
or dirname.lower() in map(str.lower, PROGRAM_FILES_DIRS_TO_IGNORE)
):
continue
program_files_dirname_by_path[program_files_subdir] = dirname

for program_root_dir, program_root_dir_name in program_files_dirname_by_path.items():
for (
program_root_dir,
program_root_dir_name,
) in program_files_dirname_by_path.items():
package = Package(
type="windows-program",
name=program_root_dir_name,
version="nv",
)
tag_installed_package_files(
project=project,
root_dir_pattern=program_root_dir,
package=package
project=project, root_dir_pattern=program_root_dir, package=package
)
42 changes: 19 additions & 23 deletions scanpipe/tests/test_pipes.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -763,24 +763,24 @@ def test_scanpipe_pipes_windows_tag_uninteresting_windows_codebase_resources(sel
project=p1,
path="root/Files/example.lnk",
rootfs_path="/Files/example.lnk",
extension=".lnk"
extension=".lnk",
)
resource2 = CodebaseResource.objects.create(
project=p1,
path="root/Hives/Software_Delta",
rootfs_path="/Hives/Software_Delta"
rootfs_path="/Hives/Software_Delta",
)
resource3 = CodebaseResource.objects.create(
project=p1,
path="root/Files/example.dat",
rootfs_path="/Files/example.dat",
extension=".dat"
extension=".dat",
)
resource4 = CodebaseResource.objects.create(
project=p1,
path="root/Files/User/Test/foo.dat",
rootfs_path="/Files/User/Test/foo.dat",
extension=".dat"
extension=".dat",
)

windows.tag_uninteresting_windows_codebase_resources(p1)
Expand All @@ -798,22 +798,22 @@ def test_scanpipe_pipes_windows_tag_known_software(self):
resource1 = CodebaseResource.objects.create(
project=p1,
path="root/Files/Python/py.exe",
rootfs_path="/Files/Python/py.exe"
rootfs_path="/Files/Python/py.exe",
)
resource2 = CodebaseResource.objects.create(
project=p1,
path="root/Files/Python27/python2.exe",
rootfs_path="/Files/Python27/python2.exe"
rootfs_path="/Files/Python27/python2.exe",
)
resource3 = CodebaseResource.objects.create(
project=p1,
path="root/Files/Python3/python3.exe",
rootfs_path="/Files/Python3/python3.exe"
rootfs_path="/Files/Python3/python3.exe",
)
resource4 = CodebaseResource.objects.create(
project=p1,
path="root/Files/Python39/python3.9",
rootfs_path="/Files/Python39/python3.9.exe"
rootfs_path="/Files/Python39/python3.9.exe",
)

windows.tag_known_software(p1)
Expand All @@ -831,32 +831,32 @@ def test_scanpipe_pipes_windows_tag_program_files(self):
resource1 = CodebaseResource.objects.create(
project=p1,
path="root/Files/Program Files (x86)/Microsoft/example.exe",
rootfs_path="/Files/Program Files (x86)/Microsoft/example.exe"
rootfs_path="/Files/Program Files (x86)/Microsoft/example.exe",
)
resource2 = CodebaseResource.objects.create(
project=p1,
path="root/Files/Program Files/Microsoft/example.exe",
rootfs_path="/Files/Program Files/Microsoft/example.exe"
rootfs_path="/Files/Program Files/Microsoft/example.exe",
)
resource3 = CodebaseResource.objects.create(
project=p1,
path="root/Files/Program Files (x86)/7Zip/7z.exe",
rootfs_path="/Files/Program Files (x86)/7Zip/7z.exe"
rootfs_path="/Files/Program Files (x86)/7Zip/7z.exe",
)
resource4 = CodebaseResource.objects.create(
project=p1,
path="root/Files/Program Files/7Zip/7z.exe",
rootfs_path="/Files/Program Files/7Zip/7z.exe"
rootfs_path="/Files/Program Files/7Zip/7z.exe",
)
resource5 = CodebaseResource.objects.create(
project=p1,
path="root/Files/Program Files (x86)/common files/sample.dat",
rootfs_path="/Files/Program Files (x86)/common files/sample.dat"
rootfs_path="/Files/Program Files (x86)/common files/sample.dat",
)
resource6 = CodebaseResource.objects.create(
project=p1,
path="root/Files/Program Files/common files/sample.dat",
rootfs_path="/Files/Program Files/common files/sample.dat"
rootfs_path="/Files/Program Files/common files/sample.dat",
)
windows.tag_program_files(p1)
resource1.refresh_from_db()
Expand All @@ -877,27 +877,23 @@ def test_scanpipe_pipes_rootfs_tag_ignorable_codebase_resources(self):
resource1 = CodebaseResource.objects.create(
project=p1,
path="root/user/cmake_install.cmake",
rootfs_path="/user/cmake_install.cmake"
rootfs_path="/user/cmake_install.cmake",
)
resource2 = CodebaseResource.objects.create(
project=p1,
path="root/user/example.pot",
rootfs_path="/user/example.pot"
project=p1, path="root/user/example.pot", rootfs_path="/user/example.pot"
)
resource3 = CodebaseResource.objects.create(
project=p1,
path="root/user/__pycache__/foo.pyc",
rootfs_path="/user/__pycache__/foo.pyc"
rootfs_path="/user/__pycache__/foo.pyc",
)
resource4 = CodebaseResource.objects.create(
project=p1,
path="root/user/foo.css.map",
rootfs_path="/user/foo.css.map"
project=p1, path="root/user/foo.css.map", rootfs_path="/user/foo.css.map"
)
resource5 = CodebaseResource.objects.create(
project=p1,
path="root/user/should-not-be-ignored.txt",
rootfs_path="/user/should-not-be-ignored.txt"
rootfs_path="/user/should-not-be-ignored.txt",
)
rootfs.tag_ignorable_codebase_resources(p1)
resource1.refresh_from_db()
Expand Down

0 comments on commit 0dec201

Please sign in to comment.