Replace binwalk generic carver functionality with unblob

.github/workflows/build-unblob-image.yml

@@ -0,0 +1,27 @@
+name: Publish Docker image based on unblob-2 branch
+
+on:
+  push:
+    branches: ['unblob-2']
+  workflow_dispatch:
+
+jobs:
+  build-and-publish-image:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_HUB_USER }}
+          password: ${{ secrets.DOCKER_HUB_TOKEN }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          push: true
+          tags: fkiecad/fact_extractor:unblob

.gitignore

@@ -9,3 +9,4 @@
 __pycache__
 bin/
 install.log
+unblob

fact_extractor/install/unpacker.py

@@ -16,8 +16,8 @@
     apt_install_packages,
     apt_remove_packages,
     install_github_project,
-    pip_install_packages,
     load_requirements_file,
+    pip_install_packages,
 )
 
 BIN_DIR = Path(__file__).parent.parent / 'bin'
@@ -115,11 +115,13 @@
             'gzip',
             'lhasa',
             'libchm-dev',
+            'liblz4-tool',
             'lrzip',
             'lzip',
             'lzop',
             'ncompress',
             'nomarch',
+            'p7zip-full',
             'rpm2cpio',
             'rzip',
             'sharutils',
@@ -130,6 +132,7 @@
             'unrar',
             'xdms',
             'zpaq',
+            'zstd',
             # Freetz
             'autoconf',
             'automake',
@@ -199,13 +202,21 @@
         ),
     ]
 
+
+def check_mod_kit_installed() -> bool:
+    return all((Path(__file__).parent.parent / 'bin' / tool).exists() for tool in ['tpl-tool', 'untrx', 'unyaffs2'])
+
+
 def install_dependencies(dependencies):
     apt = dependencies.get('apt', [])
     github = dependencies.get('github', [])
     apt_install_packages(*apt)
     pip_install_packages(*load_requirements_file(PIP_DEPENDENCY_FILE))
     for repo in github:
-        install_github_project(*repo)
+        if repo[0].endswith('firmware-mod-kit') and check_mod_kit_installed():
+            logging.info('Skipping firmware-mod-kit since it is already installed')
+        else:
+            install_github_project(*repo)
 
 
 def main(distribution):
@@ -237,73 +248,85 @@ def _edit_sudoers():
     logging.info('add rules to sudo...')
     username = getuser()
     sudoers_content = '\n'.join(
-        (
-            f'{username}\tALL=NOPASSWD: {command}'
-            for command in (
-                '/sbin/kpartx',
-                '/sbin/losetup',
-                '/bin/mount',
-                '/bin/umount',
-                '/bin/mknod',
-                '/usr/bin/sasquatch',
-                '/bin/rm',
-                '/bin/cp',
-                '/bin/dd',
-                '/bin/chown',
-            )
+        f'{username}\tALL=NOPASSWD: {command}'
+        for command in (
+            '/sbin/kpartx',
+            '/sbin/losetup',
+            '/bin/mount',
+            '/bin/umount',
+            '/bin/mknod',
+            '/usr/bin/sasquatch',
+            '/bin/rm',
+            '/bin/cp',
+            '/bin/dd',
+            '/bin/chown',
         )
     )
-    Path('/tmp/fact_overrides').write_text(f'{sudoers_content}\n')  # pylint: disable=unspecified-encoding
+    Path('/tmp/fact_overrides').write_text(f'{sudoers_content}\n', encoding='utf-8')
     _, chown_code = execute_shell_command_get_return_code('sudo chown root:root /tmp/fact_overrides')
     _, mv_code = execute_shell_command_get_return_code('sudo mv /tmp/fact_overrides /etc/sudoers.d/fact_overrides')
     if not chown_code == mv_code == 0:
         raise InstallationError('Editing sudoers file did not succeed\n{chown_output}\n{mv_output}')
 
 
 def _install_external_deb_deps():
-    '''
+    """
     install deb packages that aren't available through Debian/Ubuntu package sources
-    '''
-    with TemporaryDirectory(prefix='patool') as build_directory:
-        with OperateInDirectory(build_directory):
-            for file_name, url, sha256 in EXTERNAL_DEB_DEPS:
-                try:
-                    run(split(f'wget {url}/{file_name}'), check=True, env=os.environ)
-                    if not _sha256_hash_file(Path(file_name)) == sha256:
-                        raise InstallationError(f'Wrong file hash: {file_name}')
-                    run(split(f'sudo dpkg -i {file_name}'), capture_output=True, check=True)
-                except CalledProcessError as error:
-                    raise InstallationError(f'Error during {file_name} unpacker installation') from error
+    """
+    with TemporaryDirectory(prefix='patool') as build_directory, OperateInDirectory(build_directory):
+        for file_name, url, sha256 in EXTERNAL_DEB_DEPS:
+            try:
+                run(split(f'wget {url}/{file_name}'), check=True, env=os.environ)
+                if not _sha256_hash_file(Path(file_name)) == sha256:
+                    raise InstallationError(f'Wrong file hash: {file_name}')
+                run(split(f'sudo dpkg -i {file_name}'), capture_output=True, check=True)
+            except CalledProcessError as error:
+                raise InstallationError(f'Error during {file_name} unpacker installation') from error
 
 
 def _sha256_hash_file(file_path: Path) -> str:
     return hashlib.sha256(file_path.read_bytes()).hexdigest()
 
 
 def _install_freetz():
+    if all(
+        (Path(__file__).parent.parent / 'bin' / tool).exists()
+        for tool in [
+            'find-squashfs',
+            'unpack-kernel',
+            'freetz_bin_functions',
+            'unlzma',
+            'sfk',
+            'unsquashfs4-avm-be',
+            'unsquashfs4-avm-le',
+            'unsquashfs3-multi',
+        ]
+    ):
+        logging.info('Skipping FREETZ as it is already installed')
+        return
+
     logging.info('Installing FREETZ')
     current_user = getuser()
     freetz_build_config = Path(__file__).parent / 'freetz.config'
-    with TemporaryDirectory(prefix='fact_freetz') as build_directory:
-        with OperateInDirectory(build_directory):
-            os.umask(0o022)
-            install_github_project(
-                'Freetz-NG/freetz-ng',
-                [
-                    # add user only if it does not exist to fix issues with re-running the installation after an error
-                    'id -u makeuser || sudo useradd -M makeuser',
-                    'sudo mkdir -p /home/makeuser',
-                    'sudo chown -R makeuser /home/makeuser',
-                    f'cp {freetz_build_config} ./.config',
-                    f'sudo chown -R makeuser {build_directory}',
-                    'sudo su makeuser -c "make -j$(nproc) tools"',
-                    f'sudo chmod -R 777 {build_directory}',
-                    f'sudo chown -R {current_user} {build_directory}',
-                    'cp tools/find-squashfs tools/unpack-kernel tools/freetz_bin_functions tools/unlzma tools/sfk '
-                    f'tools/unsquashfs4-avm-be tools/unsquashfs4-avm-le tools/unsquashfs3-multi {BIN_DIR}',
-                    'sudo userdel makeuser',
-                ],
-            )
+    with TemporaryDirectory(prefix='fact_freetz') as build_directory, OperateInDirectory(build_directory):
+        os.umask(0o022)
+        install_github_project(
+            'Freetz-NG/freetz-ng',
+            [
+                # add user only if it does not exist to fix issues with re-running the installation after an error
+                'id -u makeuser || sudo useradd -M makeuser',
+                'sudo mkdir -p /home/makeuser',
+                'sudo chown -R makeuser /home/makeuser',
+                f'cp {freetz_build_config} ./.config',
+                f'sudo chown -R makeuser {build_directory}',
+                'sudo su makeuser -c "make -j$(nproc) tools"',
+                f'sudo chmod -R 777 {build_directory}',
+                f'sudo chown -R {current_user} {build_directory}',
+                'cp tools/find-squashfs tools/unpack-kernel tools/freetz_bin_functions tools/unlzma tools/sfk '
+                f'tools/unsquashfs4-avm-be tools/unsquashfs4-avm-le tools/unsquashfs3-multi {BIN_DIR}',
+                'sudo userdel makeuser',
+            ],
+        )
 
 
 def _install_plugins():