Worked on CI tests

.github/workflows/test_docker.yml

@@ -0,0 +1,84 @@
+# Run tests on Fedora and Ubuntu Docker images using GIFT CORP and GIFT PPA on commit
+name: test_docker
+on: [push]
+permissions: read-all
+jobs:
+  test_fedora:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        version: ['38']
+    container:
+      image: registry.fedoraproject.org/fedora:${{ matrix.version }}
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up container
+      run: |
+        dnf install -y dnf-plugins-core langpacks-en
+    - name: Install dependencies
+      run: |
+        dnf copr -y enable @gift/dev
+        dnf install -y @development-tools python3 python3-devel libbde-python3 libcreg-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libregf-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsapm-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-artifacts python3-cffi python3-cryptography python3-dfdatetime python3-dfimagetools python3-dfvfs python3-dfwinreg python3-dtfabric python3-idna python3-pytsk3 python3-pyxattr python3-pyyaml python3-setuptools
+    - name: Run tests
+      env:
+        LANG: C.utf8
+      run: |
+        python3 ./run_tests.py
+    - name: Run end-to-end tests
+      run: |
+        if test -f tests/end-to-end.py; then PYTHONPATH=. python3 ./tests/end-to-end.py --debug -c config/end-to-end.ini; fi
+    - name: Build source distribution
+      run: |
+        python3 ./setup.py sdist
+    - name: Build binary distribution
+      run: |
+        python3 ./setup.py bdist
+    - name: Run build and install test
+      run: |
+        python3 ./setup.py build
+        python3 ./setup.py install
+  test_ubuntu:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        version: ['22.04']
+    container:
+      image: ubuntu:${{ matrix.version }}
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up container
+      env:
+        DEBIAN_FRONTEND: noninteractive
+      run: |
+        apt-get update -q
+        apt-get install -y libterm-readline-gnu-perl locales software-properties-common
+        locale-gen en_US.UTF-8
+        ln -f -s /usr/share/zoneinfo/UTC /etc/localtime
+    - name: Install dependencies
+      run: |
+        add-apt-repository -y ppa:gift/dev
+        apt-get update -q
+        apt-get install -y build-essential python3 python3-dev libbde-python3 libcreg-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libregf-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsapm-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-artifacts python3-cffi-backend python3-cryptography python3-dfdatetime python3-dfimagetools python3-dfvfs python3-dfwinreg python3-distutils python3-dtfabric python3-idna python3-pip python3-pytsk3 python3-pyxattr python3-setuptools python3-wheel python3-yaml
+    - name: Run tests
+      env:
+        LANG: en_US.UTF-8
+      run: |
+        python3 ./run_tests.py
+    - name: Run end-to-end tests
+      env:
+        LANG: en_US.UTF-8
+      run: |
+        if test -f tests/end-to-end.py; then PYTHONPATH=. python3 ./tests/end-to-end.py --debug -c config/end-to-end.ini; fi
+    - name: Update setuptools
+      run: |
+        python3 -m pip install -U setuptools
+    - name: Build source distribution
+      run: |
+        python3 ./setup.py sdist
+    - name: Build binary distribution
+      run: |
+        python3 ./setup.py bdist
+    - name: Run build and install test
+      run: |
+        python3 ./setup.py build
+        python3 ./setup.py install

.github/workflows/test_docs.yml

@@ -0,0 +1,47 @@
+# Run docs tox tests on Ubuntu Docker images using GIFT PPA
+name: test_docs
+on:
+  pull_request:
+    branches:
+    - main
+  push:
+    branches:
+    - main
+permissions: read-all
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+        - python-version: '3.11'
+          toxenv: 'docs'
+    container:
+      image: ubuntu:22.04
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up container
+      env:
+        DEBIAN_FRONTEND: noninteractive
+      run: |
+        apt-get update -q
+        apt-get install -y libterm-readline-gnu-perl locales software-properties-common
+        locale-gen en_US.UTF-8
+        ln -f -s /usr/share/zoneinfo/UTC /etc/localtime
+    - name: Install dependencies
+      env:
+        DEBIAN_FRONTEND: noninteractive
+      run: |
+        add-apt-repository -y universe
+        add-apt-repository -y ppa:deadsnakes/ppa
+        add-apt-repository -y ppa:gift/dev
+        apt-get update -q
+        apt-get install -y build-essential git libffi-dev python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv libbde-python3 libcreg-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libregf-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsapm-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-artifacts python3-cffi-backend python3-cryptography python3-dfdatetime python3-dfimagetools python3-dfvfs python3-dfwinreg python3-distutils python3-dtfabric python3-idna python3-pip python3-pytsk3 python3-pyxattr python3-setuptools python3-yaml
+    - name: Install tox
+      run: |
+        python3 -m pip install tox
+    - name: Run tests
+      env:
+        LANG: en_US.UTF-8
+      run: |
+        tox -e${{ matrix.toxenv }}

.github/workflows/test_tox.yml

@@ -0,0 +1,131 @@
+# Run tox tests on Ubuntu Docker images using GIFT PPA
+name: test_tox
+on:
+  pull_request:
+    branches:
+    - main
+  push:
+    branches:
+    - main
+permissions: read-all
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+        - python-version: '3.7'
+          toxenv: 'py37,wheel'
+        - python-version: '3.8'
+          toxenv: 'py38,wheel'
+        - python-version: '3.9'
+          toxenv: 'py39,wheel'
+        - python-version: '3.10'
+          toxenv: 'py310,wheel'
+        - python-version: '3.11'
+          toxenv: 'py311,wheel'
+        - python-version: '3.12'
+          toxenv: 'py312,wheel'
+    container:
+      image: ubuntu:22.04
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up container
+      env:
+        DEBIAN_FRONTEND: noninteractive
+      run: |
+        apt-get update -q
+        apt-get install -y libterm-readline-gnu-perl locales software-properties-common
+        locale-gen en_US.UTF-8
+        ln -f -s /usr/share/zoneinfo/UTC /etc/localtime
+    - name: Install dependencies
+      env:
+        DEBIAN_FRONTEND: noninteractive
+      run: |
+        add-apt-repository -y universe
+        add-apt-repository -y ppa:deadsnakes/ppa
+        add-apt-repository -y ppa:gift/dev
+        apt-get update -q
+        apt-get install -y build-essential git libffi-dev python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv libbde-python3 libcreg-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libregf-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsapm-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-artifacts python3-cffi-backend python3-cryptography python3-dfdatetime python3-dfimagetools python3-dfvfs python3-dfwinreg python3-distutils python3-dtfabric python3-idna python3-pip python3-pytsk3 python3-pyxattr python3-setuptools python3-yaml
+    - name: Install tox
+      run: |
+        python3 -m pip install tox
+    - name: Run tests
+      env:
+        LANG: en_US.UTF-8
+      run: |
+        tox -e${{ matrix.toxenv }}
+  coverage:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+        - python-version: '3.10'
+          toxenv: 'coverage'
+    container:
+      image: ubuntu:22.04
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up container
+      env:
+        DEBIAN_FRONTEND: noninteractive
+      run: |
+        apt-get update -q
+        apt-get install -y libterm-readline-gnu-perl locales software-properties-common
+        locale-gen en_US.UTF-8
+        ln -f -s /usr/share/zoneinfo/UTC /etc/localtime
+    - name: Install dependencies
+      env:
+        DEBIAN_FRONTEND: noninteractive
+      run: |
+        add-apt-repository -y universe
+        add-apt-repository -y ppa:deadsnakes/ppa
+        add-apt-repository -y ppa:gift/dev
+        apt-get update -q
+        apt-get install -y build-essential git libffi-dev python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv libbde-python3 libcreg-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libregf-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsapm-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-artifacts python3-cffi-backend python3-cryptography python3-dfdatetime python3-dfimagetools python3-dfvfs python3-dfwinreg python3-distutils python3-dtfabric python3-idna python3-pip python3-pytsk3 python3-pyxattr python3-setuptools python3-yaml
+    - name: Install tox
+      run: |
+        python3 -m pip install tox
+    - name: Run tests with coverage
+      env:
+        LANG: en_US.UTF-8
+      run: |
+        tox -e${{ matrix.toxenv }}
+    - name: Upload coverage report to Codecov
+      uses: codecov/codecov-action@v3
+  lint:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+        - python-version: '3.11'
+          toxenv: 'lint'
+    container:
+      image: ubuntu:22.04
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up container
+      env:
+        DEBIAN_FRONTEND: noninteractive
+      run: |
+        apt-get update -q
+        apt-get install -y libterm-readline-gnu-perl locales software-properties-common
+        locale-gen en_US.UTF-8
+        ln -f -s /usr/share/zoneinfo/UTC /etc/localtime
+    - name: Install dependencies
+      env:
+        DEBIAN_FRONTEND: noninteractive
+      run: |
+        add-apt-repository -y universe
+        add-apt-repository -y ppa:deadsnakes/ppa
+        add-apt-repository -y ppa:gift/dev
+        apt-get update -q
+        apt-get install -y build-essential git libffi-dev python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv libbde-python3 libcreg-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libregf-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsapm-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-artifacts python3-cffi-backend python3-cryptography python3-dfdatetime python3-dfimagetools python3-dfvfs python3-dfwinreg python3-distutils python3-dtfabric python3-idna python3-pip python3-pytsk3 python3-pyxattr python3-setuptools python3-yaml
+    - name: Install tox
+      run: |
+        python3 -m pip install tox
+    - name: Run linter
+      env:
+        LANG: en_US.UTF-8
+      run: |
+        tox -e${{ matrix.toxenv }}

appveyor.yml

@@ -0,0 +1,54 @@
+environment:
+  matrix:
+  - DESCRIPTION: "Run tests on Windows with 32-bit Python 3.11"
+    MACHINE_TYPE: "x86"
+    APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022
+    PYTHON: "C:\\Python311"
+    PYTHON_VERSION: "3.11"
+    L2TBINARIES_TRACK: "dev"
+    TARGET: tests
+  - DESCRIPTION: "Run tests on Windows with 64-bit Python 3.11"
+    MACHINE_TYPE: "amd64"
+    APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022
+    PYTHON: "C:\\Python311-x64"
+    PYTHON_VERSION: "3.11"
+    L2TBINARIES_TRACK: "dev"
+    TARGET: tests
+  - DESCRIPTION: "Build wheel on Windows with 32-bit Python 3.11"
+    MACHINE_TYPE: "amd64"
+    APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022
+    PYTHON: "C:\\Python311-x64"
+    PYTHON_VERSION: "3.11"
+    L2TBINARIES_TRACK: "dev"
+    TARGET: wheel
+  - DESCRIPTION: "Build wheel on Windows with 64-bit Python 3.11"
+    MACHINE_TYPE: "amd64"
+    APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022
+    PYTHON: "C:\\Python311-x64"
+    PYTHON_VERSION: "3.11"
+    L2TBINARIES_TRACK: "dev"
+    TARGET: wheel
+  - DESCRIPTION: "Run tests on Mac OS with Python 3.11"
+    APPVEYOR_BUILD_WORKER_IMAGE: macos-monterey
+    HOMEBREW_NO_INSTALL_CLEANUP: 1
+    TARGET: tests
+
+install:
+- cmd: "%PYTHON%\\python.exe -m pip install -U build pip setuptools twine wheel"
+- ps: If ($isWindows) { .\config\appveyor\install.ps1 }
+- sh: config/appveyor/install.sh
+
+build_script:
+- cmd: IF [%TARGET%]==[wheel] (
+    "%PYTHON%\\python.exe" -m build --wheel )
+
+test_script:
+- cmd: IF [%TARGET%]==[tests] (
+    "%PYTHON%\\python.exe" run_tests.py &&
+    IF EXIST "tests\\end-to-end.py" (
+        set PYTHONPATH=. &&
+        "%PYTHON%\\python.exe" "tests\\end-to-end.py" --debug -c "config\\end-to-end.ini" ) )
+- sh: config/appveyor/runtests.sh
+
+artifacts:
+- path: dist\*.whl

config/appveyor/install.ps1

@@ -0,0 +1,27 @@
+# Script to set up tests on AppVeyor Windows.
+
+$Dependencies = "PyYAML artifacts cffi cryptography dfdatetime dfimagetools dfvfs dfwinreg dtfabric idna libbde libcreg libewf libfsapfs libfsext libfsfat libfshfs libfsntfs libfsxfs libfvde libfwnt libluksde libmodi libphdi libqcow libregf libsigscan libsmdev libsmraw libvhdi libvmdk libvsapm libvsgpt libvshadow libvslvm pytsk3 xattr"
+
+If ($Dependencies.Length -gt 0)
+{
+	$Dependencies = ${Dependencies} -split " "
+
+	$Output = Invoke-Expression -Command "git clone https://github.com/log2timeline/l2tdevtools.git ..\l2tdevtools 2>&1" | %{ "$_" }
+	Write-Host (${Output} | Out-String)
+
+	If ($env:APPVEYOR_REPO_BRANCH -eq "main")
+	{
+		$Track = "stable"
+	}
+	Else
+	{
+		$Track = $env:APPVEYOR_REPO_BRANCH
+	}
+	New-Item -ItemType "directory" -Name "dependencies"
+
+	$env:PYTHONPATH = "..\l2tdevtools"
+
+	$Output = Invoke-Expression -Command "& '${env:PYTHON}\python.exe' ..\l2tdevtools\tools\update.py --download-directory dependencies --machine-type ${env:MACHINE_TYPE} --track ${env:L2TBINARIES_TRACK} ${Dependencies} 2>&1" | %{ "$_" }
+	Write-Host (${Output} | Out-String)
+}
+

config/appveyor/install.sh

@@ -0,0 +1,7 @@
+# Script to set up tests on AppVeyor MacOS.
+
+set -e
+
+brew update -q
+brew install -q gettext gnu-sed [email protected] tox || true
+

config/appveyor/runtests.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+# Script to run tests
+
+# Set the following environment variables to build libyal with gettext.
+export CPPFLAGS="-I/usr/local/include -I/usr/local/opt/gettext/include ${CPPFLAGS}";
+export LDFLAGS="-L/usr/local/lib -L/usr/local/opt/gettext/lib ${LDFLAGS}";
+
+# Set the following environment variables to build pycrypto and yara-python.
+export CPPFLAGS="-I/usr/local/opt/[email protected]/include ${CPPFLAGS}";
+export LDFLAGS="-L/usr/local/opt/[email protected]/lib ${LDFLAGS}";
+
+# Set the following environment variables to ensure tox can find Python 3.11.
+export PATH="/usr/local/opt/[email protected]/bin:${PATH}";
+
+tox -e py311