move to github actions and build jammy package (#55)

* move to github actions and build jammy package

* build jammy package

* remove deprecated unoconv usage

* add doc test

* add svg test

* fix inkscape conversions

.docker/Dockerfile.debian.builder

@@ -0,0 +1,34 @@
+ARG DISTRO=ubuntu:focal
+
+FROM ${DISTRO} as base
+
+
+RUN apt-get update -qq -o Acquire::Languages=none && \
+    env DEBIAN_FRONTEND=noninteractive apt-get install -yqq lsb-release && \
+    if test "$(lsb_release -cs)" = 'focal' ; then \
+       env DEBIAN_FRONTEND=noninteractive apt-get install -yqq software-properties-common wget && \
+       printf "deb [arch=amd64] https://packages.geotrek.fr/ubuntu focal main" > /etc/apt/sources.list.d/geotrek.list && \
+       wget -O- "https://packages.geotrek.fr/geotrek.gpg.key" | apt-key add - && \
+       add-apt-repository ppa:jyrki-pulliainen/dh-virtualenv; fi &&\
+    env DEBIAN_FRONTEND=noninteractive apt-get install -yqq \
+    dpkg-dev \
+    debhelper \
+    dh-virtualenv \
+    git \
+    devscripts \
+    equivs
+
+
+WORKDIR /dpkg-build
+COPY debian ./debian
+
+RUN env DEBIAN_FRONTEND=noninteractive mk-build-deps --install --tool='apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends --yes' debian/control
+
+COPY . ./
+WORKDIR /dpkg-build
+
+RUN sed -i -re "1s/..UNRELEASED/.ubuntu$(lsb_release -rs)) $(lsb_release -cs)/" debian/changelog \
+    && chmod a-x debian/convertit.* \
+    && dpkg-buildpackage -us -uc -b && mkdir -p /dpkg && cp -pl /convertit[-_]* /dpkg \
+    && dpkg-deb -I /dpkg/convertit*.deb
+WORKDIR /dpkg

.github/workflows/release.yml

@@ -0,0 +1,239 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Release
+
+on:
+  pull_request:
+    paths-ignore:
+      - 'docs/**'
+  push:
+    branches:
+      - master
+    paths-ignore:
+      - 'docs/**'
+  release:
+    types:
+      - created
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  DEBIAN_FRONTEND: noninteractive
+  LANG: C.UTF-8
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    strategy:
+      matrix:
+        version: [22.04, 20.04, docker]
+        include:
+          - version: 22.04
+            distro: jammy
+          - version: 20.04
+            distro: focal
+          - version: docker
+            distro: docker image
+    name: Build ${{ matrix.distro }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Prepare debian versioning
+        if: ${{ matrix.version != 'docker' }}
+        run: |
+          sed -i 's/+dev/.ubuntu${{ matrix.version }}~dev'$GITHUB_RUN_ID'/' debian/changelog
+          sed -i 's/convertit (\([0-9]\+\.[0-9]\+\.[0-9]\+\)\(.*\)) RELEASED;/convertit (\1.ubuntu${{ matrix.version }}\2) ${{ matrix.distro }};/' debian/changelog
+
+      - name: Building package
+        if: ${{ matrix.version != 'docker' }}
+        run: |
+          DISTRO=ubuntu:${{ matrix.distro }} make build_deb
+          cp ./dpkg/*.deb /home/runner/work/convertit/
+
+      - name: Archive package artifact
+        if: ${{ matrix.version != 'docker' }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: debian-${{ matrix.distro }}
+          path: |
+            /home/runner/work/convertit/*.deb
+
+      - name: Set up Docker Buildx
+        if: ${{ matrix.version == 'docker' }}
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build Docker image
+        if: ${{ matrix.version == 'docker' }}
+        run: |
+          docker build -t convertit:latest .
+
+      - name: Upload image
+        if: ${{ matrix.version == 'docker' }}
+        uses: ishworkh/docker-image-artifact-upload@v1
+        with:
+          image: "convertit:latest"
+
+  e2e:
+    continue-on-error: true
+    strategy:
+      matrix:
+        distro: [ jammy, focal, docker ]
+        include:
+          - distro: jammy
+            version: 22.04
+
+          - distro: focal
+            version: 20.04
+
+          - distro: docker
+            version: latest
+
+    name: Test ${{ matrix.distro }}
+    runs-on: ubuntu-${{ matrix.version }}
+    needs: [ build ]
+    steps:
+      - uses: actions/download-artifact@v3
+        if: ${{ matrix.distro != 'docker' }}
+        with:
+          name: debian-${{ matrix.distro }}
+
+      - name: Download image
+        if: ${{ matrix.distro == 'docker' }}
+        uses: ishworkh/docker-image-artifact-download@v1
+        with:
+          image: "convertit:latest"
+
+      - name: Install package
+        if: ${{ matrix.distro != 'docker' }}
+        run: |
+          sudo apt-get -qq update
+          sudo apt-get -yqq install /home/runner/work/convertit/convertit/*.deb || exit 0;
+
+      - name: Run docker image
+        if: ${{ matrix.distro == 'docker' }}
+        run: |
+          docker run -p 6543:6543 -d convertit:latest
+
+      - name: Test ODT / PDF conversion
+        run: |
+          wget -O myfile.pdf "http://localhost:6543/?url=https://file-examples.com/storage/febf69dcf3656dfd992b0fa/2017/10/file-sample_100kB.odt&from=application/vnd.oasis.opendocument.text"
+          
+          if file -b --mime-type ./myfile.pdf |grep -q 'application/pdf'; then
+            echo "File is PDF";
+            exit 0;
+          else
+            echo "File is not PDF";
+            file -b -f ./myfile.pdf;
+            exit 1;
+          fi
+
+      - name: Test ODT / DOC conversion
+        run: |
+          wget -O myfile.doc "http://localhost:6543/?url=https://file-examples.com/storage/febf69dcf3656dfd992b0fa/2017/10/file-sample_100kB.odt&from=application/vnd.oasis.opendocument.text&to=application/msword"
+          
+          if file -b --mime-type ./myfile.doc |grep -q 'application/msword'; then
+            echo "File is DOC";
+            exit 0;
+          else
+            echo "File is not DOC";
+            file -b -f ./myfile.doc;
+            exit 1;
+          fi
+
+      - name: Test SVG / PNG conversion
+        run: |
+          wget -O myfile.png "http://localhost:6543/?url=https://upload.wikimedia.org/wikipedia/commons/3/30/Vector-based_example.svg&from=image/svg%2Bxml&to=image/png"
+      
+          if file -b --mime-type ./myfile.png |grep -q 'image/png'; then
+            echo "File is PNG";
+            exit 0;
+          else
+            echo "File is not PNG";
+            file -b -f ./myfile.png;
+            exit 1;
+          fi
+
+      - name: Test SVG / PDF conversion
+        run: |
+          wget -O myfile.pdf "http://localhost:6543/?url=https://upload.wikimedia.org/wikipedia/commons/3/30/Vector-based_example.svg&from=image/svg%2Bxml&to=application/pdf"
+
+          if file -b --mime-type ./myfile.pdf |grep -q 'application/pdf'; then
+            echo "File is PDF";
+            exit 0;
+          else
+            echo "File is not PDF";
+            file -b -f ./myfile.pdf;
+            exit 1;
+          fi
+  deploy:
+    name: Publish (on release only)
+    runs-on: ubuntu-latest
+    needs: [ e2e ]
+    if: ${{ github.event_name == 'release' && github.event.action == 'created' }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Download 22.04 debian artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: debian-jammy
+
+      - name: Download 20.04 debian artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: debian-focal
+
+      - name: Download docker image
+        uses: ishworkh/docker-image-artifact-download@v1
+        with:
+          image: "convertit:latest"
+
+      - name: Attach debian packages as release binaries
+        uses: skx/github-action-publish-binaries@master
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          args: '*.deb'
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_LOGIN }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Publish Docker image (v2 and latest)
+        run: |
+          if [[ "${{ github.ref }}" == *"dev"* ]]; then
+            export DOCKER_TAG=dev
+          else
+            docker tag convertit:latest makinacorpus/convertit:$(cat VERSION)
+            docker push makinacorpus/convertit:$(cat VERSION)
+            export DOCKER_TAG=latest
+          fi
+          docker tag convertit:latest makinacorpus/convertit:$DOCKER_TAG
+          docker push makinacorpus/convertit:$DOCKER_TAG
+
+      - name: Install SSH key
+        uses: shimataro/ssh-key-action@v2
+        with:
+          key: ${{ secrets.SSH_PRIVATE_KEY }}
+          known_hosts: ${{ secrets.SSH_KNOWN_HOSTS }}
+      - name: Publish debian packages
+        run: |
+          if [[ "${{ github.ref }}" == *"dev"* ]]; then
+            export DEB_COMPONENT=dev
+          else
+            export DEB_COMPONENT=main
+          fi
+          echo "${{ github.ref }} : Publishing as $DEB_COMPONENT package"
+          scp -P ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=no /home/runner/work/convertit/convertit/convertit_*_amd64.deb ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }}:/srv/packages/incoming/$DEB_COMPONENT/
+          if [[ "${{ github.ref }}" == *"dev"* ]]; then
+            ssh -p ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=no ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} make bionic_dev -C /srv/packages
+            ssh -p ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=no ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} make focal_dev -C /srv/packages
+          else
+            ssh -p ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=no ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} make bionic_main -C /srv/packages
+            ssh -p ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=no ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} make focal_main -C /srv/packages
+          fi

.gitignore

@@ -52,3 +52,5 @@ src/convertit/tests/data/download_dir/
 
 # IDE
 /.vscode/
+/dpkg/
+/.idea/

CHANGES.rst

@@ -5,7 +5,10 @@ CHANGELOG
 2.2.6.dev0
 ------------------
 
--
+- Add Ubuntu 22.04 official support with debian package
+- Update Docker image to ubuntu 22.04 base
+- Use directly libreoffice instead of deprecated unoconv
+- Move CI to github-action
 
 
 2.2.5 (2021-01-13)

Dockerfile

@@ -1,17 +1,29 @@
-FROM ubuntu:bionic
+ARG DISTRO=jammy
+
+FROM ubuntu:${DISTRO} as base
 MAINTAINER Makina Corpus "[email protected]"
 
-RUN apt-get update && apt-get install -y -qq build-essential wget unoconv inkscape virtualenv && \
+RUN apt-get update && apt-get install -y -qq python3 libreoffice default-jre libreoffice-java-common inkscape python3-magic && \
     apt-get autoclean && apt-get clean all && rm -rf /var/apt/lists/*
 
-ADD . /opt/apps/convertit
 
 WORKDIR /opt/apps/convertit
 
-RUN virtualenv -p python3 .
-RUN ./bin/python setup.py install
-
 ADD .docker/run.sh /usr/local/bin/run
 
 EXPOSE 6543
 CMD ["/bin/sh", "-e", "/usr/local/bin/run"]
+
+FROM base as build
+RUN apt-get update && apt-get install -y -qq build-essential python3-venv python3-dev && \
+    apt-get autoclean && apt-get clean all && rm -rf /var/apt/lists/*
+
+ADD . /opt/apps/convertit
+
+RUN python3 -m venv . && ./bin/pip install --no-cache-dir pip setuptools wheel -U
+RUN ./bin/python setup.py install
+
+FROM base as prod
+
+COPY --from=build /opt/apps/convertit /opt/apps/convertit
+ADD .docker/run.sh /usr/local/bin/run

Makefile

@@ -0,0 +1,7 @@
+build_deb:
+	docker pull $(DISTRO)
+	docker build -t convertit_deb -f .docker/Dockerfile.debian.builder --build-arg DISTRO=$(DISTRO) .
+	docker run --name convertit_deb_run -t convertit_deb bash -c "exit"
+	docker cp convertit_deb_run:/dpkg ./
+	docker stop convertit_deb_run
+	docker rm convertit_deb_run

convertit/__init__.py

@@ -3,7 +3,6 @@
 
 from pyramid.config import Configurator
 
-
 PROGRAMS = {}
 
 

convertit/converters/inkscape.py

@@ -1,8 +1,10 @@
 import subprocess
 from mimetypes import types_map
+from pathlib import Path
 
-from convertit import exists
+from packaging import version
 
+from convertit import exists
 
 pdf_mimetype = 'application/pdf'
 svg_mimetype = 'image/svg+xml'
@@ -12,15 +14,38 @@ def is_available():
     return exists('inkscape')
 
 
+def get_inkscape_version():
+    dpkg_process = subprocess.Popen(['dpkg', '-s', 'inkscape'], stdout=subprocess.PIPE, text=True)
+    grep_process = subprocess.Popen(["grep", "Version"], stdin=dpkg_process.stdout, stdout=subprocess.PIPE, text=True)
+    output, error = grep_process.communicate()
+    version_str = output.strip().replace("Version: ", "").split('-')[0]
+    return version.parse(version_str)
+
+
 def svg_to_pdf(source, target):
-    command = [
-        'inkscape', '-f', source, '-A', target]
+    path = f"{target}/{Path(source).stem}.pdf"
+
+    if get_inkscape_version() < version.parse('1.0'):
+        command = [
+            'inkscape', '-f', source, '-A', path
+        ]
+    else:
+        command = [
+            'inkscape', '--export-filename', path,
+        ]
     subprocess.call(command)
 
 
 def svg_to_png(source, target):
-    command = [
-        'inkscape', '-f', source, '-e', target]
+    path = f"{target}/{Path(source).stem}.png"
+
+    if get_inkscape_version() < version.parse('1.0'):
+        command = [
+            'inkscape', '-f', source, '-e', path]
+
+    else:
+        command = [
+            'inkscape', '--export-filename', path, ]
     subprocess.call(command)