Merge pull request #6 from fccn/igobranco/improvements

Multiple improvements

Dockerfile

@@ -3,8 +3,8 @@
 # Because we are using features that are only available on the patched qt version of wkhtmltopdf.
 # It is based on ubuntu image because the wkhtmltopdf deb depends on 'libjpeg-turbo8' package that was removed from the debian repositories.
 # In future we hope that wkhtmltopdf maintainer review the code and its dependencies.
-FROM ubuntu:20.04
-LABEL maintainer="ivo.branco@fccn.pt"
+FROM ubuntu:24.04
+LABEL maintainer="[email protected].pt"
 
 ENV DEBIAN_FRONTEND noninteractive
 
@@ -15,29 +15,58 @@ RUN apt-get upgrade -y
 RUN apt-get install -y build-essential xorg libssl-dev libxrender-dev wget
 
 # Install wkhtmltopdf dependencies
-RUN apt-get update && apt-get install -y --no-install-recommends xvfb libfontconfig libjpeg-turbo8 xfonts-75dpi fontconfig
+RUN apt-get install -y --no-install-recommends xvfb libfontconfig libjpeg-turbo8 xfonts-75dpi fontconfig
 
 # Download and install wkhtmltopdf from maintainers page so we include a version with a patched qt and include support for more features.
-RUN wget --quiet https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.bionic_amd64.deb
-RUN dpkg -i wkhtmltox_0.12.6-1.bionic_amd64.deb
-RUN rm wkhtmltox_0.12.6-1.bionic_amd64.deb
+RUN wget --quiet https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6.1-2/wkhtmltox_0.12.6.1-2.jammy_amd64.deb
+RUN dpkg -i wkhtmltox_0.12.6.1-2.jammy_amd64.deb
+RUN rm wkhtmltox_0.12.6.1-2.jammy_amd64.deb
 
 # Install swig debian package for pip requirement endesive
 RUN apt-get install -y swig
 
-# Install python3 and pip
-RUN apt-get install -y python3.9 python3-pip
+RUN apt-get install -y libssl-dev zlib1g-dev libbz2-dev \
+    libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \
+    xz-utils tk-dev libffi-dev liblzma-dev python3-openssl git
+
+ARG PYTHON_VERSION=3.11.8
+ENV PYENV_ROOT /opt/pyenv
+RUN git clone https://github.com/pyenv/pyenv $PYENV_ROOT --branch v2.3.36 --depth 1
+
+# Install Python
+RUN $PYENV_ROOT/bin/pyenv install $PYTHON_VERSION
+
+# Create virtualenv
+RUN $PYENV_ROOT/versions/$PYTHON_VERSION/bin/python -m venv /opt/venv
+
+# Create virtual environment
+RUN python3 -m venv /opt/venv
+
+# Activate virtual environment
+ENV PATH /opt/venv/bin:${PATH}
+ENV VIRTUAL_ENV /opt/venv/
 
 # Cleanup apt cache
 RUN apt-get -y clean && \
     apt-get -y purge && \
     rm -rf /var/lib/apt/lists/* /tmp/*
 
 WORKDIR /app
+
+RUN pip install \
+    # https://pypi.org/project/setuptools/
+    # https://pypi.org/project/pip/
+    # https://pypi.org/project/wheel/
+    setuptools==69.1.1 pip==24.0 wheel==0.43.0
+
+# Install requirements file
 COPY requirements.txt .
-RUN python3 -m pip install -r requirements.txt
+RUN python -m pip install -r requirements.txt
+
+# Default amount of uWSGI processes
+ENV UWSGI_WORKERS=2
 
-COPY app.py uwsgi.ini ./
+COPY app.py uwsgi.ini default-config.yml ./
 COPY static static
 COPY nau nau
 

README.md

@@ -5,6 +5,9 @@ This should be installed as a docker container.
 
 For development proposes you can run using flask (recomended), uwsgi or uwsgi inside of a docker container.
 
+## Python
+Tested using the Python version `3.11.8`.
+
 ## Virtual environment
 
 ```bash

config.sample.yml

@@ -1,45 +1,47 @@
-HTTP_HEADER_NAME: X-NAU-Certificate-force-html
-HTTP_HEADER_VALUE: true
+# Or alternatively use `OPENEDX_LMS_URL` configuration or `OPENEDX_LMS_URL` environment variable
 LMS_SERVER_URL: https://lms.ENV.nau.fccn.pt
-CERTIFICATE_FILE_NAME: certificate.pdf
-CERTIFICATE_IMAGE_FILE_NAME: certificate
-HTTP_HEADER_META_PREFIX: pdfkit-
-HTTP_HEADER_META_IMAGE_PREFIX: imgkit-
-HTTP_HEADER_META_IMAGE_FORMAT: imgkit-format
-HTTP_HEADER_META_VERSION_NAME: nau-course-certificate-version
-HTTP_HEADER_META_FILENAME_NAME: nau-course-certificate-filename
-HTTP_HEADER_META_IMAGE_FILENAME_NAME: nau-course-certificate-image-filename
-HTTP_HEADER_META_LIMIT_NUMBER_PAGES: nau-course-certificate-limit-pages
-BUCKET_NAME: nau-ENV-certificates
-BUCKET_AWS_ACCESS_KEY_ID: xxxxxxxxxxxxxxxxxxxx
-BUCKET_AWS_SECRET_ACCESS_KEY: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
-BUCKET_ENDPOINT_URL: http://rgw.nau.fccn.pt
-BUCKET_CERTIFICATE_NO_VERSION_KEY: no-version
+
 DIGITAL_SIGNATURE:
   CERTIFICATE_P12_PATH: ./digital_signature_dev/sign-pdf.dev.nau.fccn.pt.p12
   CERTIFICATE_P12_PASSWORD: "1234"
   # SIGNATURE_ALGORITHM: sha256
-  signaturebox: 742,30,810,60
+  signaturebox: 742,50,810,80
   contact: [email protected]
   location: Lisboa
   reason: 
     pt-pt: Certificado de curso assinado digitalmente por NAU
     en: Digitally signed course certificate by NAU
-LOGGING: 
-  version: 1
-  disable_existing_loggers: False
-  root:
-    level: INFO
-    handlers: [console]
-  formatters:
-    standard:
-      datefmt: "%Y-%m-%d %H:%M:%S"
-      format: "%(asctime)s %(levelname)-10s %(message)s"
-    error:
-      format: "%(levelname)s <PID %(process)d:%(processName)s> %(name)s.%(funcName)s(): %(message)s"
-  handlers:
-    console:
-      class: logging.StreamHandler
-      level: DEBUG
-      stream: ext://sys.stdout
-      formatter: standard
+# BUCKET_NAME: nau-ENV-certificates
+# BUCKET_AWS_ACCESS_KEY_ID: xxxxxxxxxxxxxxxxxxxx
+# BUCKET_AWS_SECRET_ACCESS_KEY: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+# BUCKET_ENDPOINT_URL: http://rgw.nau.fccn.pt
+# BUCKET_CERTIFICATE_NO_VERSION_KEY: no-version
+# CERTIFICATE_FILE_NAME: certificate.pdf
+# CERTIFICATE_IMAGE_FILE_NAME: certificate
+# HTTP_HEADER_NAME: X-NAU-Certificate-force-html
+# HTTP_HEADER_VALUE: true
+# HTTP_HEADER_META_PREFIX: pdfkit-
+# HTTP_HEADER_META_IMAGE_PREFIX: imgkit-
+# HTTP_HEADER_META_IMAGE_FORMAT: imgkit-format
+# HTTP_HEADER_META_VERSION_NAME: nau-course-certificate-version
+# HTTP_HEADER_META_FILENAME_NAME: nau-course-certificate-filename
+# HTTP_HEADER_META_IMAGE_FILENAME_NAME: nau-course-certificate-image-filename
+# HTTP_HEADER_META_LIMIT_NUMBER_PAGES: nau-course-certificate-limit-pages
+# LOGGING: 
+#   version: 1
+#   disable_existing_loggers: False
+#   root:
+#     level: INFO
+#     handlers: [console]
+#   formatters:
+#     standard:
+#       datefmt: "%Y-%m-%d %H:%M:%S"
+#       format: "%(asctime)s %(levelname)-10s %(message)s"
+#     error:
+#       format: "%(levelname)s <PID %(process)d:%(processName)s> %(name)s.%(funcName)s(): %(message)s"
+#   handlers:
+#     console:
+#       class: logging.StreamHandler
+#       level: DEBUG
+#       stream: ext://sys.stdout
+#       formatter: standard

default-config.yml

@@ -0,0 +1,18 @@
+LOGGING: 
+  version: 1
+  disable_existing_loggers: False
+  root:
+    level: INFO
+    handlers: [console]
+  formatters:
+    standard:
+      datefmt: "%Y-%m-%d %H:%M:%S"
+      format: "%(asctime)s %(levelname)-10s %(message)s"
+    error:
+      format: "%(levelname)s <PID %(process)d:%(processName)s> %(name)s.%(funcName)s(): %(message)s"
+  handlers:
+    console:
+      class: logging.StreamHandler
+      level: DEBUG
+      stream: ext://sys.stdout
+      formatter: standard

docker-compose.yml

@@ -10,4 +10,5 @@ services:
       - ./digital_signature_dev:/app/digital_signature_dev
     ports:
       - "5000:5000"
-
+    environment:
+      - OPENEDX_LMS_URL=https://lms.dev.nau.fccn.pt

nau/course/certificate/course_certificate_to_pdf.py

@@ -13,6 +13,7 @@
 from nau.course.certificate.cut_pdf import cut_pdf_limit_pages
 from requests.auth import HTTPBasicAuth
 from nau.course.certificate.digital_sign_pdf import digital_sign_pdf
+import os
 
 from urllib.parse import parse_qs
 
@@ -36,7 +37,7 @@ def __init__(self, config: Configuration, path: str, query_string: str):
 
         # https://www.digitalocean.com/community/tutorials/how-to-use-logging-in-python-3
         # https://docs.python.org/3/library/logging.config.html#logging-config-dictschema
-        logging.config.dictConfig(self._config.get('LOGGING'))
+        logging.config.dictConfig(self._config.get('LOGGING', Configuration('default-config.yml').config().get('LOGGING')))
 
         self._path = path
         # parse query string to a dict where its value is a list.
@@ -47,7 +48,9 @@ def __init__(self, config: Configuration, path: str, query_string: str):
         self._language = language_query_values[0] if language_query_values else None
 
         # https://lms.dev.nau.fccn.pt
-        lms_server_url = self._config['LMS_SERVER_URL']
+        lms_server_url = self._config.get('LMS_SERVER_URL', self._config.get('OPENEDX_LMS_URL', os.getenv('OPENEDX_LMS_URL')))
+        if not lms_server_url:
+            raise Exception("Bad configuration, configure `LMS_SERVER_URL` or `OPENEDX_LMS_URL` on the config.yml or alternatively `OPENEDX_LMS_URL` environment variable.")
         self._url = lms_server_url + '/' + path
         if query_string and len(query_string) > 0:
             self._url += "?" + query_string.decode('ascii')
@@ -65,28 +68,31 @@ def convert(self):
         '''
         logger.info(
             "Converting html certificate to PDF with URL: {}".format(self._url))
-
-        certificate_version = self._get_certificate_http_meta(
-            self.http_header_meta_version_name())
-        logger.info("certificate_version: {}".format(certificate_version))
-        s3_bucket_certificate_key = self._path + '/' + \
-            (certificate_version if certificate_version else self.bucket_no_version()).replace(
-                ' ', '_') + self.s3_suffix()
-
+
         binary_output = None
-        if (self._certificate_id is not None):
-            binary_output = self.get_certificate_on_s3_bucket(
-                self.bucket_name(),
-                self.bucket_endpoint_url(),
-                self.aws_access_key_id(),
-                self.aws_secret_access_key(),
-                s3_bucket_certificate_key
-            )
+        if self.cache_to_bucket():
+            certificate_version = self._get_certificate_http_meta(
+                self.http_header_meta_version_name())
+            logger.info("certificate_version: {}".format(certificate_version))
+            s3_bucket_certificate_key = self._path + '/' + \
+                (certificate_version if certificate_version else self.bucket_no_version()).replace(
+                    ' ', '_') + self.s3_suffix()
+
+            if (self._certificate_id is not None):
+                binary_output = self.get_certificate_on_s3_bucket(
+                    self.bucket_name(),
+                    self.bucket_endpoint_url(),
+                    self.aws_access_key_id(),
+                    self.aws_secret_access_key(),
+                    s3_bucket_certificate_key
+                )
+        else:
+            logger.warning("No caching on buckets configured")
 
         if (binary_output is None):
             binary_output = self.generate_new_certificate_to_dest_format()
 
-        if (self._certificate_id is not None):
+        if self.cache_to_bucket() and self._certificate_id is not None:
             self.save_certificate(s3_bucket_certificate_key, binary_output)
 
         return binary_output
@@ -100,37 +106,40 @@ def s3_suffix(self):
         raise NotImplementedError("To be redefined in subclasses")
 
     def http_header_name(self):
-        return self._config['HTTP_HEADER_NAME']
+        return self._config.get('HTTP_HEADER_NAME', 'X-NAU-Certificate-force-html')
 
     def http_header_meta_prefix(self):
-        return self._config['HTTP_HEADER_META_PREFIX']
+        return self._config.get('HTTP_HEADER_META_PREFIX', 'pdfkit-')
 
     def http_header_value(self):
-        return str(self._config['HTTP_HEADER_VALUE'])
+        return str(self._config.get('HTTP_HEADER_VALUE', True))
+
+    def cache_to_bucket(self):
+        return self.bucket_name() and self.aws_access_key_id() and self.aws_secret_access_key()
 
     def bucket_name(self):
-        return self._config['BUCKET_NAME']
+        return self._config.get('BUCKET_NAME')
 
     def aws_access_key_id(self):
-        return self._config['BUCKET_AWS_ACCESS_KEY_ID']
+        return self._config.get('BUCKET_AWS_ACCESS_KEY_ID')
 
     def aws_secret_access_key(self):
-        return self._config['BUCKET_AWS_SECRET_ACCESS_KEY']
+        return self._config.get('BUCKET_AWS_SECRET_ACCESS_KEY')
 
     def bucket_endpoint_url(self):
-        return self._config['BUCKET_ENDPOINT_URL']
+        return self._config.get('BUCKET_ENDPOINT_URL')
 
     def http_header_meta_version_name(self):
-        return self._config['HTTP_HEADER_META_VERSION_NAME']
+        return self._config.get('HTTP_HEADER_META_VERSION_NAME', 'nau-course-certificate-version')
 
     def http_header_meta_filename_name(self):
-        return self._config['HTTP_HEADER_META_FILENAME_NAME']
+        return self._config.get('HTTP_HEADER_META_FILENAME_NAME', 'nau-course-certificate-filename')
 
     def http_header_meta_limit_number_pages(self):
-        return self._config.get('HTTP_HEADER_META_LIMIT_NUMBER_PAGES', None)
+        return self._config.get('HTTP_HEADER_META_LIMIT_NUMBER_PAGES', 'nau-course-certificate-limit-pages')
 
     def bucket_no_version(self):
-        return self._config['BUCKET_CERTIFICATE_NO_VERSION_KEY']
+        return self._config.get('BUCKET_CERTIFICATE_NO_VERSION_KEY', 'no-version')
 
     def lms_servers_auth_user(self):
         return self._config.get('LMS_SERVER_AUTH_USER', None)
@@ -139,10 +148,10 @@ def lms_servers_auth_pass(self):
         return self._config.get('LMS_SERVER_AUTH_PASS', None)
 
     def http_header_meta_image_filename_name(self):
-        return self._config['HTTP_HEADER_META_IMAGE_FILENAME_NAME']
+        return self._config.get('HTTP_HEADER_META_IMAGE_FILENAME_NAME', 'nau-course-certificate-image-filename')
 
     def http_header_meta_image_prefix(self):
-        return self._config['HTTP_HEADER_META_IMAGE_PREFIX']
+        return self._config.get('HTTP_HEADER_META_IMAGE_PREFIX', 'imgkit-')
 
     def http_header_meta_image_format(self):
         return self._config.get('HTTP_HEADER_META_IMAGE_FORMAT', 'imgkit-format')
@@ -256,7 +265,7 @@ def __init__(self, config: Configuration, path: str, query_string: str):
     def get_filename(self):
         filename = self._get_certificate_http_meta(
             self.http_header_meta_filename_name())
-        return filename if filename is not None else self._config['CERTIFICATE_FILE_NAME']
+        return filename if filename is not None else self._config.get('CERTIFICATE_FILE_NAME', 'certificate.pdf')
 
     def s3_suffix(self):
         return ".pdf"

nau/course/certificate/digital_sign_pdf.py

@@ -45,7 +45,7 @@ def _get_config_value(config, language: str, default_value):
         if type(config) is dict:
             by_lang = config.get(language)
             if type(by_lang) is not str:
-                logger.warn("Incorrect configuration on the digital signature configuration")
+                logger.warning("Incorrect configuration on the digital signature configuration for '%s' on language '%s'", config, language)
                 return default_value
             return by_lang
         return default_value

requirements.txt

@@ -1,15 +1,15 @@
 pdfkit==1.0.0
-imgkit==1.2.2
-Flask==2.2.2
-uWSGI==2.0.21
-PyYAML==6.0
-beautifulsoup4==4.11.1
-requests==2.28.1
-boto3==1.26.37
-PyPDF2==1.28.6
-endesive==2.0.13
-pyOpenSSL==22.1.0
-cryptography==38.0.4
-# Flask doesn't specify the dependency correctly, the new version of Werkzeug 
-# isn't compatible with older version of Flask
-Werkzeug==2.2.2
+imgkit==1.2.3
+Flask==3.0.3
+uWSGI==2.0.27
+PyYAML==6.0.2
+beautifulsoup4==4.12.3
+requests==2.32.3
+boto3==1.35.46
+PyPDF2==3.0.1
+endesive==2.17.3
+pyOpenSSL==24.2.1
+cryptography==43.0.3
+Werkzeug==3.0.4
+# To fix "oscrypto.errors.LibraryNotFoundError: Error detecting the version of libcrypto" https://github.com/wbond/oscrypto/issues/78
+git+https://github.com/wbond/oscrypto.git@d5f3437

uwsgi.ini

@@ -4,7 +4,10 @@ http-socket = :5000
 endif =
 
 master = true
-processes = 5
+workers = 1
+if-env = UWSGI_WORKERS
+workers = %(_)
+endif =
 
 strict = true
 enable-threads = true