From b06bac9b12ceec3a8a7450957bb5ee56a6fe34d4 Mon Sep 17 00:00:00 2001 From: Stefano Ortolani Date: Fri, 12 Jul 2024 17:41:37 +0100 Subject: [PATCH 1/7] Update to debian bookworm and python 3.12 --- core/Dockerfile | 155 ++++++++++++++++++++++++++------------------- modules/Dockerfile | 66 +++++++++++-------- 2 files changed, 129 insertions(+), 92 deletions(-) diff --git a/core/Dockerfile b/core/Dockerfile index 8e2bd0a..976221f 100644 --- a/core/Dockerfile +++ b/core/Dockerfile @@ -1,25 +1,42 @@ ARG DOCKER_HUB_PROXY="" -FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as composer-build + +FROM "${DOCKER_HUB_PROXY}python:3.12-slim-bookworm" as php-base + ENV DEBIAN_FRONTEND noninteractive + + # Uncomment when building in corporate environments + # COPY ./rootca.crt /usr/local/share/ca-certificates/rootca.pem + # COPY ./rootca.crt /usr/lib/ssl/cert.pem + + RUN apt-get update; apt-get install -y --no-install-recommends \ + lsb-release \ + ca-certificates \ + curl + RUN curl -sSLo /tmp/debsuryorg-archive-keyring.deb https://packages.sury.org/debsuryorg-archive-keyring.deb + RUN dpkg -i /tmp/debsuryorg-archive-keyring.deb + RUN echo "deb [signed-by=/usr/share/keyrings/deb.sury.org-php.gpg] https://packages.sury.org/php/ $(lsb_release -sc) main" > /etc/apt/sources.list.d/php.list + RUN apt-get update + + +FROM php-base as composer-build ENV DEBIAN_FRONTEND noninteractive ENV COMPOSER_ALLOW_SUPERUSER 1 ARG CORE_TAG ARG CORE_COMMIT - RUN apt-get update; apt-get install -y --no-install-recommends \ - ca-certificates \ - php \ - php-apcu \ - php-curl \ - php-xml \ - php-intl \ - php-bcmath \ - php-mbstring \ - php-mysql \ - php-redis \ - php-gd \ - php-fpm \ - php-zip \ + RUN apt-get install -y --no-install-recommends \ + php7.4 \ + php7.4-apcu \ + php7.4-curl \ + php7.4-xml \ + php7.4-intl \ + php7.4-bcmath \ + php7.4-mbstring \ + php7.4-mysql \ + php7.4-redis \ + php7.4-gd \ + php7.4-fpm \ + php7.4-zip \ unzip \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* @@ -29,41 +46,53 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as composer-build RUN composer config --no-interaction allow-plugins.composer/installers true RUN composer install RUN composer require --with-all-dependencies --no-interaction \ - supervisorphp/supervisor:^4.0 \ - guzzlehttp/guzzle \ - lstrojny/fxmlrpc \ - php-http/message \ - php-http/message-factory \ - # docker image specific dependencies - elasticsearch/elasticsearch:^8.7.0 \ - jakub-onderka/openid-connect-php:^1.0.0 \ - aws/aws-sdk-php - -FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as php-build + supervisorphp/supervisor:^4.0 \ + guzzlehttp/guzzle \ + lstrojny/fxmlrpc \ + php-http/message \ + php-http/message-factory \ + # docker image specific dependencies + elasticsearch/elasticsearch:^8.7.0 \ + jakub-onderka/openid-connect-php:^1.0.0 \ + aws/aws-sdk-php + + +FROM php-base as php-build ENV DEBIAN_FRONTEND noninteractive ENV TZ Etc/UTC - RUN apt-get update; apt-get install -y --no-install-recommends \ + RUN apt-get install -y --no-install-recommends \ gcc \ g++ \ make \ + php7.4 \ + php7.4-dev \ + php7.4-xml \ libfuzzy-dev \ - ca-certificates \ - php \ - php-dev \ - php-xml \ - php-pear \ librdkafka-dev \ libsimdjson-dev \ libzstd-dev \ git \ + php-pear \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* - RUN cp "/usr/lib/$(gcc -dumpmachine)"/libfuzzy.* /usr/lib; pecl channel-update pecl.php.net && pecl install ssdeep && pecl install rdkafka && pecl install simdjson && pecl install zstd - RUN git clone --recursive --depth=1 https://github.com/kjdev/php-ext-brotli.git && \ - cd php-ext-brotli && phpize && ./configure && make && make install + RUN apt-cache search pecl + + RUN update-alternatives --set php /usr/bin/php7.4 + RUN update-alternatives --set php-config /usr/bin/php-config7.4 + RUN update-alternatives --set phpize /usr/bin/phpize7.4 + + RUN cp "/usr/lib/$(gcc -dumpmachine)"/libfuzzy.* /usr/lib + RUN pecl channel-update pecl.php.net && \ + pecl install ssdeep && \ + pecl install rdkafka && \ + pecl install simdjson && \ + pecl install zstd + RUN git clone --recursive --depth=1 https://github.com/kjdev/php-ext-brotli.git && \ + cd php-ext-brotli && phpize && ./configure && make && make install -FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as python-build + +FROM php-base as python-build ENV DEBIAN_FRONTEND noninteractive ARG CORE_TAG ARG CORE_COMMIT @@ -77,8 +106,7 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as python-build ARG PYPI_CYBOX_VERSION ARG PYPI_PYMISP_VERSION - RUN apt-get update; apt-get install -y --no-install-recommends \ - python3-pip \ + RUN apt-get install -y --no-install-recommends \ git \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* @@ -89,7 +117,6 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as python-build else git clone --branch "${CORE_TAG}" --depth 1 https://github.com/MISP/MISP.git /var/www/MISP fi - cd /var/www/MISP || exit; git submodule update --init --recursive . EOF @@ -124,7 +151,7 @@ EOF fi done; - pip3 wheel --no-cache-dir -w /wheels/ -r /var/www/MISP/requirements.txt + pip wheel --no-cache-dir -w /wheels/ -r /var/www/MISP/requirements.txt # Remove files we do not care for rm -r /var/www/MISP/PyMISP @@ -134,13 +161,14 @@ EOF find /var/www/MISP/.git/* ! -name HEAD -exec rm -rf {} + EOF -FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" + +FROM php-base ENV DEBIAN_FRONTEND noninteractive ARG CORE_TAG ARG CORE_COMMIT ARG PHP_VER - RUN apt-get update; apt-get install -y --no-install-recommends \ + RUN apt-get install -y --no-install-recommends \ gettext \ procps \ sudo \ @@ -150,42 +178,41 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" openssl \ gpg \ gpg-agent \ - ssdeep \ - libfuzzy2 \ mariadb-client \ rsync \ - # Python Requirements - python3 \ - python3-setuptools \ - python3-pip \ # PHP Requirements - php \ - php-apcu \ - php-curl \ - php-xml \ - php-intl \ - php-bcmath \ - php-mbstring \ - php-mysql \ - php-redis \ - php-gd \ - php-fpm \ - php-zip \ - php-ldap \ + php7.4 \ + php7.4-apcu \ + php7.4-curl \ + php7.4-xml \ + php7.4-intl \ + php7.4-bcmath \ + php7.4-mbstring \ + php7.4-mysql \ + php7.4-redis \ + php7.4-gd \ + php7.4-fpm \ + php7.4-zip \ + php7.4-ldap \ + libmagic1 \ libldap-common \ librdkafka1 \ libbrotli1 \ - libsimdjson5 \ + libsimdjson14 \ libzstd1 \ + ssdeep \ + libfuzzy2 \ # Unsure we need these zip unzip \ # Require for advanced an unattended configuration curl jq \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* + RUN update-alternatives --set php /usr/bin/php7.4 + # Install python modules COPY --from=python-build /wheels /wheels - RUN pip3 install --no-cache-dir /wheels/*.whl && rm -rf /wheels + RUN pip install --no-cache-dir /wheels/*.whl && rm -rf /wheels # PHP: install prebuilt libraries, then install the app's PHP deps COPY --from=php-build ["/usr/lib/php/${PHP_VER}/ssdeep.so", "/usr/lib/php/${PHP_VER}/rdkafka.so", "/usr/lib/php/${PHP_VER}/brotli.so", "/usr/lib/php/${PHP_VER}/simdjson.so", "/usr/lib/php/${PHP_VER}/zstd.so", "/usr/lib/php/${PHP_VER}/"] @@ -208,7 +235,7 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" EOF # nginx - RUN rm /etc/nginx/sites-enabled/*; mkdir /run/php /etc/nginx/certs + RUN rm /etc/nginx/sites-enabled/*; mkdir -p /run/php /etc/nginx/certs # Make a copy of the file and configuration stores, so we can sync from it diff --git a/modules/Dockerfile b/modules/Dockerfile index f2d9218..8bb33f7 100644 --- a/modules/Dockerfile +++ b/modules/Dockerfile @@ -1,6 +1,6 @@ ARG DOCKER_HUB_PROXY="" -FROM "${DOCKER_HUB_PROXY}python:3.11-slim-bullseye" as python-build +FROM "${DOCKER_HUB_PROXY}python:3.12-slim-bookworm" as python-build ENV DEBIAN_FRONTEND noninteractive ARG MODULES_TAG ARG MODULES_COMMIT @@ -9,19 +9,17 @@ FROM "${DOCKER_HUB_PROXY}python:3.11-slim-bullseye" as python-build RUN apt-get update && apt-get install -y --no-install-recommends \ cmake \ git \ - python3-dev \ - python3-pip \ - python3-wheel \ build-essential \ - pkg-config \ libpoppler-cpp-dev \ libfuzzy-dev \ + libffi-dev \ + libxml2-dev \ + libxslt-dev \ libssl-dev \ ninja-build \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* RUN mkdir /wheels - WORKDIR /srv RUN <<-EOF if [ ! -z ${MODULES_COMMIT} ]; then @@ -29,52 +27,64 @@ FROM "${DOCKER_HUB_PROXY}python:3.11-slim-bullseye" as python-build else git clone --branch ${MODULES_TAG} --depth 1 https://github.com/MISP/misp-modules.git /srv/misp-modules fi - - cd /srv/misp-modules - sed -i "s/numpy.*/numpy<2/" REQUIREMENTS - echo "pyeti" >> REQUIREMENTS - echo "greynoise" >> REQUIREMENTS - echo "Google-Search-API" >> REQUIREMENTS - pip3 wheel -r REQUIREMENTS --no-cache-dir -w /wheels/ - rm -rf /srv/misp-modules EOF + WORKDIR /srv/misp-modules + RUN pip install pipenv + RUN sed -i "s/python_version.*/python_version = \"3.12\"/" Pipfile + RUN sed -i "/\[packages\]/a numpy = \"<2.0.0\"" Pipfile + RUN sed -i "/\[packages\]/a pysafebrowsing = \"*\"" Pipfile + RUN sed -i "/\[packages\]/a sigmf = \"*\"" Pipfile + RUN sed -i "/\[packages\]/a matplotlib = \"*\"" Pipfile + RUN pipenv lock + RUN pipenv requirements > requirements.txt + RUN pip wheel -r requirements.txt --no-cache-dir -w /wheels/ + + WORKDIR /srv/ + RUN rm -rf /srv/misp-modules + RUN <<-EOF git clone --depth 1 https://github.com/stricaud/faup.git /srv/faup cd /srv/faup if [ ! -z ${LIBFAUP_COMMIT} ]; then git checkout ${LIBFAUP_COMMIT} fi - - cd /srv/faup/build - cmake -G "Ninja" ../ - ninja - ninja install - cd /srv/faup/src/lib/bindings/python - pip3 wheel --no-cache-dir --no-dependencies -w /wheels/ . - rm -rf /srv/faup EOF + + WORKDIR /srv/faup/build + RUN cmake -G "Ninja" ../ + RUN ninja + RUN ninja install + WORKDIR /srv/faup/src/lib/bindings/python + RUN pip wheel --no-cache-dir --no-dependencies -w /wheels/ . + + WORKDIR /srv/ + RUN rm -rf /srv/faup -FROM "${DOCKER_HUB_PROXY}python:3.11-slim-bullseye" + +FROM "${DOCKER_HUB_PROXY}python:3.12-slim-bookworm" ENV DEBIAN_FRONTEND noninteractive RUN apt-get update && apt-get install -y --no-install-recommends \ libglib2.0-0 \ - libzbar0 \ - libxrender1 \ - libxext6 \ libpoppler-cpp0v5 \ libgl1 \ + libfuzzy2 \ + libffi8 \ + libxext6 \ + libxml2 \ + libxslt1.1 \ + libzbar0 \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* COPY --from=python-build /wheels /wheels COPY --from=python-build /usr/local/lib/libfaupl* /usr/local/lib/ - RUN pip3 install --no-cache-dir --use-deprecated=legacy-resolver /wheels/*.whl; ldconfig && rm -rf /wheels + RUN pip install --no-cache-dir --use-deprecated=legacy-resolver /wheels/*.whl; ldconfig && rm -rf /wheels # Since we compile faup ourselves and lua is not required anymore, we can load our own library # and skip the pre-compiled blob to improve compatibility with other architectures like ARM RUN sed -i s/LoadLibrary\(LOAD_LIB\)/LoadLibrary\(\"\\/usr\\/local\\/lib\\/libfaupl.so\"\)/ \ - /usr/local/lib/python3.11/site-packages/pyfaup/__init__.py + /usr/local/lib/python3.12/site-packages/pyfaup/__init__.py # Disable (all) warnings raised when using 'future' RUN sed -i '/import sys/a import warnings\nwarnings.warn = lambda *args, **kwargs: None' \ From 2a1c5da8386e61b8fc4ff4fbd99dd2aab0bb4a11 Mon Sep 17 00:00:00 2001 From: Stefano Ortolani Date: Fri, 12 Jul 2024 17:41:37 +0100 Subject: [PATCH 2/7] Replace Debian with Ubuntu 2404 LTS --- core/Dockerfile | 457 ++++++------ core/files/CurlDownloader.php | 676 ++++++++++++++++++ ...ondrej-ubuntu-nginx-mainline-noble.sources | 24 + .../ondrej-ubuntu-php-noble.sources | 23 + core/files/etc/nginx/sites-available/misp443 | 5 +- core/files/test.php | 22 + docker-bake.hcl | 7 +- modules/Dockerfile | 18 +- 8 files changed, 1005 insertions(+), 227 deletions(-) create mode 100644 core/files/CurlDownloader.php create mode 100644 core/files/etc/apt/sources.list.d/ondrej-ubuntu-nginx-mainline-noble.sources create mode 100644 core/files/etc/apt/sources.list.d/ondrej-ubuntu-php-noble.sources create mode 100644 core/files/test.php diff --git a/core/Dockerfile b/core/Dockerfile index 976221f..422f7f5 100644 --- a/core/Dockerfile +++ b/core/Dockerfile @@ -1,30 +1,38 @@ ARG DOCKER_HUB_PROXY="" -FROM "${DOCKER_HUB_PROXY}python:3.12-slim-bookworm" as php-base +FROM "${DOCKER_HUB_PROXY}ubuntu:24.04" as php-base ENV DEBIAN_FRONTEND noninteractive + ENV LC_ALL C.UTF-8 # Uncomment when building in corporate environments # COPY ./rootca.crt /usr/local/share/ca-certificates/rootca.pem # COPY ./rootca.crt /usr/lib/ssl/cert.pem - RUN apt-get update; apt-get install -y --no-install-recommends \ - lsb-release \ + RUN apt-get update; apt-get upgrade; apt-get install -y --no-install-recommends \ ca-certificates \ - curl - RUN curl -sSLo /tmp/debsuryorg-archive-keyring.deb https://packages.sury.org/debsuryorg-archive-keyring.deb - RUN dpkg -i /tmp/debsuryorg-archive-keyring.deb - RUN echo "deb [signed-by=/usr/share/keyrings/deb.sury.org-php.gpg] https://packages.sury.org/php/ $(lsb_release -sc) main" > /etc/apt/sources.list.d/php.list + && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* + + COPY files/etc/apt/sources.list.d/ondrej-ubuntu-php-noble.sources /etc/apt/sources.list.d/ondrej-ubuntu-php-noble.sources + COPY files/etc/apt/sources.list.d/ondrej-ubuntu-nginx-mainline-noble.sources /etc/apt/sources.list.d/ondrej-ubuntu-nginx-mainline-noble.sources + + # RUN apt-get update; apt-get install -y --no-install-recommends \ + # software-properties-common + # # && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* + # RUN add-apt-repository ppa:ondrej/php + # RUN add-apt-repository ppa:ondrej/nginx-mainline RUN apt-get update FROM php-base as composer-build ENV DEBIAN_FRONTEND noninteractive ENV COMPOSER_ALLOW_SUPERUSER 1 + ENV COMPOSER_IPRESOLVE 4 ARG CORE_TAG ARG CORE_COMMIT - RUN apt-get install -y --no-install-recommends \ + RUN apt-get update; apt-get install -y --no-install-recommends \ + ca-certificates \ php7.4 \ php7.4-apcu \ php7.4-curl \ @@ -37,15 +45,26 @@ FROM php-base as composer-build php7.4-gd \ php7.4-fpm \ php7.4-zip \ - unzip \ - && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* + unzip + # && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* + + COPY files/test.php /test.php + RUN php7.4 /test.php WORKDIR /tmp ADD https://raw.githubusercontent.com/MISP/MISP/${CORE_COMMIT:-${CORE_TAG}}/app/composer.json /tmp COPY --from=composer:latest /usr/bin/composer /usr/bin/composer - RUN composer config --no-interaction allow-plugins.composer/installers true - RUN composer install - RUN composer require --with-all-dependencies --no-interaction \ + + #RUN echo '140.82.113.6 api.github.com' >> /etc/hosts + RUN cp /usr/bin/composer /composer.phar + RUN mkdir /out/ + RUN php -r '$phar = new Phar("/composer.phar"); $phar->extractTo("/out/");' + + COPY files/CurlDownloader.php /out/src/Composer/Util/Http/CurlDownloader.php + RUN php /out/bin/composer config --no-interaction allow-plugins.composer/installers true + RUN php /out/bin/composer config --no-interaction secure-http false + RUN php /out/bin/composer install -vvvvv + RUN php /out/bin/composer require --with-all-dependencies --no-interaction \ supervisorphp/supervisor:^4.0 \ guzzlehttp/guzzle \ lstrojny/fxmlrpc \ @@ -56,207 +75,211 @@ FROM php-base as composer-build jakub-onderka/openid-connect-php:^1.0.0 \ aws/aws-sdk-php - -FROM php-base as php-build - ENV DEBIAN_FRONTEND noninteractive - ENV TZ Etc/UTC - - RUN apt-get install -y --no-install-recommends \ - gcc \ - g++ \ - make \ - php7.4 \ - php7.4-dev \ - php7.4-xml \ - libfuzzy-dev \ - librdkafka-dev \ - libsimdjson-dev \ - libzstd-dev \ - git \ - php-pear \ - && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* - - RUN apt-cache search pecl - - RUN update-alternatives --set php /usr/bin/php7.4 - RUN update-alternatives --set php-config /usr/bin/php-config7.4 - RUN update-alternatives --set phpize /usr/bin/phpize7.4 - - RUN cp "/usr/lib/$(gcc -dumpmachine)"/libfuzzy.* /usr/lib - RUN pecl channel-update pecl.php.net && \ - pecl install ssdeep && \ - pecl install rdkafka && \ - pecl install simdjson && \ - pecl install zstd - RUN git clone --recursive --depth=1 https://github.com/kjdev/php-ext-brotli.git && \ - cd php-ext-brotli && phpize && ./configure && make && make install - - -FROM php-base as python-build - ENV DEBIAN_FRONTEND noninteractive - ARG CORE_TAG - ARG CORE_COMMIT - ARG PYPI_REDIS_VERSION - ARG PYPI_LIEF_VERSION - ARG PYPI_PYDEEP2_VERSION - ARG PYPI_PYTHON_MAGIC_VERSION - ARG PYPI_MISP_LIB_STIX2_VERSION - ARG PYPI_MAEC_VERSION - ARG PYPI_MIXBOX_VERSION - ARG PYPI_CYBOX_VERSION - ARG PYPI_PYMISP_VERSION - - RUN apt-get install -y --no-install-recommends \ - git \ - && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* - - # Download MISP using git in the /var/www/ directory. Remove unnecessary items. - RUN <<-EOF - if [ ! -z "${CORE_COMMIT}" ]; then - git clone https://github.com/MISP/MISP.git /var/www/MISP && cd /var/www/MISP && git checkout "${CORE_COMMIT}" - else - git clone --branch "${CORE_TAG}" --depth 1 https://github.com/MISP/MISP.git /var/www/MISP - fi - cd /var/www/MISP || exit; git submodule update --init --recursive . -EOF - - RUN <<-EOF - mkdir /wheels - - # Add additional dependencies (container specific) - # The "set" line contains the list of modules we want to ensure are present. - # PYPI_MODULE_NAME_VERSION env vars can be set to specify the version desired, - # e.g. PYPI_SURICATA_VERSION="==2.0" to specify exactly version 2.0 for the suricata package - # - # 1. Check for presence of each module in requirements.txt - # 2. If missing, add it (with optional version from env (defaults to empty string)) - # 3. If present, replace with our specified version if it exists, otherwise leave - # the upstream version alone. - set -- "redis" "lief" "pydeep2" "python-magic" "misp-lib-stix2" "maec" "mixbox" "cybox" "pymisp" - for mod in "$@"; do - mod_version_var=$(echo "PYPI_${mod}_VERSION" | tr '[:lower:]' '[:upper:]' | tr '-' '_') - mod_version=$(eval "echo \"\$$mod_version_var\"") - grep -q ${mod} /var/www/MISP/requirements.txt - exists=$? - if [ "${exists}" -eq "1" ]; then - echo "Adding missing module ${mod} with version '${mod_version}'" - echo ${mod}${mod_version} >> /var/www/MISP/requirements.txt - else - if [ "$(echo ${mod_version} | wc -m)" -gt 1 ]; then - echo "Overwriting existing module ${mod}, version '${mod_version}'" - sed -i "/${mod}/s/.*/${mod}${mod_version}/" /var/www/MISP/requirements.txt - else - echo "Skipping overwriting ${mod} due to missing version variable" - fi - fi - done; - - pip wheel --no-cache-dir -w /wheels/ -r /var/www/MISP/requirements.txt - - # Remove files we do not care for - rm -r /var/www/MISP/PyMISP - find /var/www/MISP/INSTALL/* ! -name 'MYSQL.sql' -type f -exec rm {} + - find /var/www/MISP/INSTALL/* ! -name 'MYSQL.sql' -type l -exec rm {} + - # Remove most files in .git - we do not use git functionality in docker - find /var/www/MISP/.git/* ! -name HEAD -exec rm -rf {} + -EOF - - -FROM php-base - ENV DEBIAN_FRONTEND noninteractive - ARG CORE_TAG - ARG CORE_COMMIT - ARG PHP_VER - - RUN apt-get install -y --no-install-recommends \ - gettext \ - procps \ - sudo \ - nginx \ - supervisor \ - cron \ - openssl \ - gpg \ - gpg-agent \ - mariadb-client \ - rsync \ - # PHP Requirements - php7.4 \ - php7.4-apcu \ - php7.4-curl \ - php7.4-xml \ - php7.4-intl \ - php7.4-bcmath \ - php7.4-mbstring \ - php7.4-mysql \ - php7.4-redis \ - php7.4-gd \ - php7.4-fpm \ - php7.4-zip \ - php7.4-ldap \ - libmagic1 \ - libldap-common \ - librdkafka1 \ - libbrotli1 \ - libsimdjson14 \ - libzstd1 \ - ssdeep \ - libfuzzy2 \ - # Unsure we need these - zip unzip \ - # Require for advanced an unattended configuration - curl jq \ - && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* - - RUN update-alternatives --set php /usr/bin/php7.4 - - # Install python modules - COPY --from=python-build /wheels /wheels - RUN pip install --no-cache-dir /wheels/*.whl && rm -rf /wheels - - # PHP: install prebuilt libraries, then install the app's PHP deps - COPY --from=php-build ["/usr/lib/php/${PHP_VER}/ssdeep.so", "/usr/lib/php/${PHP_VER}/rdkafka.so", "/usr/lib/php/${PHP_VER}/brotli.so", "/usr/lib/php/${PHP_VER}/simdjson.so", "/usr/lib/php/${PHP_VER}/zstd.so", "/usr/lib/php/${PHP_VER}/"] - - # Do an early chown to limit image size - COPY --from=python-build --chown=www-data:www-data --chmod=0550 /var/www/MISP /var/www/MISP - COPY --from=composer-build --chown=www-data:www-data --chmod=0550 /tmp/Vendor /var/www/MISP/app/Vendor - COPY --from=composer-build --chown=www-data:www-data --chmod=0550 /tmp/Plugin /var/www/MISP/app/Plugin - - # Gather these in one layer, only act on actual directories under /etc/php/ - RUN <<-EOF - set -- "ssdeep" "rdkafka" "brotli" "simdjson" "zstd" - for mod in "$@"; do - for dir in /etc/php/*/; do - echo "extension=${mod}.so" > "${dir}mods-available/${mod}.ini" - done; - phpenmod "${mod}" - done; - phpenmod redis -EOF - - # nginx - RUN rm /etc/nginx/sites-enabled/*; mkdir -p /run/php /etc/nginx/certs - - # Make a copy of the file and configuration stores, so we can sync from it - - # The spirit of the upstream dockerization is to make: - # 1) User and group aligned in terms of permissions - # 2) Files executable and read only, because of some rogue scripts like 'cake' - # 3) Directories writable, because sometimes MISP add new files - - RUN <<-EOF - cp -R /var/www/MISP/app/files /var/www/MISP/app/files.dist - cp -R /var/www/MISP/app/Config /var/www/MISP/app/Config.dist - find /var/www/MISP \( ! -user www-data -or ! -group www-data \) -exec chown www-data:www-data '{}' +; - find /var/www/MISP -not -perm 550 -type f -exec chmod 0550 '{}' +; - find /var/www/MISP -not -perm 770 -type d -exec chmod 0770 '{}' +; - # Diagnostics wants this file to be present and writable even if we do not use git in docker land - touch /var/www/MISP/.git/ORIG_HEAD && chmod 0600 /var/www/MISP/.git/ORIG_HEAD && chown www-data:www-data /var/www/MISP/.git/ORIG_HEAD -EOF - - # Copy all our image specific files to appropriate locations - COPY files/ / - ENTRYPOINT [ "/entrypoint.sh" ] - - # Change Workdirectory - WORKDIR /var/www/MISP + ENTRYPOINT ["tail", "-f", "/dev/null"] + + +# FROM php-base as php-build +# ENV DEBIAN_FRONTEND noninteractive +# ENV TZ Etc/UTC +# +# RUN apt-get install -y --no-install-recommends \ +# gcc \ +# g++ \ +# make \ +# php7.4 \ +# php7.4-dev \ +# php7.4-xml \ +# libfuzzy-dev \ +# librdkafka-dev \ +# libsimdjson-dev \ +# libzstd-dev \ +# git \ +# php-pear \ +# && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* +# +# RUN apt-cache search pecl +# +# RUN update-alternatives --set php /usr/bin/php7.4 +# RUN update-alternatives --set php-config /usr/bin/php-config7.4 +# RUN update-alternatives --set phpize /usr/bin/phpize7.4 +# +# RUN cp "/usr/lib/$(gcc -dumpmachine)"/libfuzzy.* /usr/lib +# RUN pecl channel-update pecl.php.net && \ +# pecl install ssdeep && \ +# pecl install rdkafka && \ +# pecl install simdjson && \ +# pecl install zstd +# RUN git clone --recursive --depth=1 https://github.com/kjdev/php-ext-brotli.git && \ +# cd php-ext-brotli && phpize && ./configure && make && make install +# +# +# FROM php-base as python-build +# ENV DEBIAN_FRONTEND noninteractive +# ARG CORE_TAG +# ARG CORE_COMMIT +# ARG PYPI_REDIS_VERSION +# ARG PYPI_LIEF_VERSION +# ARG PYPI_PYDEEP2_VERSION +# ARG PYPI_PYTHON_MAGIC_VERSION +# ARG PYPI_MISP_LIB_STIX2_VERSION +# ARG PYPI_MAEC_VERSION +# ARG PYPI_MIXBOX_VERSION +# ARG PYPI_CYBOX_VERSION +# ARG PYPI_PYMISP_VERSION +# +# RUN apt-get install -y --no-install-recommends \ +# git \ +# python3-pip \ +# && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* +# +# # Download MISP using git in the /var/www/ directory. Remove unnecessary items. +# RUN <<-EOF +# if [ ! -z "${CORE_COMMIT}" ]; then +# git clone https://github.com/MISP/MISP.git /var/www/MISP && cd /var/www/MISP && git checkout "${CORE_COMMIT}" +# else +# git clone --branch "${CORE_TAG}" --depth 1 https://github.com/MISP/MISP.git /var/www/MISP +# fi +# cd /var/www/MISP || exit; git submodule update --init --recursive . +# EOF +# +# RUN <<-EOF +# mkdir /wheels +# +# # Add additional dependencies (container specific) +# # The "set" line contains the list of modules we want to ensure are present. +# # PYPI_MODULE_NAME_VERSION env vars can be set to specify the version desired, +# # e.g. PYPI_SURICATA_VERSION="==2.0" to specify exactly version 2.0 for the suricata package +# # +# # 1. Check for presence of each module in requirements.txt +# # 2. If missing, add it (with optional version from env (defaults to empty string)) +# # 3. If present, replace with our specified version if it exists, otherwise leave +# # the upstream version alone. +# set -- "redis" "lief" "pydeep2" "python-magic" "misp-lib-stix2" "maec" "mixbox" "cybox" "pymisp" +# for mod in "$@"; do +# mod_version_var=$(echo "PYPI_${mod}_VERSION" | tr '[:lower:]' '[:upper:]' | tr '-' '_') +# mod_version=$(eval "echo \"\$$mod_version_var\"") +# grep -q ${mod} /var/www/MISP/requirements.txt +# exists=$? +# if [ "${exists}" -eq "1" ]; then +# echo "Adding missing module ${mod} with version '${mod_version}'" +# echo ${mod}${mod_version} >> /var/www/MISP/requirements.txt +# else +# if [ "$(echo ${mod_version} | wc -m)" -gt 1 ]; then +# echo "Overwriting existing module ${mod}, version '${mod_version}'" +# sed -i "/${mod}/s/.*/${mod}${mod_version}/" /var/www/MISP/requirements.txt +# else +# echo "Skipping overwriting ${mod} due to missing version variable" +# fi +# fi +# done; +# +# pip wheel --no-cache-dir -w /wheels/ -r /var/www/MISP/requirements.txt +# +# # Remove files we do not care for +# rm -r /var/www/MISP/PyMISP +# find /var/www/MISP/INSTALL/* ! -name 'MYSQL.sql' -type f -exec rm {} + +# find /var/www/MISP/INSTALL/* ! -name 'MYSQL.sql' -type l -exec rm {} + +# # Remove most files in .git - we do not use git functionality in docker +# find /var/www/MISP/.git/* ! -name HEAD -exec rm -rf {} + +# EOF +# +# +# FROM php-base +# ENV DEBIAN_FRONTEND noninteractive +# ARG CORE_TAG +# ARG CORE_COMMIT +# ARG PHP_VER +# +# RUN apt-get install -y --no-install-recommends \ +# gettext \ +# procps \ +# sudo \ +# nginx \ +# supervisor \ +# cron \ +# openssl \ +# gpg \ +# gpg-agent \ +# mariadb-client \ +# rsync \ +# python3-pip \ +# # PHP Requirements +# php7.4 \ +# php7.4-apcu \ +# php7.4-curl \ +# php7.4-xml \ +# php7.4-intl \ +# php7.4-bcmath \ +# php7.4-mbstring \ +# php7.4-mysql \ +# php7.4-redis \ +# php7.4-gd \ +# php7.4-fpm \ +# php7.4-zip \ +# php7.4-ldap \ +# libmagic1 \ +# libldap-common \ +# librdkafka1 \ +# libbrotli1 \ +# libsimdjson19 \ +# libzstd1 \ +# ssdeep \ +# libfuzzy2 \ +# # Unsure we need these +# zip unzip \ +# # Require for advanced an unattended configuration +# curl jq \ +# && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* +# +# RUN update-alternatives --set php /usr/bin/php7.4 +# +# # Install python modules +# COPY --from=python-build /wheels /wheels +# RUN pip install --break-system-packages --no-cache-dir /wheels/*.whl && rm -rf /wheels +# +# # PHP: install prebuilt libraries, then install the app's PHP deps +# COPY --from=php-build ["/usr/lib/php/${PHP_VER}/ssdeep.so", "/usr/lib/php/${PHP_VER}/rdkafka.so", "/usr/lib/php/${PHP_VER}/brotli.so", "/usr/lib/# php/${PHP_VER}/simdjson.so", "/usr/lib/php/${PHP_VER}/zstd.so", "/usr/lib/php/${PHP_VER}/"] +# +# # Do an early chown to limit image size +# COPY --from=python-build --chown=www-data:www-data --chmod=0550 /var/www/MISP /var/www/MISP +# COPY --from=composer-build --chown=www-data:www-data --chmod=0550 /tmp/Vendor /var/www/MISP/app/Vendor +# COPY --from=composer-build --chown=www-data:www-data --chmod=0550 /tmp/Plugin /var/www/MISP/app/Plugin +# +# # Gather these in one layer, only act on actual directories under /etc/php/ +# RUN <<-EOF +# set -- "ssdeep" "rdkafka" "brotli" "simdjson" "zstd" +# for mod in "$@"; do +# for dir in /etc/php/*/; do +# echo "extension=${mod}.so" > "${dir}mods-available/${mod}.ini" +# done; +# phpenmod "${mod}" +# done; +# phpenmod redis +# EOF +# +# # nginx +# RUN rm /etc/nginx/sites-enabled/*; mkdir -p /run/php /etc/nginx/certs +# +# # Make a copy of the file and configuration stores, so we can sync from it +# +# # The spirit of the upstream dockerization is to make: +# # 1) User and group aligned in terms of permissions +# # 2) Files executable and read only, because of some rogue scripts like 'cake' +# # 3) Directories writable, because sometimes MISP add new files +# +# RUN <<-EOF +# cp -R /var/www/MISP/app/files /var/www/MISP/app/files.dist +# cp -R /var/www/MISP/app/Config /var/www/MISP/app/Config.dist +# find /var/www/MISP \( ! -user www-data -or ! -group www-data \) -exec chown www-data:www-data '{}' +; +# find /var/www/MISP -not -perm 550 -type f -exec chmod 0550 '{}' +; +# find /var/www/MISP -not -perm 770 -type d -exec chmod 0770 '{}' +; +# # Diagnostics wants this file to be present and writable even if we do not use git in docker land +# touch /var/www/MISP/.git/ORIG_HEAD && chmod 0600 /var/www/MISP/.git/ORIG_HEAD && chown www-data:www-data /var/www/MISP/.git/ORIG_HEAD +# EOF +# +# # Copy all our image specific files to appropriate locations +# COPY files/ / +# ENTRYPOINT [ "/entrypoint.sh" ] +# +# # Change Workdirectory +# WORKDIR /var/www/MISP# \ No newline at end of file diff --git a/core/files/CurlDownloader.php b/core/files/CurlDownloader.php new file mode 100644 index 0000000..294d6a2 --- /dev/null +++ b/core/files/CurlDownloader.php @@ -0,0 +1,676 @@ + ['CURLM_BAD_HANDLE', 'The passed-in handle is not a valid CURLM handle.'], +CURLM_BAD_EASY_HANDLE => ['CURLM_BAD_EASY_HANDLE', "An easy handle was not good/valid. It could mean that it isn't an easy handle at all, or possibly that the handle already is in used by this or another multi handle."], +CURLM_OUT_OF_MEMORY => ['CURLM_OUT_OF_MEMORY', 'You are doomed.'], +CURLM_INTERNAL_ERROR => ['CURLM_INTERNAL_ERROR', 'This can only be returned if libcurl bugs. Please report it to us!'], +]; + + +private static $options = [ +'http' => [ +'method' => CURLOPT_CUSTOMREQUEST, +'content' => CURLOPT_POSTFIELDS, +'header' => CURLOPT_HTTPHEADER, +'timeout' => CURLOPT_TIMEOUT, +], +'ssl' => [ +'cafile' => CURLOPT_CAINFO, +'capath' => CURLOPT_CAPATH, +'verify_peer' => CURLOPT_SSL_VERIFYPEER, +'verify_peer_name' => CURLOPT_SSL_VERIFYHOST, +'local_cert' => CURLOPT_SSLCERT, +'local_pk' => CURLOPT_SSLKEY, +'passphrase' => CURLOPT_SSLKEYPASSWD, +], +]; + + +private static $timeInfo = [ +'total_time' => true, +'namelookup_time' => true, +'connect_time' => true, +'pretransfer_time' => true, +'starttransfer_time' => true, +'redirect_time' => true, +]; + + + + +public function __construct(IOInterface $io, Config $config, array $options = [], bool $disableTls = false) +{ +$this->io = $io; +$this->config = $config; + +$this->multiHandle = $mh = curl_multi_init(); +if (function_exists('curl_multi_setopt')) { +curl_multi_setopt($mh, CURLMOPT_PIPELINING, PHP_VERSION_ID >= 70400 ? 2 : 3); +if (defined('CURLMOPT_MAX_HOST_CONNECTIONS') && !defined('HHVM_VERSION')) { +curl_multi_setopt($mh, CURLMOPT_MAX_HOST_CONNECTIONS, 8); +} +} + +curl_multi_setopt($mh, CURLMOPT_MAX_HOST_CONNECTIONS, 8); + +if (function_exists('curl_share_init')) { +$this->shareHandle = $sh = curl_share_init(); +curl_share_setopt($sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_COOKIE); +curl_share_setopt($sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS); +curl_share_setopt($sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION); +} + +$this->authHelper = new AuthHelper($io, $config); +} + + + + + + + +public function download(callable $resolve, callable $reject, string $origin, string $url, array $options, ?string $copyTo = null): int +{ +$attributes = []; +if (isset($options['retry-auth-failure'])) { +$attributes['retryAuthFailure'] = $options['retry-auth-failure']; +unset($options['retry-auth-failure']); +} + +return $this->initDownload($resolve, $reject, $origin, $url, $options, $copyTo, $attributes); +} + + + + + + + + + +private function initDownload(callable $resolve, callable $reject, string $origin, string $url, array $options, ?string $copyTo = null, array $attributes = []): int +{ +$attributes = array_merge([ +'retryAuthFailure' => true, +'redirects' => 0, +'retries' => 3, +'storeAuth' => false, +'ipResolve' => null, +], $attributes); + +if ($attributes['ipResolve'] === null && Platform::getEnv('COMPOSER_IPRESOLVE') === '4') { +$attributes['ipResolve'] = 4; +} elseif ($attributes['ipResolve'] === null && Platform::getEnv('COMPOSER_IPRESOLVE') === '6') { +$attributes['ipResolve'] = 6; +} + +$originalOptions = $options; + + +if (!Preg::isMatch('{^http://(repo\.)?packagist\.org/p/}', $url) || (false === strpos($url, '$') && false === strpos($url, '%24'))) { +$this->config->prohibitUrlByConfig($url, $this->io, $options); +} + +$curlHandle = curl_init(); +$headerHandle = fopen('php://temp/maxmemory:32768', 'w+b'); +if (false === $headerHandle) { +throw new \RuntimeException('Failed to open a temp stream to store curl headers'); +} + +if ($copyTo !== null) { +$bodyTarget = $copyTo.'~'; +} else { +$bodyTarget = 'php://temp/maxmemory:524288'; +} + +$errorMessage = ''; +set_error_handler(static function (int $code, string $msg) use (&$errorMessage): bool { +if ($errorMessage) { +$errorMessage .= "\n"; +} +$errorMessage .= Preg::replace('{^fopen\(.*?\): }', '', $msg); + +return true; +}); +$bodyHandle = fopen($bodyTarget, 'w+b'); +restore_error_handler(); +if (false === $bodyHandle) { +throw new TransportException('The "'.$url.'" file could not be written to '.($copyTo ?? 'a temporary file').': '.$errorMessage); +} + +curl_setopt($curlHandle, CURLOPT_VERBOSE, true); +curl_setopt($curlHandle, CURLOPT_STDERR, fopen('php://stdout', 'w')); +curl_setopt($curlHandle, CURLOPT_URL, $url); +curl_setopt($curlHandle, CURLOPT_FOLLOWLOCATION, false); +curl_setopt($curlHandle, CURLOPT_CONNECTTIMEOUT, 300); +curl_setopt($curlHandle, CURLOPT_TIMEOUT, 300); +curl_setopt($curlHandle, CURLOPT_WRITEHEADER, $headerHandle); +curl_setopt($curlHandle, CURLOPT_FILE, $bodyHandle); +curl_setopt($curlHandle, CURLOPT_ENCODING, ""); +curl_setopt($curlHandle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); +curl_setopt($curlHandle, CURLOPT_SSLVERSION, CURL_SSLVERSION_MAX_TLSv1_2); + +if ($attributes['ipResolve'] === 4) { +curl_setopt($curlHandle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); +} elseif ($attributes['ipResolve'] === 6) { +curl_setopt($curlHandle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6); +} + +if (function_exists('curl_share_init')) { +curl_setopt($curlHandle, CURLOPT_SHARE, $this->shareHandle); +} + +if (!isset($options['http']['header'])) { +$options['http']['header'] = []; +} + +$options['http']['header'] = array_diff($options['http']['header'], ['Connection: close']); +$options['http']['header'][] = 'Connection: keep-alive'; + +$version = curl_version(); +$features = $version['features']; +if (0 === strpos($url, 'https://') && \defined('CURL_VERSION_HTTP2') && \defined('CURL_HTTP_VERSION_2_0') && (CURL_VERSION_HTTP2 & $features) !== 0) { +curl_setopt($curlHandle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0); +} + + + +if (isset($version['version']) && in_array($version['version'], ['8.7.0', '8.7.1'], true) && \defined('CURL_VERSION_LIBZ') && (CURL_VERSION_LIBZ & $features) !== 0) { +curl_setopt($curlHandle, CURLOPT_ENCODING, "gzip"); +} + +$options['http']['header'] = $this->authHelper->addAuthenticationHeader($options['http']['header'], $origin, $url); +$options = StreamContextFactory::initOptions($url, $options, true); + +foreach (self::$options as $type => $curlOptions) { +foreach ($curlOptions as $name => $curlOption) { +if (isset($options[$type][$name])) { +if ($type === 'ssl' && $name === 'verify_peer_name') { +curl_setopt($curlHandle, $curlOption, $options[$type][$name] === true ? 2 : $options[$type][$name]); +} else { +curl_setopt($curlHandle, $curlOption, $options[$type][$name]); +} +} +} +} + +$proxy = ProxyManager::getInstance()->getProxyForRequest($url); +curl_setopt_array($curlHandle, $proxy->getCurlOptions($options['ssl'] ?? [])); + +$progress = array_diff_key(curl_getinfo($curlHandle), self::$timeInfo); + +$this->jobs[(int) $curlHandle] = [ +'url' => $url, +'origin' => $origin, +'attributes' => $attributes, +'options' => $originalOptions, +'progress' => $progress, +'curlHandle' => $curlHandle, +'filename' => $copyTo, +'headerHandle' => $headerHandle, +'bodyHandle' => $bodyHandle, +'resolve' => $resolve, +'reject' => $reject, +'primaryIp' => '', +]; + +$usingProxy = $proxy->getStatus(' using proxy (%s)'); +$ifModified = false !== stripos(implode(',', $options['http']['header']), 'if-modified-since:') ? ' if modified' : ''; +if ($attributes['redirects'] === 0 && $attributes['retries'] === 0) { +$this->io->writeError('Downloading ' . Url::sanitize($url) . $usingProxy . $ifModified, true, IOInterface::DEBUG); +} + +$this->checkCurlResult(curl_multi_add_handle($this->multiHandle, $curlHandle)); + + +return (int) $curlHandle; +} + +public function abortRequest(int $id): void +{ +if (isset($this->jobs[$id], $this->jobs[$id]['curlHandle'])) { +$job = $this->jobs[$id]; +curl_multi_remove_handle($this->multiHandle, $job['curlHandle']); +curl_close($job['curlHandle']); +if (is_resource($job['headerHandle'])) { +fclose($job['headerHandle']); +} +if (is_resource($job['bodyHandle'])) { +fclose($job['bodyHandle']); +} +if (null !== $job['filename']) { +@unlink($job['filename'].'~'); +} +unset($this->jobs[$id]); +} +} + +public function tick(): void +{ +static $timeoutWarning = false; + +if (count($this->jobs) === 0) { +return; +} + +$active = true; +$this->checkCurlResult(curl_multi_exec($this->multiHandle, $active)); +if (-1 === curl_multi_select($this->multiHandle, $this->selectTimeout)) { + +usleep(150); +} + +while ($progress = curl_multi_info_read($this->multiHandle)) { +$curlHandle = $progress['handle']; +$result = $progress['result']; +$i = (int) $curlHandle; +if (!isset($this->jobs[$i])) { +continue; +} + +$progress = curl_getinfo($curlHandle); +if (false === $progress) { +throw new \RuntimeException('Failed getting info from curl handle '.$i.' ('.$this->jobs[$i]['url'].')'); +} +$job = $this->jobs[$i]; +unset($this->jobs[$i]); +$error = curl_error($curlHandle); +$errno = curl_errno($curlHandle); +curl_multi_remove_handle($this->multiHandle, $curlHandle); +curl_close($curlHandle); + +$headers = null; +$statusCode = null; +$response = null; +try { + +if (CURLE_OK !== $errno || $error || $result !== CURLE_OK) { +$errno = $errno ?: $result; +if (!$error && function_exists('curl_strerror')) { +$error = curl_strerror($errno); +} +$progress['error_code'] = $errno; + +if ( +(!isset($job['options']['http']['method']) || $job['options']['http']['method'] === 'GET') +&& ( +in_array($errno, [7 , 16 , 92 , 6 ], true) +|| (in_array($errno, [56 , 35 ], true) && str_contains((string) $error, 'Connection reset by peer')) +) && $job['attributes']['retries'] < $this->maxRetries +) { +$attributes = ['retries' => $job['attributes']['retries'] + 1]; +if ($errno === 7 && !isset($job['attributes']['ipResolve'])) { +$attributes['ipResolve'] = 4; +} +$this->io->writeError('Retrying ('.($job['attributes']['retries'] + 1).') ' . Url::sanitize($job['url']) . ' due to curl error '. $errno, true, IOInterface::DEBUG); +$this->restartJobWithDelay($job, $job['url'], $attributes); +continue; +} + + +if ($errno === 55 ) { +$this->io->writeError('Retrying ('.($job['attributes']['retries'] + 1).') ' . Url::sanitize($job['url']) . ' due to curl error '. $errno, true, IOInterface::DEBUG); +$this->restartJobWithDelay($job, $job['url'], ['retries' => $job['attributes']['retries'] + 1]); +continue; +} + +if ($errno === 28 && PHP_VERSION_ID >= 70300 && $progress['namelookup_time'] === 0.0 && !$timeoutWarning) { +$timeoutWarning = true; +$this->io->writeError('A connection timeout was encountered. If you intend to run Composer without connecting to the internet, run the command again prefixed with COMPOSER_DISABLE_NETWORK=1 to make Composer run in offline mode.'); +} + +throw new TransportException('curl error '.$errno.' while downloading '.Url::sanitize($progress['url']).': '.$error); +} +$statusCode = $progress['http_code']; +rewind($job['headerHandle']); +$headers = explode("\r\n", rtrim(stream_get_contents($job['headerHandle']))); +fclose($job['headerHandle']); + +if ($statusCode === 0) { +throw new \LogicException('Received unexpected http status code 0 without error for '.Url::sanitize($progress['url']).': headers '.var_export($headers, true).' curl info '.var_export($progress, true)); +} + + +if (null !== $job['filename']) { +$contents = $job['filename'].'~'; +if ($statusCode >= 300) { +rewind($job['bodyHandle']); +$contents = stream_get_contents($job['bodyHandle']); +} +$response = new CurlResponse(['url' => $job['url']], $statusCode, $headers, $contents, $progress); +$this->io->writeError('['.$statusCode.'] '.Url::sanitize($job['url']), true, IOInterface::DEBUG); +} else { +$maxFileSize = $job['options']['max_file_size'] ?? null; +rewind($job['bodyHandle']); +if ($maxFileSize !== null) { +$contents = stream_get_contents($job['bodyHandle'], $maxFileSize); + + +if ($contents !== false && Platform::strlen($contents) >= $maxFileSize) { +throw new MaxFileSizeExceededException('Maximum allowed download size reached. Downloaded ' . Platform::strlen($contents) . ' of allowed ' . $maxFileSize . ' bytes'); +} +} else { +$contents = stream_get_contents($job['bodyHandle']); +} + +$response = new CurlResponse(['url' => $job['url']], $statusCode, $headers, $contents, $progress); +$this->io->writeError('['.$statusCode.'] '.Url::sanitize($job['url']), true, IOInterface::DEBUG); +} +fclose($job['bodyHandle']); + +if ($response->getStatusCode() >= 400 && $response->getHeader('content-type') === 'application/json') { +HttpDownloader::outputWarnings($this->io, $job['origin'], json_decode($response->getBody(), true)); +} + +$result = $this->isAuthenticatedRetryNeeded($job, $response); +if ($result['retry']) { +$this->restartJob($job, $job['url'], ['storeAuth' => $result['storeAuth'], 'retries' => $job['attributes']['retries'] + 1]); +continue; +} + + +if ($statusCode >= 300 && $statusCode <= 399 && $statusCode !== 304 && $job['attributes']['redirects'] < $this->maxRedirects) { +$location = $this->handleRedirect($job, $response); +if ($location) { +$this->restartJob($job, $location, ['redirects' => $job['attributes']['redirects'] + 1]); +continue; +} +} + + +if ($statusCode >= 400 && $statusCode <= 599) { +if ( +(!isset($job['options']['http']['method']) || $job['options']['http']['method'] === 'GET') +&& in_array($statusCode, [423, 425, 500, 502, 503, 504, 507, 510], true) +&& $job['attributes']['retries'] < $this->maxRetries +) { +$this->io->writeError('Retrying ('.($job['attributes']['retries'] + 1).') ' . Url::sanitize($job['url']) . ' due to status code '. $statusCode, true, IOInterface::DEBUG); +$this->restartJobWithDelay($job, $job['url'], ['retries' => $job['attributes']['retries'] + 1]); +continue; +} + +throw $this->failResponse($job, $response, $response->getStatusMessage()); +} + +if ($job['attributes']['storeAuth'] !== false) { +$this->authHelper->storeAuth($job['origin'], $job['attributes']['storeAuth']); +} + + +if (null !== $job['filename']) { +rename($job['filename'].'~', $job['filename']); +$job['resolve']($response); +} else { +$job['resolve']($response); +} +} catch (\Exception $e) { +if ($e instanceof TransportException) { +if (null !== $headers) { +$e->setHeaders($headers); +$e->setStatusCode($statusCode); +} +if (null !== $response) { +$e->setResponse($response->getBody()); +} +$e->setResponseInfo($progress); +} + +$this->rejectJob($job, $e); +} +} + +foreach ($this->jobs as $i => $curlHandle) { +$curlHandle = $this->jobs[$i]['curlHandle']; +$progress = array_diff_key(curl_getinfo($curlHandle), self::$timeInfo); + +if ($this->jobs[$i]['progress'] !== $progress) { +$this->jobs[$i]['progress'] = $progress; + +if (isset($this->jobs[$i]['options']['max_file_size'])) { + +if ($this->jobs[$i]['options']['max_file_size'] < $progress['download_content_length']) { +$this->rejectJob($this->jobs[$i], new MaxFileSizeExceededException('Maximum allowed download size reached. Content-length header indicates ' . $progress['download_content_length'] . ' bytes. Allowed ' . $this->jobs[$i]['options']['max_file_size'] . ' bytes')); +} + + +if ($this->jobs[$i]['options']['max_file_size'] < $progress['size_download']) { +$this->rejectJob($this->jobs[$i], new MaxFileSizeExceededException('Maximum allowed download size reached. Downloaded ' . $progress['size_download'] . ' of allowed ' . $this->jobs[$i]['options']['max_file_size'] . ' bytes')); +} +} + +if (isset($progress['primary_ip']) && $progress['primary_ip'] !== $this->jobs[$i]['primaryIp']) { +if ( +isset($this->jobs[$i]['options']['prevent_ip_access_callable']) && +is_callable($this->jobs[$i]['options']['prevent_ip_access_callable']) && +$this->jobs[$i]['options']['prevent_ip_access_callable']($progress['primary_ip']) +) { +$this->rejectJob($this->jobs[$i], new TransportException(sprintf('IP "%s" is blocked for "%s".', $progress['primary_ip'], $progress['url']))); +} + +$this->jobs[$i]['primaryIp'] = (string) $progress['primary_ip']; +} + + +} +} +} + + + + +private function handleRedirect(array $job, Response $response): string +{ +if ($locationHeader = $response->getHeader('location')) { +if (parse_url($locationHeader, PHP_URL_SCHEME)) { + +$targetUrl = $locationHeader; +} elseif (parse_url($locationHeader, PHP_URL_HOST)) { + +$targetUrl = parse_url($job['url'], PHP_URL_SCHEME).':'.$locationHeader; +} elseif ('/' === $locationHeader[0]) { + +$urlHost = parse_url($job['url'], PHP_URL_HOST); + + +$targetUrl = Preg::replace('{^(.+(?://|@)'.preg_quote($urlHost).'(?::\d+)?)(?:[/\?].*)?$}', '\1'.$locationHeader, $job['url']); +} else { + + +$targetUrl = Preg::replace('{^(.+/)[^/?]*(?:\?.*)?$}', '\1'.$locationHeader, $job['url']); +} +} + +if (!empty($targetUrl)) { +$this->io->writeError(sprintf('Following redirect (%u) %s', $job['attributes']['redirects'] + 1, Url::sanitize($targetUrl)), true, IOInterface::DEBUG); + +return $targetUrl; +} + +throw new TransportException('The "'.$job['url'].'" file could not be downloaded, got redirect without Location ('.$response->getStatusMessage().')'); +} + + + + + +private function isAuthenticatedRetryNeeded(array $job, Response $response): array +{ +if (in_array($response->getStatusCode(), [401, 403]) && $job['attributes']['retryAuthFailure']) { +$result = $this->authHelper->promptAuthIfNeeded($job['url'], $job['origin'], $response->getStatusCode(), $response->getStatusMessage(), $response->getHeaders(), $job['attributes']['retries']); + +if ($result['retry']) { +return $result; +} +} + +$locationHeader = $response->getHeader('location'); +$needsAuthRetry = false; + + +if ( +$job['origin'] === 'bitbucket.org' +&& !$this->authHelper->isPublicBitBucketDownload($job['url']) +&& substr($job['url'], -4) === '.zip' +&& (!$locationHeader || substr($locationHeader, -4) !== '.zip') +&& Preg::isMatch('{^text/html\b}i', $response->getHeader('content-type')) +) { +$needsAuthRetry = 'Bitbucket requires authentication and it was not provided'; +} + + +if ( +$response->getStatusCode() === 404 +&& in_array($job['origin'], $this->config->get('gitlab-domains'), true) +&& false !== strpos($job['url'], 'archive.zip') +) { +$needsAuthRetry = 'GitLab requires authentication and it was not provided'; +} + +if ($needsAuthRetry) { +if ($job['attributes']['retryAuthFailure']) { +$result = $this->authHelper->promptAuthIfNeeded($job['url'], $job['origin'], 401, null, [], $job['attributes']['retries']); +if ($result['retry']) { +return $result; +} +} + +throw $this->failResponse($job, $response, $needsAuthRetry); +} + +return ['retry' => false, 'storeAuth' => false]; +} + + + + + + + +private function restartJob(array $job, string $url, array $attributes = []): void +{ +if (null !== $job['filename']) { +@unlink($job['filename'].'~'); +} + +$attributes = array_merge($job['attributes'], $attributes); +$origin = Url::getOrigin($this->config, $url); + +$this->initDownload($job['resolve'], $job['reject'], $origin, $url, $job['options'], $job['filename'], $attributes); +} + + + + + + + +private function restartJobWithDelay(array $job, string $url, array $attributes): void +{ +if ($attributes['retries'] >= 3) { +usleep(500000); +} elseif ($attributes['retries'] >= 2) { +usleep(100000); +} + +$this->restartJob($job, $url, $attributes); +} + + + + +private function failResponse(array $job, Response $response, string $errorMessage): TransportException +{ +if (null !== $job['filename']) { +@unlink($job['filename'].'~'); +} + +$details = ''; +if (in_array(strtolower((string) $response->getHeader('content-type')), ['application/json', 'application/json; charset=utf-8'], true)) { +$details = ':'.PHP_EOL.substr($response->getBody(), 0, 200).(strlen($response->getBody()) > 200 ? '...' : ''); +} + +return new TransportException('The "'.$job['url'].'" file could not be downloaded ('.$errorMessage.')' . $details, $response->getStatusCode()); +} + + + + +private function rejectJob(array $job, \Exception $e): void +{ +if (is_resource($job['headerHandle'])) { +fclose($job['headerHandle']); +} +if (is_resource($job['bodyHandle'])) { +fclose($job['bodyHandle']); +} +if (null !== $job['filename']) { +@unlink($job['filename'].'~'); +} +$job['reject']($e); +} + +private function checkCurlResult(int $code): void +{ +if ($code !== CURLM_OK && $code !== CURLM_CALL_MULTI_PERFORM) { +throw new \RuntimeException( +isset($this->multiErrors[$code]) +? "cURL error: {$code} ({$this->multiErrors[$code][0]}): cURL message: {$this->multiErrors[$code][1]}" +: 'Unexpected cURL error: ' . $code +); +} +} +} diff --git a/core/files/etc/apt/sources.list.d/ondrej-ubuntu-nginx-mainline-noble.sources b/core/files/etc/apt/sources.list.d/ondrej-ubuntu-nginx-mainline-noble.sources new file mode 100644 index 0000000..0536632 --- /dev/null +++ b/core/files/etc/apt/sources.list.d/ondrej-ubuntu-nginx-mainline-noble.sources @@ -0,0 +1,24 @@ +Types: deb +URIs: https://ppa.launchpadcontent.net/ondrej/nginx-mainline/ubuntu/ +Suites: noble +Components: main +Signed-By: + -----BEGIN PGP PUBLIC KEY BLOCK----- + . + mI0ESX35nAEEALKDCUDVXvmW9n+T/+3G1DnTpoWh9/1xNaz/RrUH6fQKhHr568F8 + hfnZP/2CGYVYkW9hxP9LVW9IDvzcmnhgIwK+ddeaPZqh3T/FM4OTA7Q78HSvR81m + Jpf2iMLm/Zvh89ZsmP2sIgZuARiaHo8lxoTSLtmKXsM3FsJVlusyewHfABEBAAG0 + H0xhdW5jaHBhZCBQUEEgZm9yIE9uZMWZZWogU3Vyw72ItgQTAQIAIAUCSX35nAIb + AwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEE9OoKrlJnpsQjYD/jW1NlIFAlT6 + EvF2xfVbkhERii9MapjaUsSso4XLCEmZdEGX54GQ01svXnrivwnd/kmhKvyxCqiN + LDY/dOaK8MK//bDI6mqdKmG8XbP2vsdsxhifNC+GH/OwaDPvn1TyYB653kwyruCG + FjEnCreZTcRUu2oBQyolORDl+BmF4DjLiQEzBBABCgAdFiEECvaBvTqO/UqmWMI/ + thEcm0xImQEFAmXTV0AACgkQthEcm0xImQGTTggAhuMHGeBZlRUAsZE7jJM7Mf06 + /WIhcgUfBfSFnJFlFH+xdEe/GFYyVk9kingDsPh90Ecnt4n8DJHTlsuUV1+SPBIO + JfbQTUjx1n/+Ck+TVKzRByvrpRXtiZQ214m3zbhZpme2eBBMItZByjG7g925NUIq + rL+R5ZoEcZvVlYscfsA0Sr8yJTsGJPefuLYI6eJkNDa1QkzBkSSW4XaCfNIxNBRs + zN/qGe3xy0bibOaC4T2TcbZPSAVP855ahNbLAdqkyfAutiEWcKZmQpR9qNh4482k + 0pXVlQJ8UB860gVFHjwjFm/MsCeX8yfeAi38ZyInWL2OSG2pDx5ZzNESwnCPIg== + =N1rh + -----END PGP PUBLIC KEY BLOCK----- + diff --git a/core/files/etc/apt/sources.list.d/ondrej-ubuntu-php-noble.sources b/core/files/etc/apt/sources.list.d/ondrej-ubuntu-php-noble.sources new file mode 100644 index 0000000..18d8fe1 --- /dev/null +++ b/core/files/etc/apt/sources.list.d/ondrej-ubuntu-php-noble.sources @@ -0,0 +1,23 @@ +Types: deb +URIs: https://ppa.launchpadcontent.net/ondrej/php/ubuntu/ +Suites: noble +Components: main +Signed-By: -----BEGIN PGP PUBLIC KEY BLOCK----- + . + mI0ESX35nAEEALKDCUDVXvmW9n+T/+3G1DnTpoWh9/1xNaz/RrUH6fQKhHr568F8 + hfnZP/2CGYVYkW9hxP9LVW9IDvzcmnhgIwK+ddeaPZqh3T/FM4OTA7Q78HSvR81m + Jpf2iMLm/Zvh89ZsmP2sIgZuARiaHo8lxoTSLtmKXsM3FsJVlusyewHfABEBAAG0 + H0xhdW5jaHBhZCBQUEEgZm9yIE9uZMWZZWogU3Vyw72ItgQTAQIAIAUCSX35nAIb + AwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEE9OoKrlJnpsQjYD/jW1NlIFAlT6 + EvF2xfVbkhERii9MapjaUsSso4XLCEmZdEGX54GQ01svXnrivwnd/kmhKvyxCqiN + LDY/dOaK8MK//bDI6mqdKmG8XbP2vsdsxhifNC+GH/OwaDPvn1TyYB653kwyruCG + FjEnCreZTcRUu2oBQyolORDl+BmF4DjLiQEzBBABCgAdFiEECvaBvTqO/UqmWMI/ + thEcm0xImQEFAmXTV0AACgkQthEcm0xImQGTTggAhuMHGeBZlRUAsZE7jJM7Mf06 + /WIhcgUfBfSFnJFlFH+xdEe/GFYyVk9kingDsPh90Ecnt4n8DJHTlsuUV1+SPBIO + JfbQTUjx1n/+Ck+TVKzRByvrpRXtiZQ214m3zbhZpme2eBBMItZByjG7g925NUIq + rL+R5ZoEcZvVlYscfsA0Sr8yJTsGJPefuLYI6eJkNDa1QkzBkSSW4XaCfNIxNBRs + zN/qGe3xy0bibOaC4T2TcbZPSAVP855ahNbLAdqkyfAutiEWcKZmQpR9qNh4482k + 0pXVlQJ8UB860gVFHjwjFm/MsCeX8yfeAi38ZyInWL2OSG2pDx5ZzNESwnCPIg== + =N1rh + -----END PGP PUBLIC KEY BLOCK----- + diff --git a/core/files/etc/nginx/sites-available/misp443 b/core/files/etc/nginx/sites-available/misp443 index d38b810..06492d0 100644 --- a/core/files/etc/nginx/sites-available/misp443 +++ b/core/files/etc/nginx/sites-available/misp443 @@ -1,6 +1,7 @@ server { - listen 443 ssl http2; - listen [::]:443 ssl http2; + listen 443 ssl; + listen [::]:443 ssl; + http2 on; # disable access logs access_log off; diff --git a/core/files/test.php b/core/files/test.php new file mode 100644 index 0000000..ed97471 --- /dev/null +++ b/core/files/test.php @@ -0,0 +1,22 @@ + diff --git a/docker-bake.hcl b/docker-bake.hcl index 9cb37ab..4063c1d 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -1,5 +1,8 @@ variable "PLATFORMS" { - default = ["linux/amd64", "linux/arm64"] + default = [ + # "linux/amd64", + "linux/arm64", + ] } variable "PYPI_REDIS_VERSION" { @@ -72,7 +75,7 @@ variable "PHP_VER" { group "default" { targets = [ - "misp-modules", + # "misp-modules", "misp-core", ] } diff --git a/modules/Dockerfile b/modules/Dockerfile index 8bb33f7..075ce49 100644 --- a/modules/Dockerfile +++ b/modules/Dockerfile @@ -1,7 +1,8 @@ ARG DOCKER_HUB_PROXY="" -FROM "${DOCKER_HUB_PROXY}python:3.12-slim-bookworm" as python-build +FROM "${DOCKER_HUB_PROXY}ubuntu:24.04" as python-build ENV DEBIAN_FRONTEND noninteractive + ARG MODULES_TAG ARG MODULES_COMMIT ARG LIBFAUP_COMMIT @@ -9,6 +10,10 @@ FROM "${DOCKER_HUB_PROXY}python:3.12-slim-bookworm" as python-build RUN apt-get update && apt-get install -y --no-install-recommends \ cmake \ git \ + python3-dev \ + python3-pip \ + python3-wheel \ + pipenv \ build-essential \ libpoppler-cpp-dev \ libfuzzy-dev \ @@ -30,7 +35,6 @@ FROM "${DOCKER_HUB_PROXY}python:3.12-slim-bookworm" as python-build EOF WORKDIR /srv/misp-modules - RUN pip install pipenv RUN sed -i "s/python_version.*/python_version = \"3.12\"/" Pipfile RUN sed -i "/\[packages\]/a numpy = \"<2.0.0\"" Pipfile RUN sed -i "/\[packages\]/a pysafebrowsing = \"*\"" Pipfile @@ -62,7 +66,7 @@ EOF RUN rm -rf /srv/faup -FROM "${DOCKER_HUB_PROXY}python:3.12-slim-bookworm" +FROM "${DOCKER_HUB_PROXY}ubuntu:24.04" ENV DEBIAN_FRONTEND noninteractive RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -75,16 +79,18 @@ FROM "${DOCKER_HUB_PROXY}python:3.12-slim-bookworm" libxml2 \ libxslt1.1 \ libzbar0 \ + python3-pip \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* COPY --from=python-build /wheels /wheels COPY --from=python-build /usr/local/lib/libfaupl* /usr/local/lib/ - RUN pip install --no-cache-dir --use-deprecated=legacy-resolver /wheels/*.whl; ldconfig && rm -rf /wheels - + RUN ldconfig + RUN pip install --break-system-packages --no-cache-dir --use-deprecated=legacy-resolver /wheels/*.whl && rm -rf /wheels + # Since we compile faup ourselves and lua is not required anymore, we can load our own library # and skip the pre-compiled blob to improve compatibility with other architectures like ARM RUN sed -i s/LoadLibrary\(LOAD_LIB\)/LoadLibrary\(\"\\/usr\\/local\\/lib\\/libfaupl.so\"\)/ \ - /usr/local/lib/python3.12/site-packages/pyfaup/__init__.py + /usr/local/lib/python3.12/dist-packages/pyfaup/__init__.py # Disable (all) warnings raised when using 'future' RUN sed -i '/import sys/a import warnings\nwarnings.warn = lambda *args, **kwargs: None' \ From 0eca37b41e4106985ef7023ab91cc88142ae3f2f Mon Sep 17 00:00:00 2001 From: Stefano Ortolani Date: Mon, 15 Jul 2024 12:38:06 +0100 Subject: [PATCH 3/7] Debug build --- core/files/CurlDownloader.php | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/core/files/CurlDownloader.php b/core/files/CurlDownloader.php index 294d6a2..f9fec95 100644 --- a/core/files/CurlDownloader.php +++ b/core/files/CurlDownloader.php @@ -105,8 +105,6 @@ public function __construct(IOInterface $io, Config $config, array $options = [] } } -curl_multi_setopt($mh, CURLMOPT_MAX_HOST_CONNECTIONS, 8); - if (function_exists('curl_share_init')) { $this->shareHandle = $sh = curl_share_init(); curl_share_setopt($sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_COOKIE); @@ -147,7 +145,7 @@ private function initDownload(callable $resolve, callable $reject, string $origi $attributes = array_merge([ 'retryAuthFailure' => true, 'redirects' => 0, -'retries' => 3, +'retries' => 0, 'storeAuth' => false, 'ipResolve' => null, ], $attributes); @@ -196,13 +194,12 @@ private function initDownload(callable $resolve, callable $reject, string $origi curl_setopt($curlHandle, CURLOPT_STDERR, fopen('php://stdout', 'w')); curl_setopt($curlHandle, CURLOPT_URL, $url); curl_setopt($curlHandle, CURLOPT_FOLLOWLOCATION, false); -curl_setopt($curlHandle, CURLOPT_CONNECTTIMEOUT, 300); -curl_setopt($curlHandle, CURLOPT_TIMEOUT, 300); +curl_setopt($curlHandle, CURLOPT_CONNECTTIMEOUT, 10); +curl_setopt($curlHandle, CURLOPT_TIMEOUT, max((int) ini_get("default_socket_timeout"), 300)); curl_setopt($curlHandle, CURLOPT_WRITEHEADER, $headerHandle); curl_setopt($curlHandle, CURLOPT_FILE, $bodyHandle); curl_setopt($curlHandle, CURLOPT_ENCODING, ""); curl_setopt($curlHandle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); -curl_setopt($curlHandle, CURLOPT_SSLVERSION, CURL_SSLVERSION_MAX_TLSv1_2); if ($attributes['ipResolve'] === 4) { curl_setopt($curlHandle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); From cac4d13f0adb110d3cb3ee7cda6cca0d761f2249 Mon Sep 17 00:00:00 2001 From: Stefano Ortolani Date: Mon, 15 Jul 2024 13:00:32 +0100 Subject: [PATCH 4/7] Debug base image --- core/Dockerfile | 76 +++++++++++++++++++++++-------------------------- 1 file changed, 36 insertions(+), 40 deletions(-) diff --git a/core/Dockerfile b/core/Dockerfile index 422f7f5..a9431ab 100644 --- a/core/Dockerfile +++ b/core/Dockerfile @@ -1,30 +1,30 @@ ARG DOCKER_HUB_PROXY="" -FROM "${DOCKER_HUB_PROXY}ubuntu:24.04" as php-base - ENV DEBIAN_FRONTEND noninteractive - ENV LC_ALL C.UTF-8 - - # Uncomment when building in corporate environments - # COPY ./rootca.crt /usr/local/share/ca-certificates/rootca.pem - # COPY ./rootca.crt /usr/lib/ssl/cert.pem - - RUN apt-get update; apt-get upgrade; apt-get install -y --no-install-recommends \ - ca-certificates \ - && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* - - COPY files/etc/apt/sources.list.d/ondrej-ubuntu-php-noble.sources /etc/apt/sources.list.d/ondrej-ubuntu-php-noble.sources - COPY files/etc/apt/sources.list.d/ondrej-ubuntu-nginx-mainline-noble.sources /etc/apt/sources.list.d/ondrej-ubuntu-nginx-mainline-noble.sources - - # RUN apt-get update; apt-get install -y --no-install-recommends \ - # software-properties-common - # # && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* - # RUN add-apt-repository ppa:ondrej/php - # RUN add-apt-repository ppa:ondrej/nginx-mainline - RUN apt-get update +# FROM "${DOCKER_HUB_PROXY}ubuntu:24.04" as php-base +# ENV DEBIAN_FRONTEND noninteractive +# ENV LC_ALL C.UTF-8 +# +# # Uncomment when building in corporate environments +# # COPY ./rootca.crt /usr/local/share/ca-certificates/rootca.pem +# # COPY ./rootca.crt /usr/lib/ssl/cert.pem +# +# RUN apt-get update; apt-get upgrade; apt-get install -y --no-install-recommends \ +# ca-certificates \ +# && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* +# +# # COPY files/etc/apt/sources.list.d/ondrej-ubuntu-php-noble.sources /etc/apt/sources.list.d/ondrej-ubuntu-php-noble.sources +# # COPY files/etc/apt/sources.list.d/ondrej-ubuntu-nginx-mainline-noble.sources /etc/apt/sources.list.d/ondrej-ubuntu-nginx-mainline-noble.sources +# +# # RUN apt-get update; apt-get install -y --no-install-recommends \ +# # software-properties-common +# # # && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* +# # RUN add-apt-repository ppa:ondrej/php +# # RUN add-apt-repository ppa:ondrej/nginx-mainline +# # RUN apt-get update -FROM php-base as composer-build +FROM "${DOCKER_HUB_PROXY}ubuntu:24.04" as composer-build ENV DEBIAN_FRONTEND noninteractive ENV COMPOSER_ALLOW_SUPERUSER 1 ENV COMPOSER_IPRESOLVE 4 @@ -33,29 +33,25 @@ FROM php-base as composer-build RUN apt-get update; apt-get install -y --no-install-recommends \ ca-certificates \ - php7.4 \ - php7.4-apcu \ - php7.4-curl \ - php7.4-xml \ - php7.4-intl \ - php7.4-bcmath \ - php7.4-mbstring \ - php7.4-mysql \ - php7.4-redis \ - php7.4-gd \ - php7.4-fpm \ - php7.4-zip \ + php8.3 \ + php8.3-apcu \ + php8.3-curl \ + php8.3-xml \ + php8.3-intl \ + php8.3-bcmath \ + php8.3-mbstring \ + php8.3-mysql \ + php8.3-redis \ + php8.3-gd \ + php8.3-fpm \ + php8.3-zip \ unzip # && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* - COPY files/test.php /test.php - RUN php7.4 /test.php - WORKDIR /tmp ADD https://raw.githubusercontent.com/MISP/MISP/${CORE_COMMIT:-${CORE_TAG}}/app/composer.json /tmp COPY --from=composer:latest /usr/bin/composer /usr/bin/composer - #RUN echo '140.82.113.6 api.github.com' >> /etc/hosts RUN cp /usr/bin/composer /composer.phar RUN mkdir /out/ RUN php -r '$phar = new Phar("/composer.phar"); $phar->extractTo("/out/");' @@ -63,8 +59,8 @@ FROM php-base as composer-build COPY files/CurlDownloader.php /out/src/Composer/Util/Http/CurlDownloader.php RUN php /out/bin/composer config --no-interaction allow-plugins.composer/installers true RUN php /out/bin/composer config --no-interaction secure-http false - RUN php /out/bin/composer install -vvvvv - RUN php /out/bin/composer require --with-all-dependencies --no-interaction \ + RUN php /out/bin/composer install -vvvvv --ignore-platform-reqs + RUN php /out/bin/composer require --ignore-platform-reqs --with-all-dependencies --no-interaction \ supervisorphp/supervisor:^4.0 \ guzzlehttp/guzzle \ lstrojny/fxmlrpc \ From b9a45066ae938311f460dae238d08a90a9a932f4 Mon Sep 17 00:00:00 2001 From: Stefano Ortolani Date: Mon, 15 Jul 2024 18:00:34 +0100 Subject: [PATCH 5/7] Debug --- core/Dockerfile | 254 +++--------------------------------------------- 1 file changed, 11 insertions(+), 243 deletions(-) diff --git a/core/Dockerfile b/core/Dockerfile index a9431ab..6d6df52 100644 --- a/core/Dockerfile +++ b/core/Dockerfile @@ -1,35 +1,9 @@ ARG DOCKER_HUB_PROXY="" - -# FROM "${DOCKER_HUB_PROXY}ubuntu:24.04" as php-base -# ENV DEBIAN_FRONTEND noninteractive -# ENV LC_ALL C.UTF-8 -# -# # Uncomment when building in corporate environments -# # COPY ./rootca.crt /usr/local/share/ca-certificates/rootca.pem -# # COPY ./rootca.crt /usr/lib/ssl/cert.pem -# -# RUN apt-get update; apt-get upgrade; apt-get install -y --no-install-recommends \ -# ca-certificates \ -# && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* -# -# # COPY files/etc/apt/sources.list.d/ondrej-ubuntu-php-noble.sources /etc/apt/sources.list.d/ondrej-ubuntu-php-noble.sources -# # COPY files/etc/apt/sources.list.d/ondrej-ubuntu-nginx-mainline-noble.sources /etc/apt/sources.list.d/ondrej-ubuntu-nginx-mainline-noble.sources -# -# # RUN apt-get update; apt-get install -y --no-install-recommends \ -# # software-properties-common -# # # && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* -# # RUN add-apt-repository ppa:ondrej/php -# # RUN add-apt-repository ppa:ondrej/nginx-mainline -# # RUN apt-get update - - FROM "${DOCKER_HUB_PROXY}ubuntu:24.04" as composer-build ENV DEBIAN_FRONTEND noninteractive ENV COMPOSER_ALLOW_SUPERUSER 1 ENV COMPOSER_IPRESOLVE 4 - ARG CORE_TAG - ARG CORE_COMMIT RUN apt-get update; apt-get install -y --no-install-recommends \ ca-certificates \ @@ -49,6 +23,7 @@ FROM "${DOCKER_HUB_PROXY}ubuntu:24.04" as composer-build # && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* WORKDIR /tmp + COPY files/composer.json /tmp/composer.json ADD https://raw.githubusercontent.com/MISP/MISP/${CORE_COMMIT:-${CORE_TAG}}/app/composer.json /tmp COPY --from=composer:latest /usr/bin/composer /usr/bin/composer @@ -60,222 +35,15 @@ FROM "${DOCKER_HUB_PROXY}ubuntu:24.04" as composer-build RUN php /out/bin/composer config --no-interaction allow-plugins.composer/installers true RUN php /out/bin/composer config --no-interaction secure-http false RUN php /out/bin/composer install -vvvvv --ignore-platform-reqs - RUN php /out/bin/composer require --ignore-platform-reqs --with-all-dependencies --no-interaction \ - supervisorphp/supervisor:^4.0 \ - guzzlehttp/guzzle \ - lstrojny/fxmlrpc \ - php-http/message \ - php-http/message-factory \ - # docker image specific dependencies - elasticsearch/elasticsearch:^8.7.0 \ - jakub-onderka/openid-connect-php:^1.0.0 \ - aws/aws-sdk-php + # RUN php /out/bin/composer require --ignore-platform-reqs --with-all-dependencies --no-interaction \ + # supervisorphp/supervisor:^4.0 \ + # guzzlehttp/guzzle \ + # lstrojny/fxmlrpc \ + # php-http/message \ + # php-http/message-factory \ + # # docker image specific dependencies + # elasticsearch/elasticsearch:^8.7.0 \ + # jakub-onderka/openid-connect-php:^1.0.0 \ + # aws/aws-sdk-php ENTRYPOINT ["tail", "-f", "/dev/null"] - - -# FROM php-base as php-build -# ENV DEBIAN_FRONTEND noninteractive -# ENV TZ Etc/UTC -# -# RUN apt-get install -y --no-install-recommends \ -# gcc \ -# g++ \ -# make \ -# php7.4 \ -# php7.4-dev \ -# php7.4-xml \ -# libfuzzy-dev \ -# librdkafka-dev \ -# libsimdjson-dev \ -# libzstd-dev \ -# git \ -# php-pear \ -# && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* -# -# RUN apt-cache search pecl -# -# RUN update-alternatives --set php /usr/bin/php7.4 -# RUN update-alternatives --set php-config /usr/bin/php-config7.4 -# RUN update-alternatives --set phpize /usr/bin/phpize7.4 -# -# RUN cp "/usr/lib/$(gcc -dumpmachine)"/libfuzzy.* /usr/lib -# RUN pecl channel-update pecl.php.net && \ -# pecl install ssdeep && \ -# pecl install rdkafka && \ -# pecl install simdjson && \ -# pecl install zstd -# RUN git clone --recursive --depth=1 https://github.com/kjdev/php-ext-brotli.git && \ -# cd php-ext-brotli && phpize && ./configure && make && make install -# -# -# FROM php-base as python-build -# ENV DEBIAN_FRONTEND noninteractive -# ARG CORE_TAG -# ARG CORE_COMMIT -# ARG PYPI_REDIS_VERSION -# ARG PYPI_LIEF_VERSION -# ARG PYPI_PYDEEP2_VERSION -# ARG PYPI_PYTHON_MAGIC_VERSION -# ARG PYPI_MISP_LIB_STIX2_VERSION -# ARG PYPI_MAEC_VERSION -# ARG PYPI_MIXBOX_VERSION -# ARG PYPI_CYBOX_VERSION -# ARG PYPI_PYMISP_VERSION -# -# RUN apt-get install -y --no-install-recommends \ -# git \ -# python3-pip \ -# && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* -# -# # Download MISP using git in the /var/www/ directory. Remove unnecessary items. -# RUN <<-EOF -# if [ ! -z "${CORE_COMMIT}" ]; then -# git clone https://github.com/MISP/MISP.git /var/www/MISP && cd /var/www/MISP && git checkout "${CORE_COMMIT}" -# else -# git clone --branch "${CORE_TAG}" --depth 1 https://github.com/MISP/MISP.git /var/www/MISP -# fi -# cd /var/www/MISP || exit; git submodule update --init --recursive . -# EOF -# -# RUN <<-EOF -# mkdir /wheels -# -# # Add additional dependencies (container specific) -# # The "set" line contains the list of modules we want to ensure are present. -# # PYPI_MODULE_NAME_VERSION env vars can be set to specify the version desired, -# # e.g. PYPI_SURICATA_VERSION="==2.0" to specify exactly version 2.0 for the suricata package -# # -# # 1. Check for presence of each module in requirements.txt -# # 2. If missing, add it (with optional version from env (defaults to empty string)) -# # 3. If present, replace with our specified version if it exists, otherwise leave -# # the upstream version alone. -# set -- "redis" "lief" "pydeep2" "python-magic" "misp-lib-stix2" "maec" "mixbox" "cybox" "pymisp" -# for mod in "$@"; do -# mod_version_var=$(echo "PYPI_${mod}_VERSION" | tr '[:lower:]' '[:upper:]' | tr '-' '_') -# mod_version=$(eval "echo \"\$$mod_version_var\"") -# grep -q ${mod} /var/www/MISP/requirements.txt -# exists=$? -# if [ "${exists}" -eq "1" ]; then -# echo "Adding missing module ${mod} with version '${mod_version}'" -# echo ${mod}${mod_version} >> /var/www/MISP/requirements.txt -# else -# if [ "$(echo ${mod_version} | wc -m)" -gt 1 ]; then -# echo "Overwriting existing module ${mod}, version '${mod_version}'" -# sed -i "/${mod}/s/.*/${mod}${mod_version}/" /var/www/MISP/requirements.txt -# else -# echo "Skipping overwriting ${mod} due to missing version variable" -# fi -# fi -# done; -# -# pip wheel --no-cache-dir -w /wheels/ -r /var/www/MISP/requirements.txt -# -# # Remove files we do not care for -# rm -r /var/www/MISP/PyMISP -# find /var/www/MISP/INSTALL/* ! -name 'MYSQL.sql' -type f -exec rm {} + -# find /var/www/MISP/INSTALL/* ! -name 'MYSQL.sql' -type l -exec rm {} + -# # Remove most files in .git - we do not use git functionality in docker -# find /var/www/MISP/.git/* ! -name HEAD -exec rm -rf {} + -# EOF -# -# -# FROM php-base -# ENV DEBIAN_FRONTEND noninteractive -# ARG CORE_TAG -# ARG CORE_COMMIT -# ARG PHP_VER -# -# RUN apt-get install -y --no-install-recommends \ -# gettext \ -# procps \ -# sudo \ -# nginx \ -# supervisor \ -# cron \ -# openssl \ -# gpg \ -# gpg-agent \ -# mariadb-client \ -# rsync \ -# python3-pip \ -# # PHP Requirements -# php7.4 \ -# php7.4-apcu \ -# php7.4-curl \ -# php7.4-xml \ -# php7.4-intl \ -# php7.4-bcmath \ -# php7.4-mbstring \ -# php7.4-mysql \ -# php7.4-redis \ -# php7.4-gd \ -# php7.4-fpm \ -# php7.4-zip \ -# php7.4-ldap \ -# libmagic1 \ -# libldap-common \ -# librdkafka1 \ -# libbrotli1 \ -# libsimdjson19 \ -# libzstd1 \ -# ssdeep \ -# libfuzzy2 \ -# # Unsure we need these -# zip unzip \ -# # Require for advanced an unattended configuration -# curl jq \ -# && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* -# -# RUN update-alternatives --set php /usr/bin/php7.4 -# -# # Install python modules -# COPY --from=python-build /wheels /wheels -# RUN pip install --break-system-packages --no-cache-dir /wheels/*.whl && rm -rf /wheels -# -# # PHP: install prebuilt libraries, then install the app's PHP deps -# COPY --from=php-build ["/usr/lib/php/${PHP_VER}/ssdeep.so", "/usr/lib/php/${PHP_VER}/rdkafka.so", "/usr/lib/php/${PHP_VER}/brotli.so", "/usr/lib/# php/${PHP_VER}/simdjson.so", "/usr/lib/php/${PHP_VER}/zstd.so", "/usr/lib/php/${PHP_VER}/"] -# -# # Do an early chown to limit image size -# COPY --from=python-build --chown=www-data:www-data --chmod=0550 /var/www/MISP /var/www/MISP -# COPY --from=composer-build --chown=www-data:www-data --chmod=0550 /tmp/Vendor /var/www/MISP/app/Vendor -# COPY --from=composer-build --chown=www-data:www-data --chmod=0550 /tmp/Plugin /var/www/MISP/app/Plugin -# -# # Gather these in one layer, only act on actual directories under /etc/php/ -# RUN <<-EOF -# set -- "ssdeep" "rdkafka" "brotli" "simdjson" "zstd" -# for mod in "$@"; do -# for dir in /etc/php/*/; do -# echo "extension=${mod}.so" > "${dir}mods-available/${mod}.ini" -# done; -# phpenmod "${mod}" -# done; -# phpenmod redis -# EOF -# -# # nginx -# RUN rm /etc/nginx/sites-enabled/*; mkdir -p /run/php /etc/nginx/certs -# -# # Make a copy of the file and configuration stores, so we can sync from it -# -# # The spirit of the upstream dockerization is to make: -# # 1) User and group aligned in terms of permissions -# # 2) Files executable and read only, because of some rogue scripts like 'cake' -# # 3) Directories writable, because sometimes MISP add new files -# -# RUN <<-EOF -# cp -R /var/www/MISP/app/files /var/www/MISP/app/files.dist -# cp -R /var/www/MISP/app/Config /var/www/MISP/app/Config.dist -# find /var/www/MISP \( ! -user www-data -or ! -group www-data \) -exec chown www-data:www-data '{}' +; -# find /var/www/MISP -not -perm 550 -type f -exec chmod 0550 '{}' +; -# find /var/www/MISP -not -perm 770 -type d -exec chmod 0770 '{}' +; -# # Diagnostics wants this file to be present and writable even if we do not use git in docker land -# touch /var/www/MISP/.git/ORIG_HEAD && chmod 0600 /var/www/MISP/.git/ORIG_HEAD && chown www-data:www-data /var/www/MISP/.git/ORIG_HEAD -# EOF -# -# # Copy all our image specific files to appropriate locations -# COPY files/ / -# ENTRYPOINT [ "/entrypoint.sh" ] -# -# # Change Workdirectory -# WORKDIR /var/www/MISP# \ No newline at end of file From 97d4585dd4e2e18c1c7a2ab3880cd7333d55fc70 Mon Sep 17 00:00:00 2001 From: Stefano Ortolani Date: Mon, 15 Jul 2024 18:05:06 +0100 Subject: [PATCH 6/7] WIP --- core/files/composer.json | 57 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 core/files/composer.json diff --git a/core/files/composer.json b/core/files/composer.json new file mode 100644 index 0000000..a174e0d --- /dev/null +++ b/core/files/composer.json @@ -0,0 +1,57 @@ +{ + "prefer-stable": true, + "minimum-stability": "dev", + "require": { + "php": ">=7.4.0,<8.0.0", + "ext-json": "*", + "ext-mbstring": "*", + "ext-xml": "*", + "ext-dom": "*", + "ext-simplexml": "*", + "ext-pcre": "*", + "kamisama/cake-resque": "4.1.2", + "pear/crypt_gpg": "1.6.7", + "monolog/monolog": "1.25.3", + "spomky-labs/otphp": "^10.0", + "bacon/bacon-qr-code": "^2.0", + "geoip2/geoip2": "~2.0", + "browscap/browscap-php": "5.1.0" + }, + "require-dev": { + "phpunit/phpunit": "^8", + "php-parallel-lint/php-parallel-lint": "^1.2", + "cakephp/debug_kit": "^2.2.0" + }, + "suggest": { + "ext-posix": "For process info fetching", + "ext-gd": "For creating image thumbnails", + "ext-openssl": "Enabling the openssl extension allows you to access HTTPS URLs", + "ext-redis": "For working background jobs and feed and warninglist caches", + "ext-zip": "Enabling processing feeds that are ZIP compressed", + "ext-zlib": "Allow gzip compression of HTTP responses", + "ext-brotli": "Allow brotli compression of HTTP responses and audit logs", + "ext-zstd": "For better and faster compression when fetching data from remote servers", + "ext-intl": "For handling IDN domain names", + "ext-ssdeep": "For ssdeep hashes correlation", + "ext-bcmath": "For faster validating IBAN numbers", + "ext-rdkafka": "Required for publishing events to Kafka broker", + "ext-apcu": "To cache data in memory instead of file system", + "ext-simdjson": "To decode JSON structures faster", + "ext-curl": "For faster fetching data from remote servers and feeds", + "elasticsearch/elasticsearch": "For logging to elasticsearch", + "aws/aws-sdk-php": "To upload samples to S3", + "jakub-onderka/openid-connect-php": "To enable OIDC authentication", + "supervisorphp/supervisor": "For managing background jobs", + "guzzlehttp/guzzle": "Required for supervisorphp/supervisor XML-RPC requests", + "lstrojny/fxmlrpc": "Required for supervisorphp/supervisor XML-RPC requests", + "php-http/message": "Required for supervisorphp/supervisor XML-RPC requests", + "php-http/message-factory": "Required for supervisorphp/supervisor XML-RPC requests" + }, + "config": { + "vendor-dir": "Vendor", + "optimize-autoloader": true, + "allow-plugins": { + "composer/installers": true + } + } +} From 9b1e7b1fa75e536ddd29d79f1b9caf7273cb0e93 Mon Sep 17 00:00:00 2001 From: Stefano Ortolani Date: Mon, 15 Jul 2024 18:06:09 +0100 Subject: [PATCH 7/7] WIP --- core/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/Dockerfile b/core/Dockerfile index 6d6df52..8e5854e 100644 --- a/core/Dockerfile +++ b/core/Dockerfile @@ -24,7 +24,7 @@ FROM "${DOCKER_HUB_PROXY}ubuntu:24.04" as composer-build WORKDIR /tmp COPY files/composer.json /tmp/composer.json - ADD https://raw.githubusercontent.com/MISP/MISP/${CORE_COMMIT:-${CORE_TAG}}/app/composer.json /tmp + # ADD https://raw.githubusercontent.com/MISP/MISP/${CORE_COMMIT:-${CORE_TAG}}/app/composer.json /tmp COPY --from=composer:latest /usr/bin/composer /usr/bin/composer RUN cp /usr/bin/composer /composer.phar