From 66458adc3fe70bdd7cbf4c3168a9a94a88bbfeb9 Mon Sep 17 00:00:00 2001 From: lwthiker Date: Fri, 29 Jul 2022 18:52:43 +0300 Subject: [PATCH 1/5] Push Debian-based Docker images to DockerHub Push Debian-based Docker images to DockerHub automatically on each release. The Debian images are suffixed with "-slim-buster" suffix. --- .github/workflows/publish-docker-image.yml | 52 ++++++++++++++++++---- 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/.github/workflows/publish-docker-image.yml b/.github/workflows/publish-docker-image.yml index 3a4b6671..d2871c14 100644 --- a/.github/workflows/publish-docker-image.yml +++ b/.github/workflows/publish-docker-image.yml @@ -21,8 +21,8 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Extract metadata (tags, labels) for Docker (chrome) - id: meta_chrome + - name: Extract metadata (tags, labels) for Docker (chrome, alpine) + id: meta_chrome_alpine uses: docker/metadata-action@v3 with: images: lwthiker/curl-impersonate @@ -38,11 +38,11 @@ jobs: push: true context: chrome/ file: chrome/Dockerfile.alpine - tags: ${{ steps.meta_chrome.outputs.tags }} - labels: ${{ steps.meta_chrome.outputs.labels }} + tags: ${{ steps.meta_chrome_alpine.outputs.tags }} + labels: ${{ steps.meta_chrome_alpine.outputs.labels }} - - name: Extract metadata (tags, labels) for Docker (firefox) - id: meta_firefox + - name: Extract metadata (tags, labels) for Docker (firefox, alpine) + id: meta_ff_alpine uses: docker/metadata-action@v3 with: images: lwthiker/curl-impersonate @@ -58,5 +58,41 @@ jobs: push: true context: firefox/ file: firefox/Dockerfile.alpine - tags: ${{ steps.meta_firefox.outputs.tags }} - labels: ${{ steps.meta_firefox.outputs.labels }} + tags: ${{ steps.meta_ff_alpine.outputs.tags }} + labels: ${{ steps.meta_ff_alpine.outputs.labels }} + + - name: Extract metadata (tags, labels) for Docker (chrome, debian) + id: meta_chrome_debian + uses: docker/metadata-action@v3 + with: + images: lwthiker/curl-impersonate + tags: | + type=semver,pattern={{version}},suffix=-chrome-slim-buster + type=semver,pattern={{major}}.{{minor}},suffix=-chrome-slim-buster + + - name: Build and push the Chrome version of curl-impersonate + uses: docker/build-push-action@v2 + with: + push: true + context: chrome/ + file: chrome/Dockerfile + tags: ${{ steps.meta_chrome_debian.outputs.tags }} + labels: ${{ steps.meta_chrome_debian.outputs.labels }} + + - name: Extract metadata (tags, labels) for Docker (firefox, debian) + id: meta_ff_debian + uses: docker/metadata-action@v3 + with: + images: lwthiker/curl-impersonate + tags: | + type=semver,pattern={{version}},suffix=-ff-slim-buster + type=semver,pattern={{major}}.{{minor}},suffix=-ff-slim-buster + + - name: Build and push the Firefox version of curl-impersonate + uses: docker/build-push-action@v2 + with: + push: true + context: firefox/ + file: firefox/Dockerfile + tags: ${{ steps.meta_ff_debian.outputs.tags }} + labels: ${{ steps.meta_ff_debian.outputs.labels }} From 6eaf8fafa33658efd8d8db0871c49886d4dcba57 Mon Sep 17 00:00:00 2001 From: lwthiker Date: Fri, 29 Jul 2022 19:49:00 +0300 Subject: [PATCH 2/5] Minimize Docker image size with multi-stage build Minimize the size of the resulting Docker image size by using multi-stage build and copying the resulting binaries into a minimal Debian system. This was done with the Alpine Docker images up until now but not with the Debian images. --- Dockerfile.template | 21 +++++++++++++++++---- INSTALL.md | 6 +++--- chrome/Dockerfile | 13 ++++++++++++- chrome/Dockerfile.alpine | 2 +- firefox/Dockerfile | 17 ++++++++++++++++- firefox/Dockerfile.alpine | 2 +- tests/Dockerfile | 18 ++++++++---------- 7 files changed, 58 insertions(+), 21 deletions(-) diff --git a/Dockerfile.template b/Dockerfile.template index aaabd6bf..fe4344a2 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -8,7 +8,7 @@ {{#debian}} # Python is needed for building libnss. # Use it as a common base. -FROM python:3.10.1-slim-buster +FROM python:3.10.1-slim-buster as builder {{/debian}} {{#alpine}} FROM alpine:3.15.0 as builder @@ -225,10 +225,10 @@ COPY curl_chrome* curl_edge* curl_safari* out/ RUN sed -i 's@/usr/bin/env bash@/usr/bin/env ash@' out/curl_* {{/alpine}} RUN chmod +x out/curl_* -{{#alpine}} -# When using alpine, create a final, minimal image with the compiled binaries +# Create a final, minimal image with the compiled binaries # only. +{{#alpine}} FROM alpine:3.15.0 {{#firefox}} # curl tries to load the CA certificates for libnss. @@ -236,9 +236,22 @@ FROM alpine:3.15.0 # which is supplied by 'nss' on alpine. RUN apk add --no-cache nss {{/firefox}} +{{/alpine}} +{{#debian}} +FROM debian:buster-slim +{{#firefox}} +# curl tries to load the CA certificates for libnss. +# It loads them from /usr/lib/libnssckbi.so and /usr/lib/libnsspem.so, +# which are supplied by 'libnss3' and 'nss-plugin-pem' on debian. +RUN apt-get update && apt-get install -y libnss3 nss-plugin-pem +{{/firefox}} +{{/debian}} # Copy curl-impersonate from the builder image COPY --from=builder /build/install /usr/local +{{#debian}} +# Copy to /build/out as well for backward compatibility with previous versions. +COPY --from=builder /build/out /build/out +{{/debian}} # Wrapper scripts COPY --from=builder /build/out/curl_* /usr/local/bin/ -{{/alpine}} diff --git a/INSTALL.md b/INSTALL.md index 569a7043..adee3414 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -160,9 +160,9 @@ The Docker build is a bit more reproducible and serves as the reference implemen ``` docker build -t curl-impersonate-chrome chrome/ ``` -The resulting image contains a `/build/out` directory with the following: +The resulting binaries and libraries are in the `/usr/local` directory, which contains: * `curl-impersonate-chrome`, `curl-impersonate` - The curl binary that can impersonate Chrome/Edge/Safari. It is compiled statically against libcurl, BoringSSL, and libnghttp2 so that it won't conflict with any existing libraries on your system. You can use it from the container or copy it out. Tested to work on Ubuntu 20.04. -* `curl_chrome98`, `curl_chrome99`, `...` - Wrapper scripts that launch `curl-impersonate` with all the needed flags. +* `curl_chrome99`, `curl_chrome100`, `...` - Wrapper scripts that launch `curl-impersonate` with all the needed flags. * `libcurl-impersonate-chrome.so`, `libcurl-impersonate.so` - libcurl compiled with impersonation support. See [libcurl-impersonate](#libcurl-impersonate) below for more details. You can use them inside the docker, copy them out using `docker cp` or use them in a multi-stage docker build. @@ -172,7 +172,7 @@ Build with: ``` docker build -t curl-impersonate-ff firefox/ ``` -The resulting image contains a `/build/out` directory with the following: +The resulting binaries and libraries are in the `/usr/local` directory, which contains: * `curl-impersonate-ff`, `curl-impersonate` - The curl binary that can impersonate Firefox. It is compiled statically against libcurl, nss, and libnghttp2 so that it won't conflict with any existing libraries on your system. You can use it from the container or copy it out. Tested to work on Ubuntu 20.04. * `curl_ff91esr`, `curl_ff95`, `...` - Wrapper scripts that launch `curl-impersonate` with all the needed flags. * `libcurl-impersonate-ff.so`, `libcurl-impersonate.so` - libcurl compiled with impersonation support. See [libcurl-impersonate](#libcurl-impersonate) below for more details. diff --git a/chrome/Dockerfile b/chrome/Dockerfile index 6e37f3c7..7d2af1ea 100644 --- a/chrome/Dockerfile +++ b/chrome/Dockerfile @@ -7,7 +7,7 @@ # Python is needed for building libnss. # Use it as a common base. -FROM python:3.10.1-slim-buster +FROM python:3.10.1-slim-buster as builder WORKDIR /build @@ -134,3 +134,14 @@ RUN ! (ldd ./out/curl-impersonate | grep -q -e nghttp2 -e brotli -e ssl -e crypt # Wrapper scripts COPY curl_chrome* curl_edge* curl_safari* out/ RUN chmod +x out/curl_* + +# Create a final, minimal image with the compiled binaries +# only. +FROM debian:buster-slim + +# Copy curl-impersonate from the builder image +COPY --from=builder /build/install /usr/local +# Copy to /build/out as well for backward compatibility with previous versions. +COPY --from=builder /build/out /build/out +# Wrapper scripts +COPY --from=builder /build/out/curl_* /usr/local/bin/ diff --git a/chrome/Dockerfile.alpine b/chrome/Dockerfile.alpine index b370ff3a..9188d284 100644 --- a/chrome/Dockerfile.alpine +++ b/chrome/Dockerfile.alpine @@ -134,7 +134,7 @@ COPY curl_chrome* curl_edge* curl_safari* out/ RUN sed -i 's@/usr/bin/env bash@/usr/bin/env ash@' out/curl_* RUN chmod +x out/curl_* -# When using alpine, create a final, minimal image with the compiled binaries +# Create a final, minimal image with the compiled binaries # only. FROM alpine:3.15.0 diff --git a/firefox/Dockerfile b/firefox/Dockerfile index d352d8b9..a6c45e67 100644 --- a/firefox/Dockerfile +++ b/firefox/Dockerfile @@ -7,7 +7,7 @@ # Python is needed for building libnss. # Use it as a common base. -FROM python:3.10.1-slim-buster +FROM python:3.10.1-slim-buster as builder WORKDIR /build @@ -130,3 +130,18 @@ RUN ! (ldd ./out/curl-impersonate | grep -q -e nghttp2 -e brotli -e ssl -e crypt # Wrapper scripts COPY curl_ff* out/ RUN chmod +x out/curl_* + +# Create a final, minimal image with the compiled binaries +# only. +FROM debian:buster-slim +# curl tries to load the CA certificates for libnss. +# It loads them from /usr/lib/libnssckbi.so and /usr/lib/libnsspem.so, +# which are supplied by 'libnss3' and 'nss-plugin-pem' on debian. +RUN apt-get update && apt-get install -y libnss3 nss-plugin-pem + +# Copy curl-impersonate from the builder image +COPY --from=builder /build/install /usr/local +# Copy to /build/out as well for backward compatibility with previous versions. +COPY --from=builder /build/out /build/out +# Wrapper scripts +COPY --from=builder /build/out/curl_* /usr/local/bin/ diff --git a/firefox/Dockerfile.alpine b/firefox/Dockerfile.alpine index 423acc92..c1d00ac1 100644 --- a/firefox/Dockerfile.alpine +++ b/firefox/Dockerfile.alpine @@ -126,7 +126,7 @@ COPY curl_ff* out/ RUN sed -i 's@/usr/bin/env bash@/usr/bin/env ash@' out/curl_* RUN chmod +x out/curl_* -# When using alpine, create a final, minimal image with the compiled binaries +# Create a final, minimal image with the compiled binaries # only. FROM alpine:3.15.0 # curl tries to load the CA certificates for libnss. diff --git a/tests/Dockerfile b/tests/Dockerfile index e510d19e..702ed830 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -16,21 +16,19 @@ COPY requirements.txt requirements.txt RUN pip install --upgrade pip && \ pip install -r requirements.txt -RUN mkdir /tests/firefox /tests/chrome - # Copy the built binaries from both containers -COPY --from=ff /build/out/curl-impersonate-ff /tests/install/bin/ -COPY --from=ff /build/out/curl_* /tests/install/bin/ -COPY --from=ff /build/out/libcurl-impersonate* /tests/install/lib/ -COPY --from=chrome /build/out/curl-impersonate-chrome /tests/install/bin/ -COPY --from=chrome /build/out/curl_* /tests/install/bin/ -COPY --from=chrome /build/out/libcurl-impersonate* /tests/install/lib/ +COPY --from=ff /usr/local/ /usr/local/ +COPY --from=chrome /usr/local/ /usr/local/ + +# Needed to update the loader's cache +RUN ldconfig COPY . . # Compile 'minicurl' which is used for testing libcurl-impersonate. # 'minicurl' is compiled against the "regular" libcurl. # libcurl-impersonate will replace it at runtime via LD_PRELOAD. -RUN gcc -Wall -Werror -o /tests/install/bin/minicurl minicurl.c `curl-config --libs` +RUN gcc -Wall -Werror -o minicurl minicurl.c `curl-config --libs` +RUN install minicurl /usr/local/bin -ENTRYPOINT ["pytest", "--install-dir", "/tests/install"] +ENTRYPOINT ["pytest"] From 8f19c20868363d20264bce75d344b35f6bc02e0c Mon Sep 17 00:00:00 2001 From: lwthiker Date: Fri, 29 Jul 2022 21:05:29 +0300 Subject: [PATCH 3/5] Run 'ldconfig' in Docker images Run ldconfig to update the loader's cache. This is necessary specifically on the Debian image since we put .so files in /usr/local. --- Dockerfile.template | 2 ++ chrome/Dockerfile | 2 ++ chrome/Dockerfile.alpine | 2 ++ firefox/Dockerfile | 2 ++ firefox/Dockerfile.alpine | 2 ++ 5 files changed, 10 insertions(+) diff --git a/Dockerfile.template b/Dockerfile.template index fe4344a2..2833dc66 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -249,6 +249,8 @@ RUN apt-get update && apt-get install -y libnss3 nss-plugin-pem # Copy curl-impersonate from the builder image COPY --from=builder /build/install /usr/local +# Update the loader's cache +RUN ldconfig {{#debian}} # Copy to /build/out as well for backward compatibility with previous versions. COPY --from=builder /build/out /build/out diff --git a/chrome/Dockerfile b/chrome/Dockerfile index 7d2af1ea..7a40abe6 100644 --- a/chrome/Dockerfile +++ b/chrome/Dockerfile @@ -141,6 +141,8 @@ FROM debian:buster-slim # Copy curl-impersonate from the builder image COPY --from=builder /build/install /usr/local +# Update the loader's cache +RUN ldconfig # Copy to /build/out as well for backward compatibility with previous versions. COPY --from=builder /build/out /build/out # Wrapper scripts diff --git a/chrome/Dockerfile.alpine b/chrome/Dockerfile.alpine index 9188d284..11bb6154 100644 --- a/chrome/Dockerfile.alpine +++ b/chrome/Dockerfile.alpine @@ -140,5 +140,7 @@ FROM alpine:3.15.0 # Copy curl-impersonate from the builder image COPY --from=builder /build/install /usr/local +# Update the loader's cache +RUN ldconfig # Wrapper scripts COPY --from=builder /build/out/curl_* /usr/local/bin/ diff --git a/firefox/Dockerfile b/firefox/Dockerfile index a6c45e67..ffa450db 100644 --- a/firefox/Dockerfile +++ b/firefox/Dockerfile @@ -141,6 +141,8 @@ RUN apt-get update && apt-get install -y libnss3 nss-plugin-pem # Copy curl-impersonate from the builder image COPY --from=builder /build/install /usr/local +# Update the loader's cache +RUN ldconfig # Copy to /build/out as well for backward compatibility with previous versions. COPY --from=builder /build/out /build/out # Wrapper scripts diff --git a/firefox/Dockerfile.alpine b/firefox/Dockerfile.alpine index c1d00ac1..438726f6 100644 --- a/firefox/Dockerfile.alpine +++ b/firefox/Dockerfile.alpine @@ -136,5 +136,7 @@ RUN apk add --no-cache nss # Copy curl-impersonate from the builder image COPY --from=builder /build/install /usr/local +# Update the loader's cache +RUN ldconfig # Wrapper scripts COPY --from=builder /build/out/curl_* /usr/local/bin/ From c90b33a605d478f17b8e3843b82bdda642a8a545 Mon Sep 17 00:00:00 2001 From: lwthiker Date: Fri, 29 Jul 2022 21:58:38 +0300 Subject: [PATCH 4/5] Run ldconfig in Debian Docker images only --- Dockerfile.template | 2 +- chrome/Dockerfile.alpine | 2 -- firefox/Dockerfile.alpine | 2 -- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/Dockerfile.template b/Dockerfile.template index 2833dc66..044bfc3d 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -249,9 +249,9 @@ RUN apt-get update && apt-get install -y libnss3 nss-plugin-pem # Copy curl-impersonate from the builder image COPY --from=builder /build/install /usr/local +{{#debian}} # Update the loader's cache RUN ldconfig -{{#debian}} # Copy to /build/out as well for backward compatibility with previous versions. COPY --from=builder /build/out /build/out {{/debian}} diff --git a/chrome/Dockerfile.alpine b/chrome/Dockerfile.alpine index 11bb6154..9188d284 100644 --- a/chrome/Dockerfile.alpine +++ b/chrome/Dockerfile.alpine @@ -140,7 +140,5 @@ FROM alpine:3.15.0 # Copy curl-impersonate from the builder image COPY --from=builder /build/install /usr/local -# Update the loader's cache -RUN ldconfig # Wrapper scripts COPY --from=builder /build/out/curl_* /usr/local/bin/ diff --git a/firefox/Dockerfile.alpine b/firefox/Dockerfile.alpine index 438726f6..c1d00ac1 100644 --- a/firefox/Dockerfile.alpine +++ b/firefox/Dockerfile.alpine @@ -136,7 +136,5 @@ RUN apk add --no-cache nss # Copy curl-impersonate from the builder image COPY --from=builder /build/install /usr/local -# Update the loader's cache -RUN ldconfig # Wrapper scripts COPY --from=builder /build/out/curl_* /usr/local/bin/ From a87d3d9d521a44fecbc87bf474a8073da6b0a5eb Mon Sep 17 00:00:00 2001 From: lwthiker Date: Sat, 30 Jul 2022 09:37:23 +0300 Subject: [PATCH 5/5] Install ca-certificates in the Debian containers --- Dockerfile.template | 4 ++-- chrome/Dockerfile | 2 +- chrome/Dockerfile.alpine | 1 - firefox/Dockerfile | 4 ++-- firefox/Dockerfile.alpine | 1 - 5 files changed, 5 insertions(+), 7 deletions(-) diff --git a/Dockerfile.template b/Dockerfile.template index 044bfc3d..08800745 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -239,14 +239,14 @@ RUN apk add --no-cache nss {{/alpine}} {{#debian}} FROM debian:buster-slim +RUN apt-get update && apt-get install -y ca-certificates {{#firefox}} # curl tries to load the CA certificates for libnss. # It loads them from /usr/lib/libnssckbi.so and /usr/lib/libnsspem.so, # which are supplied by 'libnss3' and 'nss-plugin-pem' on debian. -RUN apt-get update && apt-get install -y libnss3 nss-plugin-pem +RUN apt-get install -y libnss3 nss-plugin-pem {{/firefox}} {{/debian}} - # Copy curl-impersonate from the builder image COPY --from=builder /build/install /usr/local {{#debian}} diff --git a/chrome/Dockerfile b/chrome/Dockerfile index 7a40abe6..e2c1b1f7 100644 --- a/chrome/Dockerfile +++ b/chrome/Dockerfile @@ -138,7 +138,7 @@ RUN chmod +x out/curl_* # Create a final, minimal image with the compiled binaries # only. FROM debian:buster-slim - +RUN apt-get update && apt-get install -y ca-certificates # Copy curl-impersonate from the builder image COPY --from=builder /build/install /usr/local # Update the loader's cache diff --git a/chrome/Dockerfile.alpine b/chrome/Dockerfile.alpine index 9188d284..b5004f2b 100644 --- a/chrome/Dockerfile.alpine +++ b/chrome/Dockerfile.alpine @@ -137,7 +137,6 @@ RUN chmod +x out/curl_* # Create a final, minimal image with the compiled binaries # only. FROM alpine:3.15.0 - # Copy curl-impersonate from the builder image COPY --from=builder /build/install /usr/local # Wrapper scripts diff --git a/firefox/Dockerfile b/firefox/Dockerfile index ffa450db..f34bc0b9 100644 --- a/firefox/Dockerfile +++ b/firefox/Dockerfile @@ -134,11 +134,11 @@ RUN chmod +x out/curl_* # Create a final, minimal image with the compiled binaries # only. FROM debian:buster-slim +RUN apt-get update && apt-get install -y ca-certificates # curl tries to load the CA certificates for libnss. # It loads them from /usr/lib/libnssckbi.so and /usr/lib/libnsspem.so, # which are supplied by 'libnss3' and 'nss-plugin-pem' on debian. -RUN apt-get update && apt-get install -y libnss3 nss-plugin-pem - +RUN apt-get install -y libnss3 nss-plugin-pem # Copy curl-impersonate from the builder image COPY --from=builder /build/install /usr/local # Update the loader's cache diff --git a/firefox/Dockerfile.alpine b/firefox/Dockerfile.alpine index c1d00ac1..afb5d581 100644 --- a/firefox/Dockerfile.alpine +++ b/firefox/Dockerfile.alpine @@ -133,7 +133,6 @@ FROM alpine:3.15.0 # It loads them from /usr/lib/libnssckbi.so, # which is supplied by 'nss' on alpine. RUN apk add --no-cache nss - # Copy curl-impersonate from the builder image COPY --from=builder /build/install /usr/local # Wrapper scripts