From c8f0532c17effe2256af7c9b44adf17012e7a963 Mon Sep 17 00:00:00 2001 From: Yurii Kovalchuk <103324562+yumirkov@users.noreply.github.com> Date: Wed, 6 Mar 2024 14:55:27 +0200 Subject: [PATCH] ATOR-173 - Bandwidth authority setup, docker support and CI (#1) --- .github/actions/build-and-push/action.yml | 34 ++++ .github/actions/deploy/action.yml | 24 +++ .github/workflows/dev-deploy.yml | 28 +++ .github/workflows/stage-deploy.yml | 28 +++ .gitignore | 2 + docker/destination/Dockerfile | 7 + docker/destination/default.conf | 16 ++ docker/destination/docker-entrypoint.sh | 5 + docker/docker-compose.yml | 42 +++++ docker/relay/anonrc | 19 ++ docker/scanner/.sbws.ini | 33 ++++ docker/scanner/Dockerfile | 17 ++ docker/scanner/docker-entrypoint.sh | 8 + docs/source/config_tor.rst | 3 +- operations/admin-ui-ca.crt | 31 +++ operations/deploy-dev.hcl | 219 ++++++++++++++++++++++ operations/deploy-stage.hcl | 219 ++++++++++++++++++++++ sbws/config.default.ini | 1 + sbws/globals.py | 4 +- sbws/util/config.py | 6 +- sbws/util/stem.py | 8 +- scripts/tools/get-per-relay-budget.py | 3 +- 22 files changed, 749 insertions(+), 8 deletions(-) create mode 100644 .github/actions/build-and-push/action.yml create mode 100644 .github/actions/deploy/action.yml create mode 100644 .github/workflows/dev-deploy.yml create mode 100644 .github/workflows/stage-deploy.yml create mode 100755 docker/destination/Dockerfile create mode 100644 docker/destination/default.conf create mode 100755 docker/destination/docker-entrypoint.sh create mode 100755 docker/docker-compose.yml create mode 100644 docker/relay/anonrc create mode 100755 docker/scanner/.sbws.ini create mode 100755 docker/scanner/Dockerfile create mode 100755 docker/scanner/docker-entrypoint.sh create mode 100644 operations/admin-ui-ca.crt create mode 100644 operations/deploy-dev.hcl create mode 100644 operations/deploy-stage.hcl diff --git a/.github/actions/build-and-push/action.yml b/.github/actions/build-and-push/action.yml new file mode 100644 index 00000000..4bfa14dd --- /dev/null +++ b/.github/actions/build-and-push/action.yml @@ -0,0 +1,34 @@ +name: Build and Push Docker Image +inputs: + docker-username: + required: true + docker-password: + required: true + docker-tag: + required: true + +runs: + using: "composite" + steps: + - name: Log in to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ inputs.docker-username }} + password: ${{ inputs.docker-password }} + + - name: Build and push Docker image of scanner + uses: docker/build-push-action@v2 + with: + context: . + file: ./docker/scanner/Dockerfile + push: true + tags: ${{ inputs.docker-username }}/sbws-scanner:${{ github.sha }},${{ inputs.docker-username }}/sbws-scanner:${{ inputs.docker-tag }} + + - name: Build and push Docker image of destination + uses: docker/build-push-action@v2 + with: + context: ./docker/destination + file: ./docker/destination/Dockerfile + push: true + tags: ${{ inputs.docker-username }}/sbws-destination:${{ github.sha }},${{ inputs.docker-username }}/sbws-destination:${{ inputs.docker-tag }} + diff --git a/.github/actions/deploy/action.yml b/.github/actions/deploy/action.yml new file mode 100644 index 00000000..42728d8b --- /dev/null +++ b/.github/actions/deploy/action.yml @@ -0,0 +1,24 @@ +name: 'Deploy new version' +inputs: + environment: + required: true + nomad-cacert: + required: true + nomad-token: + required: true + nomad-addr: + required: true + +runs: + using: "composite" + steps: + - name: Deploy new version + shell: bash + env: + NOMAD_CACERT: ${{ inputs.nomad-cacert }} + NOMAD_TOKEN: ${{ inputs.nomad-token }} + NOMAD_ADDR: ${{ inputs.nomad-addr }} + run: | + curl -L https://releases.hashicorp.com/nomad/1.6.3/nomad_1.6.3_linux_amd64.zip -o nomad.zip + unzip nomad.zip + ./nomad job run operations/deploy-${{ inputs.environment }}.hcl diff --git a/.github/workflows/dev-deploy.yml b/.github/workflows/dev-deploy.yml new file mode 100644 index 00000000..978cade6 --- /dev/null +++ b/.github/workflows/dev-deploy.yml @@ -0,0 +1,28 @@ +name: Build and Push Docker Image + +on: + push: + branches: + - development + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Check out the repo + uses: actions/checkout@v4 + - name: Build and push + uses: ./.github/actions/build-and-push + with: + docker-username: ${{ secrets.DOCKER_HUB_USERNAME }} + docker-password: ${{ secrets.DOCKER_HUB_PASSWORD }} + docker-tag: latest-dev + - name: Deploy + uses: ./.github/actions/deploy + with: + environment: dev + nomad-cacert: operations/admin-ui-ca.crt + nomad-token: ${{ secrets.NOMAD_TOKEN_SBWS_DEPLOY }} + nomad-addr: ${{ secrets.NOMAD_DEPLOY_ADDR }} diff --git a/.github/workflows/stage-deploy.yml b/.github/workflows/stage-deploy.yml new file mode 100644 index 00000000..8d55ade6 --- /dev/null +++ b/.github/workflows/stage-deploy.yml @@ -0,0 +1,28 @@ +name: Build and Push Docker Image + +on: + push: + branches: + - main + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Check out the repo + uses: actions/checkout@v4 + - name: Build and push + uses: ./.github/actions/build-and-push + with: + docker-username: ${{ secrets.DOCKER_HUB_USERNAME }} + docker-password: ${{ secrets.DOCKER_HUB_PASSWORD }} + docker-tag: latest + - name: Deploy + uses: ./.github/actions/deploy + with: + environment: stage + nomad-cacert: operations/admin-ui-ca.crt + nomad-token: ${{ secrets.NOMAD_TOKEN_SBWS_DEPLOY }} + nomad-addr: ${{ secrets.NOMAD_DEPLOY_ADDR }} diff --git a/.gitignore b/.gitignore index 744436b7..ec4cd4ec 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ dist build *.lockfile chutney +.idea +docker/data diff --git a/docker/destination/Dockerfile b/docker/destination/Dockerfile new file mode 100755 index 00000000..7fba28c3 --- /dev/null +++ b/docker/destination/Dockerfile @@ -0,0 +1,7 @@ +FROM nginx + +WORKDIR /app/destination + +COPY docker-entrypoint.sh /app/destination + +ENTRYPOINT [ "sh", "docker-entrypoint.sh" ] diff --git a/docker/destination/default.conf b/docker/destination/default.conf new file mode 100644 index 00000000..cb9e54f3 --- /dev/null +++ b/docker/destination/default.conf @@ -0,0 +1,16 @@ +server { + + root /app/destination/data; + + autoindex on; + + listen 0.0.0.0:80; + + location / { + try_files $uri $uri/ =404; + } + + location ~/\.ht { + deny all; + } +} diff --git a/docker/destination/docker-entrypoint.sh b/docker/destination/docker-entrypoint.sh new file mode 100755 index 00000000..28d5660b --- /dev/null +++ b/docker/destination/docker-entrypoint.sh @@ -0,0 +1,5 @@ +mkdir -p data && cd data + +head -c $((1024*1024*1024)) /dev/urandom > 1GiB + +nginx -g 'daemon off;' diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100755 index 00000000..f421f529 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,42 @@ +version: '2.2' +services: + sbws-relay: + image: svforte/anon-dev + restart: always + expose: + - "9051" + volumes: + - ./data/:/var/lib/anon + - ./relay/anonrc:/etc/anon/anonrc + networks: + local: + ipv4_address: 172.18.0.2 + sbws-scanner: + image: sbws-scanner + restart: always + depends_on: + - sbws-relay + volumes: + - ./scanner/.sbws.ini:/root/.sbws.ini + - ./data:/root/.sbws + networks: + local: + ipv4_address: 172.18.0.3 + sbws-destination: + image: sbws-destination + restart: always + ports: + - "8888:80" + volumes: + - ./destination/default.conf:/etc/nginx/conf.d/default.conf + networks: + local: + ipv4_address: 172.18.0.4 + +networks: + local: + ipam: + driver: default + config: + - subnet: "172.18.0.0/24" + gateway: "172.18.0.1" diff --git a/docker/relay/anonrc b/docker/relay/anonrc new file mode 100644 index 00000000..f46a3528 --- /dev/null +++ b/docker/relay/anonrc @@ -0,0 +1,19 @@ +User anond + +Nickname AnonSBWS + +DataDirectory /var/lib/anon + +ControlPort 0.0.0.0:9051 +HashedControlPassword 16:3ACE689A3BC1B7D06025EA6BC9CB1C9B99EB21FE4877ECD803E6EAD9BE + +SocksPort auto +SafeLogging 1 +UseEntryGuards 0 +ProtocolWarnings 1 +FetchDirInfoEarly 1 +LogTimeGranularity 1 +UseMicrodescriptors 0 +FetchDirInfoExtraEarly 1 +FetchUselessDescriptors 1 +LearnCircuitBuildTimeout 0 diff --git a/docker/scanner/.sbws.ini b/docker/scanner/.sbws.ini new file mode 100755 index 00000000..fbf5b75c --- /dev/null +++ b/docker/scanner/.sbws.ini @@ -0,0 +1,33 @@ +# Minimum configuration that needs to be customized +[scanner] +# ISO 3166-1 alpha-2 country code where the scanner is located. +# Default AA, to detect it was not edited. +country = ZZ +# A human-readable string with chars in a-zA-Z0-9 to identify the dirauth +# nickname that will publish the BandwidthFiles generated from this scanner. +# Default to a non existing dirauth_nickname to detect it was not edited. +dirauth_nickname = Anon + +[destinations] +# With several destinations, the scanner can continue even if some of them +# fail, which can be caused by a network problem on their side. +# If all of them fail, the scanner will stop, which +# will happen if there is network problem on the scanner side. + +# A destination can be disabled changing `on` by `off` +foo = on + +[destinations.foo] +# the domain and path to the 1GB file or POST URL. +url = http://5.161.108.187:9177/1GiB +# Whether to verify or not the TLS certificate. Default True +verify = False +# ISO 3166-1 alpha-2 country code where the Web server destination is located. +# Default AA, to detect it was not edited. +# Use ZZ if the location is unknown (for instance, a CDN). +country = ZZ + +[tor] +datadir = docker/data +external_control_ip = 172.18.0.2 +external_control_port = 9051 diff --git a/docker/scanner/Dockerfile b/docker/scanner/Dockerfile new file mode 100755 index 00000000..51cc9360 --- /dev/null +++ b/docker/scanner/Dockerfile @@ -0,0 +1,17 @@ +FROM debian:bookworm + +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive apt install -y \ + python3-dateutil python3-stem pip cron + +WORKDIR /app/scanner + +ADD ../.. /app/scanner/temp + +RUN cd temp && ls -la && \ + pip install . --break-system-packages && \ + cp docker/scanner/docker-entrypoint.sh .. && \ + cd .. && \ + rm -rf temp + +ENTRYPOINT [ "sh", "docker-entrypoint.sh" ] diff --git a/docker/scanner/docker-entrypoint.sh b/docker/scanner/docker-entrypoint.sh new file mode 100755 index 00000000..be656771 --- /dev/null +++ b/docker/scanner/docker-entrypoint.sh @@ -0,0 +1,8 @@ +mkdir -p /root/.sbws/log + +crontab -l | { cat; echo "*/5 * * * * /usr/local/bin/sbws -c /root/.sbws.ini generate >> /root/.sbws/log/generate.log 2>&1"; } | crontab - +crontab -l | { cat; echo "30 0 * * * /usr/local/bin/sbws -c /root/.sbws.ini cleanup >> /root/.sbws/log/cleanup.log 2>&1"; } | crontab - + +service cron start + +sbws scanner diff --git a/docs/source/config_tor.rst b/docs/source/config_tor.rst index e6094687..35bd9add 100644 --- a/docs/source/config_tor.rst +++ b/docs/source/config_tor.rst @@ -10,7 +10,8 @@ connection to an existing Tor daemon. Default configuration: - ``SocksPort auto``: To proxy requests over Tor. -- ``CookieAuthentication 1``: The easiest way to authenticate to Tor. +- ``CookieAuthentication 0``: The easiest way to authenticate to Tor. +- ``HashedControlPassword``: The how to authenticate using password - ``UseEntryGuards 0``: To avoid path bias warnings. - ``UseMicrodescriptors 0``: Because full server descriptors are needed. - ``SafeLogging 0``: Useful for logging, since there's no need for anonymity. diff --git a/operations/admin-ui-ca.crt b/operations/admin-ui-ca.crt new file mode 100644 index 00000000..939344fb --- /dev/null +++ b/operations/admin-ui-ca.crt @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIIFVzCCAz+gAwIBAgIUdUo5LO+M2vEiSyfvDSOhKxVBko0wDQYJKoZIhvcNAQEL +BQAwOzELMAkGA1UEBhMCTk8xETAPBgNVBAoMCEF0b3IgTExDMRkwFwYDVQQDDBBB +dG9yIEFkbWluIFVJIENBMB4XDTIzMDUxMDE0NTk0M1oXDTQzMDIyNDE0NTk0M1ow +OzELMAkGA1UEBhMCTk8xETAPBgNVBAoMCEF0b3IgTExDMRkwFwYDVQQDDBBBdG9y +IEFkbWluIFVJIENBMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA1gcK +cqC81T4YHcu3DfoSA2ghART6ImQSStXhuJfWqJPjJG0PtT58t3OJ6jS84zxTWPoZ +e2gfcA4VunW5+gTTQVra8gBdNthT4jOZJ9TYS54kSRJ9st7ZPctOYzD97NXeMeWp +TCgtkoGhaVLBeFE2z8xI2V+coIs00WG+GyC1St4DlmbezKUrpODWdfiRZfYa3lNy +uamPLY3nsDq8APVq4BKrauAxfnVmeItIdjC82KhCykeWyVVOrj7PeXbfWus79YON +KuXsLbFnSR1xO5QEtaYk3A3XXN81Xe0i5qNoWJAczYIrPLAsjIo+fDw/8ENoC9+j +SfS6gNdgd0A2Le31J5Bd76QZipDBa9+5EhZNGK/19qfypTzayRT4JpErAa8zKGIy +V9csdacobuQ+0UuU+li8EavWl8vFUSb1Uh1cnhudhr2NbAiuqjwwffOmgcTnOESg +cbAXPUhGWSQU2DyrFiDebo/HbiFZGzKxL7FcoK1j08gqcMzc+3gIF4uCKkMM+rk0 +5GWNsp3VoQv9E5ytYYbN8Lk1yxRY3KWhISiDmK/cS3FW4LRrcS2H+Se+hWLNyHDN +P0yQXQh068DdCjZAQYOeXcQWKBAI1kcwu6KyidOpZKwhCZKbMGHsoURhKsfL9NEh +HDjQXrPXJ7d++Akmek2Eu9tXz7sVFJE7mB3SE0cCAwEAAaNTMFEwHQYDVR0OBBYE +FJAGJ5Vj7nX/cRLIKorIuudZoHcbMB8GA1UdIwQYMBaAFJAGJ5Vj7nX/cRLIKorI +uudZoHcbMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIBAFk8SYO0 +w0RHpCfrytF/rFTZUtIPNnol3/PRMq6WYT/fmTLhpt8S8+Yuz6YykRzVf4JI4LT8 +Y5nbZV/5fynbCfOP3pBWl/C7npdQtVsGVGVX7Hh1M+8Q9RKl+NA8gyHxKYfXJtEr +c6sSewBJ62eARP4dztbO8T/ydvfI6VJyKbJhGeO7vU+p13qbYAVMYeQ3sUjpazKZ +haqtTRp2lNDKK4SJ2mxW+pnac8S8E88+LP1O8U0hZphddfRgmwjuzeu+remwjkF8 +obreQRk2BR58v+tGTxfxHT0XJSK9b+G7QRmEKy+V4hXZPcynF5Xbdqu8fT7cVF6s +TfL9dCDMQoBsiWElFllSmBruWvp5gv086fwObFeQfw94Y3qvFawLMPUdI76uYPlR +3VuAs5MFncohOL12+/2o4nST2v4WKDLoWxcze49r6iCFMe2UEhhuydEXnQHp7el9 +w1NMfXxH6PzOAvMG1WtlkyT45agAIETW76/SSJH8e9j21m6cXcaaK6kJvhPga1X7 +Cd3XaiTg1r39S7Nxm/g8xDmVV738lguuj3TN5RwbNfR/rNcVUG5VUNDGea/bzQLl +eyjkXk1rFn5mYTf8tCcFfvu7pcH0Ds902zZvdysvaMGCxQu3bVpdkSn+3/6dUNCA +FFWzsbx0wrWxCyFPedz8OWgtdYu1TMtvXrWE +-----END CERTIFICATE----- diff --git a/operations/deploy-dev.hcl b/operations/deploy-dev.hcl new file mode 100644 index 00000000..aaa723ba --- /dev/null +++ b/operations/deploy-dev.hcl @@ -0,0 +1,219 @@ +job "sbws-dev" { + datacenters = ["ator-fin"] + type = "service" + namespace = "ator-network" + + group "sbws-dev-group" { + count = 3 + + spread { + attribute = "${node.unique.id}" + weight = 100 + target "067a42a8-d8fe-8b19-5851-43079e0eabb4" { + percent = 34 + } + target "16be0723-edc1-83c4-6c02-193d96ec308a" { + percent = 33 + } + target "e6e0baed-8402-fd5c-7a15-8dd49e7b60d9" { + percent = 33 + } + } + + volume "sbws-dev" { + type = "host" + read_only = false + source = "sbws-dev" + } + + network { + mode = "bridge" + + port "http-port" { + static = 9077 + to = 80 + # host_network = "wireguard" + } + + port "control-port" { + static = 9051 + host_network = "wireguard" + } + } + + task "sbws-relay-dev-task" { + driver = "docker" + + env { + ANON_USER = "root" + } + + volume_mount { + volume = "sbws-dev" + destination = "/var/lib/anon" + read_only = false + } + + config { + image = "svforte/anon-dev" + force_pull = true + volumes = [ + "local/anonrc:/etc/anon/anonrc" + ] + } + + resources { + cpu = 256 + memory = 256 + } + + template { + change_mode = "noop" + data = <