[CU-1y23865] [CU-1y23865] [CU-1yykqgq] Add a playbook to run polkadot…

…-launch and integration tests (#417) * CU-1y23865 - Add a playbook to run polkadot-launch and integration tests * CU-1y23865 Add TLS termination for WebSocket endpoints * CU-1yykqgq - Use build artifacts to deploy a local cluster of Polkadot
ComposableFi · Jan 4, 2022 · 80bbbc3 · 80bbbc3
1 parent 9de048d
commit 80bbbc3
Show file tree

Hide file tree

Showing 5 changed files with 406 additions and 4 deletions.
diff --git a/.maintain/playbooks/README.md b/.maintain/playbooks/README.md
@@ -0,0 +1,30 @@
+# run-integration-tests.yml
+
+## Description
+
+This playbook does next:
+* downloads `composable`, `basilisk` and `polkadot`
+* installs and configures [certbot](https://certbot.eff.org/) to get a [Let’s Encrypt](https://letsencrypt.org/) certificate for your domain
+* installs and configures [nginx](https://www.nginx.com/) to add a TLS termination using a [Let’s Encrypt](https://letsencrypt.org/) certificate
+* runs local cluster of Polkadot with configured `composable` and `basilisk` parachains using [polkadot-launch](https://github.com/paritytech/polkadot-launch)
+* runs [initialization script](https://github.com/ComposableFi/composable/tree/main/scripts/polkadot-launch/initialization) to add assets mappings in `composable` and `basilisk` parachains
+
+
+## Usage
+
+1. Create a VPS in your cloud
+2. Create an A-type record in your DNS server referred to the external IP of your VPS created on 1st step
+3. Add VPS in your inventory file
+4. Run playbook: 
+
+    ```bash
+    ansible-playbook -i path_to_inventory .maintain/playbooks/run-integration-tests.yml -e "target=your_vps" -e "domain=domain_of_vps" -e "[email protected]" -e "github_user=your_github_account" -e "github_password=your_github_password_or_token"
+    ```
+
+## URLs
+
+|           Node           |            URL           |
+|:-------------------------|:------------------------:|
+| Relay Chain #1           | wss://domain_of_vps:9901 |
+| Composable's collator #1 | wss://domain_of_vps:9902 |
+| Basilisk's collator #1   | wss://domain_of_vps:9903 |
diff --git a/.maintain/playbooks/run-integration-tests.yml b/.maintain/playbooks/run-integration-tests.yml
@@ -0,0 +1,239 @@
+---
+- hosts: "{{ target }}"
+  vars:
+    user: service
+    group: service
+    work_dir: /srv/composable-sandbox
+  tasks:
+
+  - name: Initialization
+    block:
+    - name: Create a group
+      group:
+        name: "{{ user }}"
+        state: present
+      become: yes
+
+    - name: Create a user
+      user:
+        name: "{{ user }}"
+        state: present
+        shell: /bin/bash
+        home: "/home/{{ user }}"
+        group: "{{ group }}"
+      become: yes
+
+    - name: Create a work directory
+      ansible.builtin.file:
+        path: "{{ work_dir }}"
+        state: directory
+        mode: "0755"
+        owner: "{{ user }}"
+        group: "{{ group }}"
+      become: yes
+
+    - name: Create systemd service
+      template:
+        src: systemd/composable-sandbox.service.j2
+        dest: /etc/systemd/system/composable-sandbox.service
+        mode: "0600"
+      become: yes
+
+  - name: TLS temination for WS endpoints
+    block:
+    - name: Install nginx and certbot
+      apt:
+        pkg:
+        - certbot
+        - nginx
+        - python3-certbot-nginx
+
+    - name: Request initial letsencrypt certificate
+      command: certbot certonly --nginx --agree-tos -d '{{ domain }}' -m '{{ letsencrypt_contact_email }}'
+      args:
+        creates: "/etc/letsencrypt/live/{{ domain }}/privkey.pem"
+
+    - name: Add nginx config
+      template:
+        src: nginx/composable-sandbox.conf.j2
+        dest: /etc/nginx/sites-enabled/composable-sandbox.conf
+        mode: "0600"
+      become: yes
+
+    - name: Enable service nginx and ensure it is not masked
+      ansible.builtin.systemd:
+        name: nginx
+        state: started
+        enabled: yes
+        masked: no
+
+    - name: Reload nginx with new config
+      shell: nginx -s reload
+
+    - name: Certbot renewal cronjob
+      cron: special_time=daily
+        name=certbot-renew-composable-sandbox
+        user=root
+        job="certbot certonly --nginx -d '{{ domain }}' --deploy-hook 'nginx -s reload'"
+
+  - name: Install dependencies
+    block:
+    - name: Update repositories cache and install apt-transport-https
+      apt:
+        name: apt-transport-https
+        update_cache: yes
+      become: yes
+
+    - name: Add Yarn apt key
+      apt_key:
+        url: https://dl.yarnpkg.com/debian/pubkey.gpg
+      become: yes
+
+    - name: Add Yarn repository
+      apt_repository:
+        repo: "deb https://dl.yarnpkg.com/debian/ stable main"
+        filename: yarn
+      become: yes
+
+    - name: Download Node v16 installer
+      get_url:
+        url: https://deb.nodesource.com/setup_16.x
+        dest: "/tmp/setup_node.sh"
+        mode: '0440'
+      become: yes
+
+    - name: Add Node repository
+      ansible.builtin.shell: |
+        cat /tmp/setup_node.sh | bash
+      become: yes
+
+    - name: Update repositories cache and install nodejs and yarn
+      apt:
+        pkg:
+        - yarn
+        - nodejs
+        update_cache: yes
+        install_recommends: no
+      become: yes
+
+  - name: Build and restart all
+    block:
+    - name: Download basilisk, composable and polkadot
+      get_url:
+        url: "{{ item.url }}"
+        dest: "{{ item.dest }}"
+        mode: '0440'
+      become: yes
+      become_user: "{{ user }}"
+      loop:
+      - url: https://github.com/galacticcouncil/Basilisk-node/releases/download/v5.0.3/basilisk
+        dest: "/home/{{ user }}/basilisk"
+      - url: https://storage.googleapis.com/composable-binaries/testnet-releases/picasso/composable-latest.tar.gz
+        dest: "/home/{{ user }}/composable-latest.tar.gz"
+      - url: https://github.com/paritytech/polkadot/releases/download/v0.9.13/polkadot
+        dest: "/home/{{ user }}/polkadot"
+
+    - name: Clone composable repo
+      ansible.builtin.git:
+        repo: "https://{{ github_user | urlencode }}:{{ github_password | urlencode }}@github.com/ComposableFi/composable.git"
+        dest: "{{ work_dir }}/composable"
+        version: main
+        force: true
+      become: yes
+      become_user: "{{ user }}"
+
+    - name: Create directories if it do not exist
+      ansible.builtin.file:
+        path: "{{ item.dir }}"
+        state: directory
+        mode: "0775"
+        owner: "{{ user }}"
+        group: "{{ group }}"
+      loop:
+      - { dir: "{{ work_dir }}/Basilisk-node/target/release" }
+      - { dir: "{{ work_dir }}/composable/target/release" }
+      - { dir: "{{ work_dir }}/polkadot/target/release" }
+
+    - name: "Extract /home/{{ user }}/composable-latest.tar.gz into /home/{{ user }}/composable"
+      ansible.builtin.unarchive:
+        src: "/home/{{ user }}/composable-latest.tar.gz"
+        dest: "/home/{{ user }}"
+        remote_src: yes
+
+    - name: Copy files with owner and permissions
+      ansible.builtin.copy:
+        src: "{{ item.src }}"
+        dest: "{{ item.dest }}"
+        owner: "{{ user }}"
+        group: "{{ group }}"
+        mode: "0775"
+        remote_src: true
+      loop:
+      - { src: "/home/{{ user }}/basilisk", dest: "{{ work_dir }}/Basilisk-node/target/release" }
+      - { src: "/home/{{ user }}/target/release/composable", dest: "{{ work_dir }}/composable/target/release" }
+      - { src: "/home/{{ user }}/polkadot", dest: "{{ work_dir }}/polkadot/target/release" }
+
+    - name: Remove downloaded files
+      ansible.builtin.file:
+        path: "{{ item.file_path }}"
+        state: absent
+      loop:
+      - { file_path: "/home/{{ user }}/basilisk" }
+      - { file_path: "/home/{{ user }}/composable-latest.tar.gz" }
+      - { file_path: "/home/{{ user }}/polkadot" }
+      - { file_path: "/home/{{ user }}/target" }
+
+    - name: Install dependencies of polkadot-launch
+      ansible.builtin.shell: yarn
+      args:
+        chdir: "{{ work_dir }}/composable/scripts/polkadot-launch"
+        executable: /bin/bash
+      become: yes
+      become_user: "{{ user }}"
+
+    - name: Install dependencies of integration tests and build integraion tests
+      ansible.builtin.shell: |
+        yarn
+        yarn build
+      args:
+        chdir: "{{ work_dir }}/composable/scripts/polkadot-launch/initialization"
+        executable: /bin/bash
+      become: yes
+      become_user: "{{ user }}"
+
+    - name: Stop composable-sandbox service, if running
+      ansible.builtin.systemd:
+        name: composable-sandbox
+        state: stopped
+        daemon_reload: yes
+        enabled: true
+      become: yes
+
+    - name: Start service composable-sandbox service
+      ansible.builtin.systemd:
+        name: composable-sandbox
+        state: started
+        daemon_reload: yes
+      become: yes
+
+  - name: Wait until the string " POLKADOT LAUNCH COMPLETE " is in the file /var/log/syslog before continuing
+    ansible.builtin.shell: |
+      tail -f /var/log/syslog | sed '/\sPOLKADOT LAUNCH COMPLETE\s/ q'
+    args:
+      executable: /bin/bash
+    become: yes
+    async: 1200
+    poll: 5
+
+  - name: Run integration tests
+    block:
+    - name: Add assets mappings
+      ansible.builtin.shell: |
+        yarn start >log 2>err
+      args:
+        chdir: "{{ work_dir }}/composable/scripts/polkadot-launch/initialization"
+        executable: /bin/bash
+      become: yes
+      become_user: "{{ user }}"
+      async: 1800
+      poll: 5
diff --git a/.maintain/playbooks/templates/nginx/composable-sandbox.conf.j2 b/.maintain/playbooks/templates/nginx/composable-sandbox.conf.j2
@@ -0,0 +1,112 @@
+server {
+    listen 80;
+    server_name {{ domain }};
+    location /.well-known/acme-challenge/ {
+        root /var/www/certbot;
+    }
+    location / {
+        return 301 https://$host$request_uri;
+    }
+}
+
+server {
+    listen 9901 ssl http2;
+    server_name {{ domain }};
+
+    ssl_certificate /etc/letsencrypt/live/{{ domain }}/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/{{ domain }}/privkey.pem;
+
+    # Various TLS hardening settings
+    # https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html
+    ssl_protocols TLSv1.2 TLSv1.3;
+    ssl_prefer_server_ciphers on;
+    ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256';
+    ssl_session_timeout  10m;
+    ssl_session_cache shared:SSL:10m;
+    ssl_session_tickets off;
+    ssl_stapling on;
+    ssl_stapling_verify on;
+
+    # Hide nginx version
+    server_tokens off;
+
+    location / {
+        proxy_pass http://127.0.0.1:9944;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header Host $host;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+        # WebSocket support
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+    }
+}
+
+server {
+    listen 9902 ssl http2;
+    server_name {{ domain }};
+
+    ssl_certificate /etc/letsencrypt/live/{{ domain }}/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/{{ domain }}/privkey.pem;
+
+    # Various TLS hardening settings
+    # https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html
+    ssl_protocols TLSv1.2 TLSv1.3;
+    ssl_prefer_server_ciphers on;
+    ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256';
+    ssl_session_timeout  10m;
+    ssl_session_cache shared:SSL:10m;
+    ssl_session_tickets off;
+    ssl_stapling on;
+    ssl_stapling_verify on;
+
+    # Hide nginx version
+    server_tokens off;
+
+    location / {
+        proxy_pass http://127.0.0.1:9988;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header Host $host;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+        # WebSocket support
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+    }
+}
+
+server {
+    listen 9903 ssl http2;
+    server_name {{ domain }};
+
+    ssl_certificate /etc/letsencrypt/live/{{ domain }}/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/{{ domain }}/privkey.pem;
+
+    # Various TLS hardening settings
+    # https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html
+    ssl_protocols TLSv1.2 TLSv1.3;
+    ssl_prefer_server_ciphers on;
+    ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256';
+    ssl_session_timeout  10m;
+    ssl_session_cache shared:SSL:10m;
+    ssl_session_tickets off;
+    ssl_stapling on;
+    ssl_stapling_verify on;
+
+    # Hide nginx version
+    server_tokens off;
+
+    location / {
+        proxy_pass http://127.0.0.1:9998;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header Host $host;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+        # WebSocket support
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+    }
+}