[ci][feat][build] Major updates for devnet (#146)

[ci] Update ci_e2e_test workflow triggered manually for now this workflow is crucial to run e2e when updating ethermint/go-ethereum/evmos [ci] Update run_e2e_test this workflow is useful to run the e2e test with an existing evmos-node docker image but new zbc-solidity version (faster than ci_e2e_test [build] Update makefile to have a local build from source, .i.e make build-local [build] Update makefile to have a local build from source using docker image, .i.e make build-docker (LOCAL_BUILD=true) [build] Update makefile to have a running local evmos-node from registry, .i.e make init-evmos-node (LOCAL_BUILD=false) [build] Update makefile to run e2e test using locally built evmos-node docker image, .i.e make e2e-test (LOCAL_BUILD=true) [build] Update makefile to run e2e test using evmos-node from registry, .i.e make e2e-test (LOCAL_BUILD=false) [feat] Use the last version of go-etherum that includes compact inputs and fhe keys (mainly pks) output mechanism based on elliptic curve fhe types (uint8, uint16, uint32 ...)
zama-ai · Jun 22, 2023 · b96e9d1 · b96e9d1
1 parent be4e092
commit b96e9d1
Show file tree

Hide file tree

Showing 34 changed files with 1,660 additions and 565 deletions.
diff --git a/.env b/.env
@@ -0,0 +1,8 @@
+# TFHE_RS_VERSION=0.2.1
+# TFHE_RS_PATH=../tfhe-rs
+# ZBC_DEVELOPMENT_PATH=../zbc-development
+# ZBC_SOLIDITY_PATH=../zbc-solidity
+# ZBC_FHE_TOOL_PATH=../zbc-fhe-tool
+#LOCAL_BUILD=false
+LOCAL_BUILD=true
+GOPRIVATE=github.com/zama-ai/*
diff --git a/.github/workflows/ci_e2e_test.yml b/.github/workflows/ci_e2e_test.yml
@@ -7,3 +7,140 @@ on:
         description: 'Oracle Db service image tag (default: latest)'
         type: string
         required: false
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    env:
+      ORACLE_DB_SERVICE_TAG: ${{ inputs.oracle_db_service_tag }}
+    steps:
+      - name: Echo user parameters 
+        run: |
+          echo "oracle_db_service_tag: ${{ inputs.oracle_db_service_tag }}"
+
+      - name: Install Node.js and npm
+        run: |
+          sudo apt update
+          sudo apt install nodejs
+          sudo apt install npm
+
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          path: evmos
+
+      - name: Get version for all needed repositories
+        working-directory: ./evmos
+        run: |
+          echo "ZBC_DEVELOPMENT_VERSION=$(./scripts/get_version_from_makefile.sh ZBC_DEVELOPMENT_VERSION)" >> $GITHUB_ENV
+          echo "TFHE_RS_VERSION=$(./scripts/get_version_from_makefile.sh TFHE_RS_VERSION)" >> $GITHUB_ENV
+          echo "ZBC_FHE_TOOL_VERSION=$(./scripts/get_version_from_makefile.sh ZBC_FHE_TOOL_VERSION)" >> $GITHUB_ENV
+          echo "ZBC_SOLIDITY_VERSION=$(./scripts/get_version_from_makefile.sh ZBC_SOLIDITY_VERSION)" >> $GITHUB_ENV
+          echo "ETHERMINT_VERSION=$(./scripts/get_module_version.sh go.mod zama.ai/ethermint)" >> $GITHUB_ENV
+          echo "GO_ETHEREUM_VERSION=$(./scripts/get_module_version.sh go.mod zama.ai/go-ethereum)" >> $GITHUB_ENV
+          
+      - name: Echo versions parsed from Makefile and go.mod
+        run: |
+          echo "ZBC_DEVELOPMENT_VERSION: $ZBC_DEVELOPMENT_VERSION"
+          echo "TFHE_RS_VERSION: $TFHE_RS_VERSION"
+          echo "ZBC_FHE_TOOL_VERSION: $ZBC_FHE_TOOL_VERSION"
+          echo "ZBC_SOLIDITY_VERSION: $ZBC_SOLIDITY_VERSION"
+          echo "ETHERMINT_VERSION: $ETHERMINT_VERSION"
+          echo "GO_ETHEREUM_VERSION: $GO_ETHEREUM_VERSION"
+      
+      - name: Checkout zbc-dev
+        uses: actions/checkout@v3
+        with:
+          repository: zama-ai/zbc-development
+          token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
+          path: ./evmos/work_dir/zbc-development
+          ref: ${{ env.ZBC_DEVELOPMENT_VERSION }}
+
+      - name: Checkout zbc-solidity
+        uses: actions/checkout@v3
+        with:
+          repository: zama-ai/zbc-solidity
+          token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
+          path: ./evmos/work_dir/zbc-solidity
+          ref: ${{ env.ZBC_SOLIDITY_VERSION }}
+
+      - name: Checkout tfhe-rs
+        uses: actions/checkout@v3
+        with:
+          repository: zama-ai/tfhe-rs
+          token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
+          path: ./evmos/work_dir/tfhe-rs
+          ref: ${{ env.TFHE_RS_VERSION }}
+
+      - name: Checkout zbc-fhe-tool
+        uses: actions/checkout@v3
+        with:
+          repository: zama-ai/zbc-fhe-tool
+          token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
+          path: ./evmos/work_dir/zbc-fhe-tool
+          ref: ${{ env.ZBC_FHE_TOOL_VERSION }}
+
+      - name: Checkout go-ethereum
+        uses: actions/checkout@v3
+        with:
+          repository: zama-ai/go-ethereum
+          token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
+          path: ./evmos/work_dir/go-ethereum
+          ref: ${{ env.GO_ETHEREUM_VERSION }}
+
+      - name: Checkout ethermint
+        uses: actions/checkout@v3
+        with:
+          repository: zama-ai/ethermint
+          token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
+          path: ./evmos/work_dir/ethermint
+          ref: ${{ env.ETHERMINT_VERSION }}
+
+      - name: List files in the repository
+        run: |
+          ls ${{ github.workspace }}
+
+      - name: List files in work_dir
+        working-directory: evmos/work_dir
+        run: |
+          ls 
+
+      - name: Prepare files for node setup
+        working-directory: evmos/work_dir/zbc-development
+        run: |
+          mkdir -p node/evmos
+          cp ../../private.ed25519 node/evmos 
+          cp ../../public.ed25519 node/evmos 
+          touch vm.log
+          ls node/evmos
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build local docker 
+        working-directory: ./evmos
+        run: |
+          make build-docker
+
+      - name: Check docker images
+        working-directory: ./evmos
+        run: |
+          docker images
+
+      - name: Install @openzeppelin/contracts
+        working-directory: evmos/work_dir/zbc-solidity
+        run: |
+          npm install @openzeppelin/contracts
+
+      - name: Run e2e test
+        working-directory: ./evmos
+        run: |
+          ls ${{ github.workspace }}
+          ls work_dir
+          ls work_dir/zbc-fhe-tool
+          ls work_dir/zbc-fhe-tool/.git
+          make e2e-test
diff --git a/.github/workflows/deploy_full_node_aws.yml b/.github/workflows/deploy_full_node_aws.yml
@@ -23,6 +23,11 @@ on:
         default: 'latest'
         type: string
         required: false
+      validator_ip_address:
+        description: 'The ip of the validator to be synced on (default: 13.37.31.214)'
+        default: '13.37.31.214'
+        type: string
+        required: false
       clean:
         description: 'Clean docker related components  (default: true, false only for first deployment)'
         default: true
@@ -144,7 +149,7 @@ jobs:
           AWS_S3_BUCKET: zbc-testnet
         run: |
           aws s3 ls s3://${AWS_S3_BUCKET}/
-          ./download_genesis.sh
+          ./download_genesis.sh ${{ inputs.validator_ip_address }}
 
       - name: Prepare test (called demo)
         working-directory: ./zbc-development

diff --git a/.github/workflows/deploy_validator_aws.yml b/.github/workflows/deploy_validator_aws.yml
@@ -3,21 +3,6 @@ name: Deploy validator on Aws
 on:
   workflow_dispatch:
     inputs:
-      ref_for_zbc_development:
-        description: 'Branch, tag or commit SHA1 to zbc-development'
-        required: true
-        default: "main"
-        type: string
-      ref_to_evmos:
-        description: 'Branch, tag or commit SHA1 to checkout evmos'
-        required: true
-        default: "v9.1.0-zama"
-        type: string
-      ref_to_zbc_solidity:
-        description: 'Branch, tag or commit SHA1 to checkout zbc-solidity'
-        required: true
-        default: "feature/CI-testing"
-        type: string
       evmos_node_tag:
         description: 'Evmos node image tag (default: latest)'
         default: 'latest'
@@ -33,6 +18,11 @@ on:
         default: true
         type: boolean
         required: false
+      run_test:
+        description: 'Run e2e test after deployment  (default: true)'
+        default: true
+        type: boolean
+        required: false
 
 
 jobs:
@@ -42,12 +32,12 @@ jobs:
     if: ${{ inputs.clean }}
     steps:
       - name: Stop docker compose
-        working-directory: ./zbc-development
+        working-directory: ./evmos
         run: |
-          docker compose -f docker-compose.validator.yml  down
+          docker compose -f docker-compose/docker-compose.validator.yml  down
 
       - name: Clean all docker related image, cache, container ...
-        working-directory: ./zbc-development
+        working-directory: ./evmos
         run: |
           docker system prune -af
 
@@ -62,9 +52,6 @@ jobs:
     steps:
       - name: Echo user parameters 
         run: |
-          echo "ref_for_zbc_development: ${{ inputs.ref_for_zbc_development }}"
-          echo "ref_to_evmos: ${{ inputs.ref_to_evmos }}"
-          echo "ref_to_zbc_solidity: ${{ inputs.ref_to_zbc_solidity }}"
           echo "evmos_node_tag: ${{ inputs.evmos_node_tag }}"
           echo "oracle_db_service_tag: ${{ inputs.oracle_db_service_tag }}"
           echo "clean: ${{ inputs.clean }}"
@@ -75,24 +62,25 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
         with:
-          path: evmos
-          ref: ${{ inputs.ref_to_evmos }}
-
-      - name: Checkout zbc-dev
-        uses: actions/checkout@v3
-        with:
-          repository: zama-ai/zbc-development
-          token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
-          path: zbc-development
-          ref: ${{ inputs.ref_for_zbc_development }}
+          path: evmos      
+
+
+      - name: Get version for all needed repositories
+        working-directory: ./evmos
+        run: |
+          echo "ZBC_SOLIDITY_VERSION=$(./scripts/get_version_from_makefile.sh ZBC_SOLIDITY_VERSION)" >> $GITHUB_ENV
+          
+      - name: Echo versions parsed from Makefile and go.mod
+        run: |
+          echo "ZBC_SOLIDITY_VERSION: $ZBC_SOLIDITY_VERSION"
 
       - name: Checkout zbc-solidity
         uses: actions/checkout@v3
         with:
           repository: zama-ai/zbc-solidity
           token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
-          path: zbc-solidity
-          ref: ${{ inputs.ref_to_zbc_solidity }}
+          path: ./evmos/work_dir/zbc-solidity
+          ref: ${{ env.ZBC_SOLIDITY_VERSION }}
 
       - name: List files in the repository
         run: |
@@ -106,15 +94,15 @@ jobs:
           aws-region: eu-west-3
 
       - name: Download keys from S3
-        working-directory: ./zbc-development
+        working-directory: ./evmos
         env:
           AWS_S3_BUCKET: zbc-testnet
         run: |
-          aws s3 ls s3://${AWS_S3_BUCKET}/users-fhe-keys/
-          ./prepare_volumes_validator_ci.sh
+          aws s3 ls s3://${AWS_S3_BUCKET}
+          ./scripts/prepare_volumes_validator_ci.sh
 
       - name: Prepare files for node setup
-        working-directory: ./zbc-development
+        working-directory: ./evmos
         run: |
           mkdir -p node/evmos
           cp ../evmos/private.ed25519 node/evmos 
@@ -123,7 +111,7 @@ jobs:
           ls node/evmos
 
       - name: List files in volumes
-        working-directory: ./zbc-development
+        working-directory: ./evmos
         run: |
           ls volumes
 
@@ -135,26 +123,26 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Prepare evmos node aka run setup.sh
-        working-directory: ./zbc-development
+        working-directory: ./evmos
         run: |
-          docker compose -f docker-compose.validator.yml run validator bash /config/setup.sh
+          docker compose -f docker-compose/docker-compose.validator.yml run validator bash /config/setup.sh
       
 
       - name: Check running node owner and change it 
-        working-directory: ./zbc-development
+        working-directory: ./evmos
         run: |
           ls -all .
-          sudo chown -R $USER:$USER running_node/
+          sudo chown -R $USER: running_node/
 
-      - name: Prepare test (called demo)
-        working-directory: ./zbc-development
+      - name: Prepare keys
+        working-directory: ./evmos
         run: |
-          ./prepare_demo_testnet_ci.sh
+          ./scripts/prepare_validator_ci.sh
       
       - name: Start whole testnet
-        working-directory: ./zbc-development
+        working-directory: ./evmos
         run: |
-          docker compose  -f docker-compose.validator.yml -f docker-compose.validator.override.yml  up --detach
+          docker compose  -f docker-compose/docker-compose.validator.yml -f docker-compose/docker-compose.validator.override.yml  up --detach
 
       - name: Check running docker
         run: |
@@ -165,19 +153,30 @@ jobs:
           sleep 5s
           
       - name: List files in zbc-solidity
-        working-directory: ./zbc-solidity
+        working-directory: evmos/work_dir/zbc-solidity
         run: |
           ls
 
       - name: Upload genesis and node_id to S3 bucket to allow other nodes to sync
-        working-directory: ./zbc-development
+        working-directory: ./evmos
         env:
           AWS_S3_BUCKET: zbc-testnet
         run: |
           aws s3 ls s3://${AWS_S3_BUCKET}/
-          ./upload_genesis.sh
+          ./scripts/upload_genesis.sh
+
+      - name: Prepare/download fhe keys
+        working-directory: evmos/work_dir/zbc-solidity
+        run: |
+          ./ci/scripts/prepare_fhe_keys_ci.sh
+
+      - name: Install @openzeppelin/contracts
+        working-directory: evmos/work_dir/zbc-solidity
+        run: |
+          npm install @openzeppelin/contracts
 
       - name: Run e2e test from zbc-solidty
-        working-directory: ./zbc-solidity
+        if: ${{ inputs.run_test }}
+        working-directory: evmos/work_dir/zbc-solidity
         run: |
-          ./run_ci_test.sh mykey1
+          ./ci/scripts/run_ERC20_ci_test.sh mykey1 ../../