diff --git a/.github/workflows/mobilecoin-dev-cd.yaml b/.github/workflows/mobilecoin-dev-cd.yaml index 69ad059da5..2f13348983 100644 --- a/.github/workflows/mobilecoin-dev-cd.yaml +++ b/.github/workflows/mobilecoin-dev-cd.yaml @@ -73,7 +73,7 @@ jobs: ######################################### # Build binaries -######################################## +######################################### build-rust-hardware-projects: needs: - generate-metadata @@ -191,7 +191,7 @@ jobs: - name: Get enclave MRSIGNER/MRENCLAVE values id: enclave - uses: mobilecoinofficial/gh-actions/enclave-measurements@ad6f1f0f1594c7afd159ab4c92b2f5aa7a738a08 + uses: mobilecoinofficial/gh-actions/enclave-measurements@v0 with: enclave_so_path: ${{ env.RUST_ARTIFACTS_PATH }}/${{ matrix.enclave }} @@ -221,16 +221,17 @@ jobs: if: steps.go_cache.outputs.cache-hit != 'true' shell: bash run: | - mkdir -p ${GO_ARTIFACTS_PATH} + mkdir -p "${GO_ARTIFACTS_PATH}" pushd go-grpc-gateway ./install_tools.sh ./build.sh popd - cp go-grpc-gateway/go-grpc-gateway ${GO_ARTIFACTS_PATH} + cp go-grpc-gateway/go-grpc-gateway "${GO_ARTIFACTS_PATH}" - name: check artifacts + shell: bash run: | - ls -alR ${GO_ARTIFACTS_PATH} + ls -alR "${GO_ARTIFACTS_PATH}" - name: Upload Artifacts uses: actions/upload-artifact@v4 @@ -327,11 +328,8 @@ jobs: matrix: chart: - consensus-node - - consensus-node-config - fog-ingest - fog-test-client - - mc-core-common-config - - mc-core-dev-env-setup - mobilecoind - watcher - fog-report diff --git a/.github/workflows/mobilecoin-dispatch-dev-mainnet-fog.yaml b/.github/workflows/mobilecoin-dispatch-dev-mainnet-fog.yaml index 49310e3bf5..806183373b 100644 --- a/.github/workflows/mobilecoin-dispatch-dev-mainnet-fog.yaml +++ b/.github/workflows/mobilecoin-dispatch-dev-mainnet-fog.yaml @@ -122,6 +122,7 @@ jobs: configMap: enabled: true database: recovery + EOF - name: Generate fog-ingest-config values run: | @@ -138,6 +139,7 @@ jobs: enabled: true password: '${{ secrets.DEV_POSTGRESQL_FOG_RECOVERY_PASSWORD }}' database: recovery + EOF - name: Deploy core-common-config setup uses: mobilecoinofficial/gha-k8s-toolbox@v1 diff --git a/.github/workflows/mobilecoin-dispatch-dev-testnet-fog.yaml b/.github/workflows/mobilecoin-dispatch-dev-testnet-fog.yaml index 878b41a752..e45c43a7f0 100644 --- a/.github/workflows/mobilecoin-dispatch-dev-testnet-fog.yaml +++ b/.github/workflows/mobilecoin-dispatch-dev-testnet-fog.yaml @@ -122,6 +122,7 @@ jobs: configMap: enabled: true database: recovery + EOF - name: Generate fog-ingest-config values run: | @@ -138,6 +139,7 @@ jobs: enabled: true password: '${{ secrets.DEV_POSTGRESQL_FOG_RECOVERY_PASSWORD }}' database: recovery + EOF - name: Deploy core-common-config setup uses: mobilecoinofficial/gha-k8s-toolbox@v1 diff --git a/.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml b/.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml index a33462a2e5..f7cb83226d 100644 --- a/.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml +++ b/.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml @@ -63,16 +63,16 @@ jobs: for i in 1 2 3 do aws s3 cp --only-show-errors --recursive --acl public-read \ - "s3://${BUCKET}/prebuilt/${VERSION}/chain/node${i}" \ - "s3://${BUCKET}/node${i}.${NAMESPACE}.development.mobilecoin.com" & + "s3://${BUCKET}/prebuilt/${VERSION}/chain/node${i}" \ + "s3://${BUCKET}/node${i}.${NAMESPACE}.development.mobilecoin.com" & # capture pids - pids[${i}]=$! + pids[i]=$! done # wait for all pids to finish - for pid in ${pids[*]} + for pid in "${pids[@]}" do - wait ${pid} + wait "${pid}" done setup-environment: @@ -81,9 +81,6 @@ jobs: - reset with: namespace: ${{ inputs.namespace }} - block_version: ${{ inputs.block_version }} - chart_repo: ${{ inputs.chart_repo }} - version: ${{ inputs.version }} secrets: inherit # We now have a db with setup-environment diff --git a/.github/workflows/mobilecoin-workflow-dev-deploy.yaml b/.github/workflows/mobilecoin-workflow-dev-deploy.yaml index 33f59ea484..594a84011b 100644 --- a/.github/workflows/mobilecoin-workflow-dev-deploy.yaml +++ b/.github/workflows/mobilecoin-workflow-dev-deploy.yaml @@ -34,7 +34,7 @@ on: type: string required: true minimum_block: - description: "The minimum block height before the enviroment is ready" + description: "The minimum block height before the environment is ready" type: string required: false default: "500" @@ -57,10 +57,7 @@ jobs: setup-environment: uses: ./.github/workflows/mobilecoin-workflow-dev-setup-environment.yaml with: - block_version: ${{ inputs.block_version }} - chart_repo: ${{ inputs.chart_repo }} namespace: ${{ inputs.namespace }} - version: ${{ inputs.version }} secrets: inherit consensus-deploy: @@ -69,24 +66,36 @@ jobs: runs-on: mcf-dev-small-x64 strategy: matrix: - release_name: - - consensus-node-1 - - consensus-node-2 - - consensus-node-3 + release: + - name: consensus-node-1 + instance: 1 + - name: consensus-node-2 + instance: 2 + - name: consensus-node-3 + instance: 3 steps: - # use values file because intel.com/sgx is hard to escape on the --set option. - name: Generate consensus-node values file run: | mkdir -p "${VALUES_BASE_PATH}" cat < "${VALUES_BASE_PATH}/consensus-node-values.yaml" - image: - org: ${{ inputs.docker_image_org }} - global: - certManagerClusterIssuer: google-public-ca + + mobilecoin: + network: ${{ inputs.namespace }} + partner: dev + node: + config: + clientHostname: node${{ matrix.release.instance }}.${{ inputs.namespace }}.development.mobilecoin.com + peerHostname: peer${{ matrix.release.instance }}.${{ inputs.namespace }}.development.mobilecoin.com + blockVersion: ${{ inputs.block_version }} + persistence: enabled: false + ingress: + common: + tls: + clusterIssuer: google-public-ca clientAttest: rateLimits: enabled: false @@ -101,7 +110,7 @@ jobs: chart_version: ${{ inputs.version }} chart_values: ${{ env.VALUES_BASE_PATH }}/consensus-node-values.yaml chart_wait_timeout: 10m - release_name: ${{ matrix.release_name }} + release_name: ${{ matrix.release.name }} namespace: ${{ inputs.namespace }} rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }} rancher_url: ${{ secrets.DEV_RANCHER_URL }} @@ -174,6 +183,7 @@ jobs: externalSecrets: signingCert: name: fog-report-signing-cert-a + EOF - name: Deploy fog-report-a uses: mobilecoinofficial/gha-k8s-toolbox@v1 @@ -219,6 +229,7 @@ jobs: externalSecrets: signingCert: name: fog-report-signing-cert-b + EOF - name: Deploy fog-report-b uses: mobilecoinofficial/gha-k8s-toolbox@v1 @@ -264,6 +275,7 @@ jobs: enabled: false tls: clusterIssuer: google-public-ca + EOF - name: Deploy fog-view uses: mobilecoinofficial/gha-k8s-toolbox@v1 @@ -310,6 +322,7 @@ jobs: enabled: false tls: clusterIssuer: google-public-ca + EOF - name: Deploy fog-ledger @@ -342,6 +355,7 @@ jobs: mobilecoin: network: ${{ inputs.namespace }} partner: dev + EOF - name: Deploy fog-ingest uses: mobilecoinofficial/gha-k8s-toolbox@v1.0.13 diff --git a/.github/workflows/mobilecoin-workflow-dev-setup-environment.yaml b/.github/workflows/mobilecoin-workflow-dev-setup-environment.yaml index 297728611c..fda53cf162 100644 --- a/.github/workflows/mobilecoin-workflow-dev-setup-environment.yaml +++ b/.github/workflows/mobilecoin-workflow-dev-setup-environment.yaml @@ -7,23 +7,10 @@ name: mobilecoin-workflow-dev-setup-environment on: workflow_call: inputs: - block_version: - description: "block_version" - type: string - required: true - chart_repo: - description: "Chart Repo URL" - type: string - required: false - default: https://harbor.mobilecoin.com/chartrepo/mobilecoinfoundation-public namespace: description: "Target Namespace" type: string required: true - version: - description: "Chart Version" - type: string - required: true secrets: DEV_FOG_REPORT_B_SIGNING_CA_CERT: description: "Fog Report signing CA cert" @@ -97,27 +84,6 @@ on: DEV_RANCHER_TOKEN: description: "Rancher access token" required: true - DEV_TOKENS_CONFIG_V1_JSON: - description: "dev signed tokens config json" - required: true - DEV_TOKENS_CONFIG_V2_JSON: - description: "dev signed tokens config json" - required: true - IP_INFO_TOKEN: - description: "ipinfo.io token for authenticated access" - required: true - MAIN_TOKENS_CONFIG_V1_JSON: - description: "MainNet signed tokens config json" - required: true - MAIN_TOKENS_CONFIG_V2_JSON: - description: "MainNet signed tokens config json" - required: true - TEST_TOKENS_CONFIG_V1_JSON: - description: "TestNet signed tokens config json" - required: true - TEST_TOKENS_CONFIG_V2_JSON: - description: "TestNet signed tokens config json" - required: true env: BASE_PATH: .tmp @@ -128,6 +94,15 @@ env: PG_PATH: .tmp/pg CERTS_BASE_PATH: .tmp/certs MOBILECOIND_BASE_PATH: ./tmp/mobilecoind + N1: "node1.${{ inputs.namespace }}.development.mobilecoin.com" + N2: "node2.${{ inputs.namespace }}.development.mobilecoin.com" + N3: "node3.${{ inputs.namespace }}.development.mobilecoin.com" + P1: "peer1.${{ inputs.namespace }}.development.mobilecoin.com" + P2: "peer2.${{ inputs.namespace }}.development.mobilecoin.com" + P3: "peer3.${{ inputs.namespace }}.development.mobilecoin.com" + TX_BASE: "s3-eu-central-1.amazonaws.com" + TX_BUCKET: "mobilecoin.eu.development.chain" + TX_BUCKET_REGION: "eu-central-1" jobs: setup-environment: @@ -145,7 +120,10 @@ jobs: rancher_url: ${{ secrets.DEV_RANCHER_URL }} rancher_token: ${{ secrets.DEV_RANCHER_TOKEN }} + ### Wallet keys and seeds and report values + - name: Write seeds and fog-report values + shell: bash run: | # Create seeds dir. mkdir -p "${SEEDS_BASE_PATH}" @@ -173,7 +151,10 @@ jobs: object_name: sample-keys-seeds src: ${{ env.SEEDS_BASE_PATH }} + ### Minting and tokens config + - name: Write minting keys + shell: bash run: | # Create minting secrets dir mkdir -p "${MINTING_BASE_PATH}" @@ -205,50 +186,10 @@ jobs: object_name: consensus-minting-secrets src: ${{ env.MINTING_BASE_PATH }} - - name: Write tokens.signed.json - env: - DEV_TOKENS_CONFIG_V1_JSON: ${{ secrets.DEV_TOKENS_CONFIG_V1_JSON }} - DEV_TOKENS_CONFIG_V2_JSON: ${{ secrets.DEV_TOKENS_CONFIG_V2_JSON }} - MAIN_TOKENS_CONFIG_V1_JSON: ${{ secrets.MAIN_TOKENS_CONFIG_V1_JSON }} - MAIN_TOKENS_CONFIG_V2_JSON: ${{ secrets.MAIN_TOKENS_CONFIG_V2_JSON }} - TEST_TOKENS_CONFIG_V1_JSON: ${{ secrets.TEST_TOKENS_CONFIG_V1_JSON }} - TEST_TOKENS_CONFIG_V2_JSON: ${{ secrets.TEST_TOKENS_CONFIG_V2_JSON }} - run: | - # Create base path - mkdir -p "${BASE_PATH}" - - # Set dev/main/test tokens file based on semver tag. - .internal-ci/util/set_tokens_config_version.sh ${{ inputs.version }} > "${TOKENS_PATH}" - - - name: Generate environment values file - env: - LEDGER_AWS_ACCESS_KEY_ID: ${{ secrets.DEV_LEDGER_AWS_ACCESS_KEY_ID }} - LEDGER_AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_LEDGER_AWS_SECRET_ACCESS_KEY }} - IP_INFO_TOKEN: ${{ secrets.IP_INFO_TOKEN }} - run: | - # Create values base path - mkdir -p "${VALUES_BASE_PATH}" - - # Generate values for standard dev cluster deployment. - .internal-ci/util/generate_dev_values-v5.1.sh ${{ inputs.version }} > "${VALUES_BASE_PATH}/mc-core-dev-env-values.yaml" - - - name: Deploy environment setup - uses: mobilecoinofficial/gha-k8s-toolbox@v1 - with: - action: helm-deploy - chart_repo: ${{ inputs.chart_repo }} - chart_name: mc-core-dev-env-setup - chart_version: ${{ inputs.version }} - chart_values: ${{ env.VALUES_BASE_PATH }}/mc-core-dev-env-values.yaml - chart_set: | - --set=global.node.nodeConfig.blockVersion=${{ inputs.block_version }} - release_name: mc-core-dev-env-setup - namespace: ${{ inputs.namespace }} - rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }} - rancher_url: ${{ secrets.DEV_RANCHER_URL }} - rancher_token: ${{ secrets.DEV_RANCHER_TOKEN }} + ### Fog report signing certificates - name: Write fog-report signing certificate + shell: bash run: | # Create secrets dir mkdir -p "${CERTS_BASE_PATH}/fog-report-a" @@ -282,7 +223,10 @@ jobs: object_name: fog-report-signing-cert-b src: ${{ env.CERTS_BASE_PATH }}/fog-report-b + ### Fog PostgreSQL setup + - name: Generate postgres cm and secret + shell: bash run: | mkdir -p "${PG_PATH}/cm" mkdir -p "${PG_PATH}/cm-reader" @@ -350,23 +294,38 @@ jobs: object_name: fog-recovery-reader-0-postgresql src: ${{ env.PG_PATH }}/sec + - name: Deploy PostgreSQL instance + uses: mobilecoinofficial/gha-k8s-toolbox@v1 + with: + action: helm-deploy + chart_repo: https://charts.bitnami.com/bitnami + chart_name: postgresql + chart_version: 15.2.2 + chart_set: | + --set=global.postgresql.auth.existingSecret=fog-recovery-postgresql + --set=global.postgresql.auth.database=fog_recovery + --set=architecture=replication + chart_wait_timeout: 5m + release_name: fog-recovery-postgresql + namespace: ${{ inputs.namespace }} + rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }} + rancher_url: ${{ secrets.DEV_RANCHER_URL }} + rancher_token: ${{ secrets.DEV_RANCHER_TOKEN }} + + ### Mobilecoind setup + - name: Generate mobilecoind ConfigMap file + shell: bash run: | mkdir -p "${MOBILECOIND_BASE_PATH}" - p1="node1.${{ inputs.namespace }}.development.mobilecoin.com" - p2="node2.${{ inputs.namespace }}.development.mobilecoin.com" - p3="node3.${{ inputs.namespace }}.development.mobilecoin.com" + tx1="https://${TX_BASE}/${TX_BUCKET}/${N1}/" + tx2="https://${TX_BASE}/${TX_BUCKET}/${N2}/" + tx3="https://${TX_BASE}/${TX_BUCKET}/${N3}/" - tx_base="s3-eu-central-1.amazonaws.com" - tx_bucket="mobilecoin.eu.development.chain" - tx1="https://${tx_base}/${tx_bucket}/${p1}/" - tx2="https://${tx_base}/${tx_bucket}/${p2}/" - tx3="https://${tx_base}/${tx_bucket}/${p3}/" + q="{\"threshold\": 2, \"members\": [{\"args\":\"${N1}:443\",\"type\":\"Node\"},{\"args\":\"${N2}:443\",\"type\":\"Node\"},{\"args\":\"${N3}:443\",\"type\":\"Node\"}]}" - q="{\"threshold\": 2, \"members\": [{\"args\":\"${p1}:443\",\"type\":\"Node\"},{\"args\":\"${p2}:443\",\"type\":\"Node\"},{\"args\":\"${p3}:443\",\"type\":\"Node\"}]}" - - echo -n "mc://${p1}:443,mc://${p2}:443,mc://${p3}:443" > "${MOBILECOIND_BASE_PATH}/MC_PEER" + echo -n "mc://${N1}:443,mc://${N2}:443,mc://${N3}:443" > "${MOBILECOIND_BASE_PATH}/MC_PEER" echo -n "${tx1},${tx2},${tx3}" > "${MOBILECOIND_BASE_PATH}/MC_TX_SOURCE_URL" echo -n "${q}" > "${MOBILECOIND_BASE_PATH}/MC_QUORUM_SET" @@ -381,35 +340,249 @@ jobs: object_name: fog-mobilecoind src: ${{ env.MOBILECOIND_BASE_PATH }} + ### Consensus setup + - name: Write tokens.signed.json - env: - DEV_TOKENS_CONFIG_V1_JSON: ${{ secrets.DEV_TOKENS_CONFIG_V1_JSON }} - DEV_TOKENS_CONFIG_V2_JSON: ${{ secrets.DEV_TOKENS_CONFIG_V2_JSON }} - MAIN_TOKENS_CONFIG_V1_JSON: ${{ secrets.MAIN_TOKENS_CONFIG_V1_JSON }} - MAIN_TOKENS_CONFIG_V2_JSON: ${{ secrets.MAIN_TOKENS_CONFIG_V2_JSON }} - TEST_TOKENS_CONFIG_V1_JSON: ${{ secrets.TEST_TOKENS_CONFIG_V1_JSON }} - TEST_TOKENS_CONFIG_V2_JSON: ${{ secrets.TEST_TOKENS_CONFIG_V2_JSON }} + shell: bash run: | # Create base path mkdir -p "${BASE_PATH}" - # Set dev/main/test tokens file based on semver tag. - .internal-ci/util/set_tokens_config_version.sh ${{ inputs.version }} > "${TOKENS_PATH}" + echo '${{ vars.DEV_TOKENS_CONFIG_JSON_V2 }}' > "${TOKENS_PATH}" - - name: Deploy PostgreSQL instance + - name: Create tokens-config configMap uses: mobilecoinofficial/gha-k8s-toolbox@v1 with: - action: helm-deploy - chart_repo: https://charts.bitnami.com/bitnami - chart_name: postgresql - chart_version: 11.9.13 - chart_set: | - --set=global.postgresql.auth.existingSecret=fog-recovery-postgresql - --set=global.postgresql.auth.database=fog_recovery - --set=architecture=replication - chart_wait_timeout: 5m - release_name: fog-recovery-postgresql + action: configmap-create-from-file + namespace: ${{ inputs.namespace }} + rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }} + rancher_url: ${{ secrets.DEV_RANCHER_URL }} + rancher_token: ${{ secrets.DEV_RANCHER_TOKEN }} + object_name: tokens-config + src: ${{ env.TOKENS_PATH }} + + - name: Generate Msg Signer Keys + id: generate-msg-signer-keys + shell: bash + run: | + for i in 1 2 3 + do + # generate ed25519 key pair + key=$(.internal-ci/util/generate_ed25519_keys.sh) + key_pub=$(echo -n "${key}" | grep public | awk -F': ' '{print $2}') + key_pri=$(echo -n "${key}" | grep private | awk -F': ' '{print $2}') + + # hide private key for gha output + echo "::add-mask::${key_pri}" + + # write private key to file + mkdir -p "${VALUES_BASE_PATH}/message-signer-key-node${i}" + echo -n "${key_pri}" > "${VALUES_BASE_PATH}/message-signer-key-node${i}/MC_MSG_SIGNER_KEY" + + # write gha outputs + echo "MSG_SIGNER_KEY_PRIVATE_NODE${i}=${key_pri}" >> "${GITHUB_ENV}" + echo "MSG_SIGNER_KEY_PUBLIC_NODE${i}=${key_pub}" >> "${GITHUB_ENV}" + done + + - name: Create node1 msg-signer-key secret + uses: mobilecoinofficial/gha-k8s-toolbox@v1 + with: + action: secrets-create-from-file + namespace: ${{ inputs.namespace }} + rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }} + rancher_url: ${{ secrets.DEV_RANCHER_URL }} + rancher_token: ${{ secrets.DEV_RANCHER_TOKEN }} + object_name: consensus-node-1-msg-signer-key + src: ${{ env.VALUES_BASE_PATH }}/message-signer-key-node1 + + - name: Create node2 msg-signer-key secret + uses: mobilecoinofficial/gha-k8s-toolbox@v1 + with: + action: secrets-create-from-file + namespace: ${{ inputs.namespace }} + rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }} + rancher_url: ${{ secrets.DEV_RANCHER_URL }} + rancher_token: ${{ secrets.DEV_RANCHER_TOKEN }} + object_name: consensus-node-2-msg-signer-key + src: ${{ env.VALUES_BASE_PATH }}/message-signer-key-node2 + + - name: Create node3 msg-signer-key secret + uses: mobilecoinofficial/gha-k8s-toolbox@v1 + with: + action: secrets-create-from-file + namespace: ${{ inputs.namespace }} + rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }} + rancher_url: ${{ secrets.DEV_RANCHER_URL }} + rancher_token: ${{ secrets.DEV_RANCHER_TOKEN }} + object_name: consensus-node-3-msg-signer-key + src: ${{ env.VALUES_BASE_PATH }}/message-signer-key-node3 + + - name: Write node 1 network config + run: | + mkdir -p "${VALUES_BASE_PATH}/consensus-node-1-network-config" + cat < "${VALUES_BASE_PATH}/consensus-node-1-network-config/network.json" + { + "broadcast_peers": [ + "mcp://${P2}:443?consensus-msg-key=${MSG_SIGNER_KEY_PUBLIC_NODE2}", + "mcp://${P3}:443?consensus-msg-key=${MSG_SIGNER_KEY_PUBLIC_NODE3}" + ], + "quorum_set": { + "members": [ + { + "args": "${P2}:443", + "type": "Node" + }, + { + "args": "${P3}:443", + "type": "Node" + } + ], + "threshold": 1 + }, + "tx_source_urls": [ + "https://${TX_BASE}/${TX_BUCKET}/${N2}/", + "https://${TX_BASE}/${TX_BUCKET}/${N3}/" + ] + } + EOF + + - name: Create consensus-node-1-network-config configMap + uses: mobilecoinofficial/gha-k8s-toolbox@v1 + with: + action: configmap-create-from-file + namespace: ${{ inputs.namespace }} + rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }} + rancher_url: ${{ secrets.DEV_RANCHER_URL }} + rancher_token: ${{ secrets.DEV_RANCHER_TOKEN }} + object_name: consensus-node-1-network-config + src: ${{ env.VALUES_BASE_PATH }}/consensus-node-1-network-config/network.json + + - name: Write node 2 network config + run: | + mkdir -p "${VALUES_BASE_PATH}/consensus-node-2-network-config" + cat < "${VALUES_BASE_PATH}/consensus-node-2-network-config/network.json" + { + "broadcast_peers": [ + "mcp://${P1}:443?consensus-msg-key=${MSG_SIGNER_KEY_PUBLIC_NODE1}", + "mcp://${P3}:443?consensus-msg-key=${MSG_SIGNER_KEY_PUBLIC_NODE3}" + ], + "quorum_set": { + "members": [ + { + "args": "${P1}:443", + "type": "Node" + }, + { + "args": "${P3}:443", + "type": "Node" + } + ], + "threshold": 1 + }, + "tx_source_urls": [ + "https://${TX_BASE}/${TX_BUCKET}/${N1}/", + "https://${TX_BASE}/${TX_BUCKET}/${N3}/" + ] + } + EOF + + - name: Create consensus-node-2-network-config configMap + uses: mobilecoinofficial/gha-k8s-toolbox@v1 + with: + action: configmap-create-from-file + namespace: ${{ inputs.namespace }} + rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }} + rancher_url: ${{ secrets.DEV_RANCHER_URL }} + rancher_token: ${{ secrets.DEV_RANCHER_TOKEN }} + object_name: consensus-node-2-network-config + src: ${{ env.VALUES_BASE_PATH }}/consensus-node-2-network-config/network.json + + - name: Write node 3 network config + run: | + mkdir -p "${VALUES_BASE_PATH}/consensus-node-3-network-config" + cat < "${VALUES_BASE_PATH}/consensus-node-3-network-config/network.json" + { + "broadcast_peers": [ + "mcp://${P1}:443?consensus-msg-key=${MSG_SIGNER_KEY_PUBLIC_NODE1}", + "mcp://${P2}:443?consensus-msg-key=${MSG_SIGNER_KEY_PUBLIC_NODE2}" + ], + "quorum_set": { + "members": [ + { + "args": "${P1}:443", + "type": "Node" + }, + { + "args": "${P2}:443", + "type": "Node" + } + ], + "threshold": 1 + }, + "tx_source_urls": [ + "https://${TX_BASE}/${TX_BUCKET}/${N1}/", + "https://${TX_BASE}/${TX_BUCKET}/${N2}/" + ] + } + EOF + + - name: Create consensus-node-3-network-config configMap + uses: mobilecoinofficial/gha-k8s-toolbox@v1 + with: + action: configmap-create-from-file + namespace: ${{ inputs.namespace }} + rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }} + rancher_url: ${{ secrets.DEV_RANCHER_URL }} + rancher_token: ${{ secrets.DEV_RANCHER_TOKEN }} + object_name: consensus-node-3-network-config + src: ${{ env.VALUES_BASE_PATH }}/consensus-node-3-network-config/network.json + + - name: Generate consensus-node-1-ledger-distribution + shell: bash + run: | + for i in 1 2 3 + do + node="N${i}" + + mkdir -p "${VALUES_BASE_PATH}/consensus-node-${i}-ledger-distribution" + + echo -n '${{ secrets.DEV_LEDGER_AWS_ACCESS_KEY_ID }}' > "${VALUES_BASE_PATH}/consensus-node-${i}-ledger-distribution/AWS_ACCESS_KEY_ID" + echo -n '${{ secrets.DEV_LEDGER_AWS_SECRET_ACCESS_KEY }}' > "${VALUES_BASE_PATH}/consensus-node-${i}-ledger-distribution/AWS_SECRET_ACCESS_KEY" + echo -n "${TX_BUCKET_REGION}" > "${VALUES_BASE_PATH}/consensus-node-${i}-ledger-distribution/AWS_REGION" + echo -n "${TX_BUCKET}" > "${VALUES_BASE_PATH}/consensus-node-${i}-ledger-distribution/LEDGER_DISTRIBUTION_S3_BUCKET" + echo -n "s3://${TX_BUCKET}/${!node}?=${TX_BUCKET_REGION}" > "${VALUES_BASE_PATH}/consensus-node-${i}-ledger-distribution/MC_DEST" + echo -n "https://${TX_BASE}/${TX_BUCKET}/${!node}/" > "${VALUES_BASE_PATH}/consensus-node-${i}-ledger-distribution/MC_TX_SOURCE_URL" + done + + - name: Create consensus-node-1-ledger-distribution secret + uses: mobilecoinofficial/gha-k8s-toolbox@v1 + with: + action: secrets-create-from-file + namespace: ${{ inputs.namespace }} + rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }} + rancher_url: ${{ secrets.DEV_RANCHER_URL }} + rancher_token: ${{ secrets.DEV_RANCHER_TOKEN }} + object_name: consensus-node-1-ledger-distribution + src: ${{ env.VALUES_BASE_PATH }}/consensus-node-1-ledger-distribution + + - name: Create consensus-node-2-ledger-distribution secret + uses: mobilecoinofficial/gha-k8s-toolbox@v1 + with: + action: secrets-create-from-file + namespace: ${{ inputs.namespace }} + rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }} + rancher_url: ${{ secrets.DEV_RANCHER_URL }} + rancher_token: ${{ secrets.DEV_RANCHER_TOKEN }} + object_name: consensus-node-2-ledger-distribution + src: ${{ env.VALUES_BASE_PATH }}/consensus-node-2-ledger-distribution + + - name: Create consensus-node-3-ledger-distribution secret + uses: mobilecoinofficial/gha-k8s-toolbox@v1 + with: + action: secrets-create-from-file namespace: ${{ inputs.namespace }} rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }} rancher_url: ${{ secrets.DEV_RANCHER_URL }} rancher_token: ${{ secrets.DEV_RANCHER_TOKEN }} + object_name: consensus-node-3-ledger-distribution + src: ${{ env.VALUES_BASE_PATH }}/consensus-node-3-ledger-distribution diff --git a/.github/workflows/mobilecoin-workflow-dev-update-consensus.yaml b/.github/workflows/mobilecoin-workflow-dev-update-consensus.yaml index 754d8bc2b5..c5a32b936c 100644 --- a/.github/workflows/mobilecoin-workflow-dev-update-consensus.yaml +++ b/.github/workflows/mobilecoin-workflow-dev-update-consensus.yaml @@ -35,32 +35,67 @@ on: description: "Rancher access token" required: true +env: + VALUES_BASE_PATH: .tmp/values + jobs: setup-environment: uses: ./.github/workflows/mobilecoin-workflow-dev-setup-environment.yaml with: namespace: ${{ inputs.namespace }} - block_version: ${{ inputs.block_version }} - chart_repo: ${{ inputs.chart_repo }} - version: ${{ inputs.version }} secrets: inherit - consensus-restart: - runs-on: mcf-dev-small-x64 + consensus-deploy: needs: - setup-environment + runs-on: mcf-dev-small-x64 strategy: matrix: - object_name: - - deployment.app/consensus-node-1 - - deployment.app/consensus-node-2 - - deployment.app/consensus-node-3 + release: + - name: consensus-node-1 + instance: 1 + - name: consensus-node-2 + instance: 2 + - name: consensus-node-3 + instance: 3 steps: - - name: Restart Consensus Nodes + - name: Generate consensus-node values file + run: | + mkdir -p "${VALUES_BASE_PATH}" + cat < "${VALUES_BASE_PATH}/consensus-node-values.yaml" + + mobilecoin: + network: main + partner: mc + + node: + config: + clientHostname: node${{ matrix.release.instance }}.${{ inputs.namespace }}.development.mobilecoin.com + peerHostname: peer${{ matrix.release.instance }}.${{ inputs.namespace }}.development.mobilecoin.com + blockVersion: ${{ inputs.block_version }} + + persistence: + enabled: false + + ingress: + common: + tls: + clusterIssuer: google-public-ca + clientAttest: + rateLimits: + enabled: false + EOF + + - name: Deploy Consensus nodes uses: mobilecoinofficial/gha-k8s-toolbox@v1 with: - action: pod-restart - object_name: ${{ matrix.object_name }} + action: helm-deploy + chart_repo: ${{ inputs.chart_repo }} + chart_name: consensus-node + chart_version: ${{ inputs.version }} + chart_values: ${{ env.VALUES_BASE_PATH }}/consensus-node-values.yaml + chart_wait_timeout: 10m + release_name: ${{ matrix.release.name }} namespace: ${{ inputs.namespace }} rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }} rancher_url: ${{ secrets.DEV_RANCHER_URL }} diff --git a/.github/workflows/refresh-ledger-bootstrap.yaml b/.github/workflows/refresh-ledger-bootstrap.yaml index 4a0faef45b..a8f48d8e61 100644 --- a/.github/workflows/refresh-ledger-bootstrap.yaml +++ b/.github/workflows/refresh-ledger-bootstrap.yaml @@ -75,8 +75,8 @@ jobs: then ls -l ./ledger-data.mdb md5sum ./ledger-data.mdb - mkdir -p ${MC_LEDGER_DB} - mv ./ledger-data.mdb ${MC_LEDGER_DB}/data.mdb + mkdir -p "${MC_LEDGER_DB}" + mv ./ledger-data.mdb "${MC_LEDGER_DB}/data.mdb" fi # Download the current watcher db @@ -85,8 +85,8 @@ jobs: then ls -l ./watcher-data.mdb md5sum ./watcher-data.mdb - mkdir -p ${MC_WATCHER_DB} - mv ./watcher-data.mdb ${MC_WATCHER_DB}/data.mdb + mkdir -p "${MC_WATCHER_DB}" + mv ./watcher-data.mdb "${MC_WATCHER_DB}/data.mdb" fi - name: Run mobilecoind mobilecoind-json - wait for ledger sync diff --git a/.internal-ci/helm/consensus-node-config/.helmignore b/.internal-ci/helm/consensus-node-config/.helmignore deleted file mode 100644 index 0e8a0eb36f..0000000000 --- a/.internal-ci/helm/consensus-node-config/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/.internal-ci/helm/consensus-node-config/Chart.yaml b/.internal-ci/helm/consensus-node-config/Chart.yaml deleted file mode 100644 index d74496c03d..0000000000 --- a/.internal-ci/helm/consensus-node-config/Chart.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright (c) 2018-2022 The MobileCoin Foundation -apiVersion: v2 -name: consensus-node-config -description: Consensus node specific configuration. -type: application -version: 0.0.0 -appVersion: 1.0.0 diff --git a/.internal-ci/helm/consensus-node-config/templates/NOTES.txt b/.internal-ci/helm/consensus-node-config/templates/NOTES.txt deleted file mode 100644 index 2671db2149..0000000000 --- a/.internal-ci/helm/consensus-node-config/templates/NOTES.txt +++ /dev/null @@ -1,5 +0,0 @@ -Common consensus config - -Client hostname: {{ include "consensusNodeConfig.clientHostname" . }} -Peer hostname: {{ include "consensusNodeConfig.peerHostname" . }} -Blockchain Path: {{ include "consensusNodeConfig.ledgerDistributionAWSPath" . }} diff --git a/.internal-ci/helm/consensus-node-config/templates/_helpers.tpl b/.internal-ci/helm/consensus-node-config/templates/_helpers.tpl deleted file mode 100644 index cb15b45bf0..0000000000 --- a/.internal-ci/helm/consensus-node-config/templates/_helpers.tpl +++ /dev/null @@ -1,97 +0,0 @@ -{{/* Copyright (c) 2018-2022 The MobileCoin Foundation */}} - -{{/* -Expand the name of the consensusNodeConfig. -*/}} -{{- define "consensusNodeConfig.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "consensusNodeConfig.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- tpl .Values.fullnameOverride . | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "consensusNodeConfig.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" | trimSuffix "." }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "consensusNodeConfig.labels" -}} -helm.sh/chart: {{ include "consensusNodeConfig.chart" . }} -{{ include "consensusNodeConfig.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "consensusNodeConfig.selectorLabels" -}} -app.kubernetes.io/name: {{ include "consensusNodeConfig.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* Figure out our node id from the name or use values clientHostname/peerHostname */}} - -{{/* clientHostname */}} -{{- define "consensusNodeConfig.clientHostname" -}} -{{- tpl .Values.node.client.hostname . }} -{{- end }} - -{{/* peerHostname */}} -{{- define "consensusNodeConfig.peerHostname" -}} -{{- tpl .Values.node.peer.hostname . }} -{{- end }} - -{{/* TX_SOURCE_URL */}} -{{- define "consensusNodeConfig.txSourceUrl" -}} -{{- tpl .Values.node.txSourceUrl . }} -{{- end }} - -{{/* ledgerDistributionAWSPath */}} -{{- define "consensusNodeConfig.ledgerDistributionAWSPath" -}} -{{ printf "s3://%s/%s?region=%s" .Values.global.node.ledgerDistribution.s3Bucket (include "consensusNodeConfig.clientHostname" .) .Values.global.node.ledgerDistribution.awsRegion }} -{{- end }} - -{{/* networkJson */}} -{{- define "consensusNodeConfig.networkJson" -}} -{{- $peers := .Values.global.node.networkConfig.peers }} -{{- $localPeer := (include "consensusNodeConfig.peerHostname" .) }} -{{- $threshold := .Values.global.node.networkConfig.threshold }} -{{- $broadcastPeers := list }} -{{- $txSourceUrls := list }} -{{- $members := list }} -{{- range $key, $value := $peers }} -{{- $peer := tpl $value.peer.hostname $ }} -{{- $archive := tpl $value.ledgerArchiveLocation $ }} -{{- if not (eq $peer $localPeer) }} -{{- $broadcastPeers = append $broadcastPeers (printf "mcp://%s:%s/?consensus-msg-key=%s" $peer $value.peer.port $value.signerPublicKey) }} -{{- $txSourceUrls = append $txSourceUrls $archive }} -{{- $members = append $members (dict "type" "Node" "args" (printf "%s:%s" $peer $value.peer.port)) }} -{{- end }} -{{- end }} -{{- $quorumSet := dict "threshold" (atoi $threshold) "members" $members }} -{{- $networkJson := dict "broadcast_peers" $broadcastPeers "tx_source_urls" $txSourceUrls "quorum_set" $quorumSet }} -{{- toPrettyJson $networkJson }} -{{- end }} diff --git a/.internal-ci/helm/consensus-node-config/templates/node-config-configmap.yaml b/.internal-ci/helm/consensus-node-config/templates/node-config-configmap.yaml deleted file mode 100644 index 23e904d5f5..0000000000 --- a/.internal-ci/helm/consensus-node-config/templates/node-config-configmap.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2018-2022 The MobileCoin Foundation -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "consensusNodeConfig.fullname" . }}-consensus-node - labels: - {{- include "consensusNodeConfig.labels" . | nindent 4 }} -data: - CLIENT_HOSTNAME: {{ include "consensusNodeConfig.clientHostname" . | squote }} - PEER_HOSTNAME: {{ include "consensusNodeConfig.peerHostname" . | squote }} - MC_TX_SOURCE_URL: {{ include "consensusNodeConfig.txSourceUrl" . | squote }} - MC_BLOCK_VERSION: {{ .Values.global.node.nodeConfig.blockVersion | squote }} - MC_CLIENT_RESPONDER_ID: "{{ include "consensusNodeConfig.clientHostname" . }}:443" - MC_PEER_RESPONDER_ID: "{{ include "consensusNodeConfig.peerHostname" . }}:443" diff --git a/.internal-ci/helm/consensus-node-config/templates/node-ingress-blocklist-configmap.yaml b/.internal-ci/helm/consensus-node-config/templates/node-ingress-blocklist-configmap.yaml deleted file mode 100644 index cf6a613f00..0000000000 --- a/.internal-ci/helm/consensus-node-config/templates/node-ingress-blocklist-configmap.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) 2018-2022 The MobileCoin Foundation -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "consensusNodeConfig.fullname" . }}-ingress-blocklist - labels: - {{- include "consensusNodeConfig.labels" . | nindent 4 }} -data: - BLOCKLIST_ENABLED: {{ .Values.global.blocklist.enabled | squote }} - BLOCKLIST_PATTERN: {{ .Values.global.blocklist.pattern | squote }} diff --git a/.internal-ci/helm/consensus-node-config/templates/node-ledger-distribution-secret.yaml b/.internal-ci/helm/consensus-node-config/templates/node-ledger-distribution-secret.yaml deleted file mode 100644 index 996119ee17..0000000000 --- a/.internal-ci/helm/consensus-node-config/templates/node-ledger-distribution-secret.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright (c) 2018-2022 The MobileCoin Foundation -apiVersion: v1 -kind: Secret -metadata: - name: {{ include "consensusNodeConfig.fullname" . }}-ledger-distribution - labels: - {{- include "consensusNodeConfig.labels" . | nindent 4 }} -type: Opaque -stringData: - {{- with .Values.global.node.ledgerDistribution }} - AWS_ACCESS_KEY_ID: {{ .awsAccessKeyId | quote }} - AWS_SECRET_ACCESS_KEY: {{ .awsSecretAccessKey | quote }} - AWS_REGION: {{ .awsRegion | quote }} - LEDGER_DISTRIBUTION_S3_BUCKET: {{ .s3Bucket | quote }} - MC_DEST: {{ tpl .awsPath $ | quote }} - {{- end }} diff --git a/.internal-ci/helm/consensus-node-config/templates/node-msg-signer-key-secret.yaml b/.internal-ci/helm/consensus-node-config/templates/node-msg-signer-key-secret.yaml deleted file mode 100644 index c3a85d0d63..0000000000 --- a/.internal-ci/helm/consensus-node-config/templates/node-msg-signer-key-secret.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) 2018-2022 The MobileCoin Foundation -apiVersion: v1 -kind: Secret -type: Opaque -metadata: - name: {{ include "consensusNodeConfig.fullname" . }}-msg-signer-key - labels: - {{- include "consensusNodeConfig.labels" . | nindent 4 }} -stringData: - MC_MSG_SIGNER_KEY: {{ .Values.node.msgSignerKey.privateKey | quote }} diff --git a/.internal-ci/helm/consensus-node-config/templates/node-network-config-configmap.yaml b/.internal-ci/helm/consensus-node-config/templates/node-network-config-configmap.yaml deleted file mode 100644 index ffbe1a2247..0000000000 --- a/.internal-ci/helm/consensus-node-config/templates/node-network-config-configmap.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) 2018-2022 The MobileCoin Foundation -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "consensusNodeConfig.fullname" . }}-network-config - labels: - {{- include "consensusNodeConfig.labels" . | nindent 4 }} -data: - network.json: |- - {{- include "consensusNodeConfig.networkJson" . | nindent 4 }} diff --git a/.internal-ci/helm/consensus-node-config/templates/node-tokens-config-configmap.yaml b/.internal-ci/helm/consensus-node-config/templates/node-tokens-config-configmap.yaml deleted file mode 100644 index 5c4ffe74f0..0000000000 --- a/.internal-ci/helm/consensus-node-config/templates/node-tokens-config-configmap.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) 2018-2022 The MobileCoin Foundation -{{- if .Values.global.node.tokensConfig }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "consensusNodeConfig.fullname" . }}-tokens-config - labels: - {{- include "consensusNodeConfig.labels" . | nindent 4 }} -data: - tokens.signed.json: | - {{- .Values.global.node.tokensConfig.tokensSignedJson | nindent 4 }} -{{- end }} diff --git a/.internal-ci/helm/consensus-node-config/values.yaml b/.internal-ci/helm/consensus-node-config/values.yaml deleted file mode 100644 index 8d0e94eed4..0000000000 --- a/.internal-ci/helm/consensus-node-config/values.yaml +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright (c) 2018-2022 The MobileCoin Foundation -### Set fullnameOverride to match the consensus-node release name. -# example consensus-node-1 -fullnameOverride: '' - -### This node's specific values -node: - ### This node's client fully qualified domain name. - client: - hostname: '' - ### This node's peer fully qualified domain name. - # maps to - peer: - hostname: '' - - ### S3 bucket for thin node's ledger - txSourceUrl: '' - - ### This nodes's Message Signing Key - msgSignerKey: - privateKey: '' - -### Values that will be shared by all nodes, but generate a unique k8s objects per node. -global: - node: - ### Node configuration settings - nodeConfig: - ### Block version - set to upgrade block version. - blockVersion: 4 - - ### Ledger distribution settings. The default for awsPath is auto-generated based on - # this values, but can be overridden. - # https://github.com/mobilecoinfoundation/mobilecoin/tree/main/ledger/distribution - ledgerDistribution: - awsAccessKeyId: '' - awsSecretAccessKey: '' - awsRegion: '' - s3Bucket: '' - awsPath: '{{ include "consensusNodeConfig.ledgerDistributionAWSPath" . }}' - startFrom: last - - ### Network config list of settings to generate the network.toml file. - # You can list all the peers here. When we generate the node's config we - # will skip the the local node. - # https://github.com/mobilecoinfoundation/mobilecoin/tree/main/consensus/service - networkConfig: - threshold: '' - peers: {} - ### Needs to be a map so we can override a specific entry. - # The keys names don't really matter. - # 1: - # peer: - # hostname: peer1.test.example.com - # port: '443' - # signerPublicKey: public-key - # ledgerArchiveLocation: https://s3-location-hostname/ledger/peer1.test.example.com - - # json formatted tokens configuration file. see consensus/service/config/src/tokens.rs - # Add signed tokens.json with --set-file=global.node.tokensConfig.tokensSignedJson=tokens.signed.json - # tokensConfig: - # tokensSignedJson: |- - - ### Enable haproxy IP blocklist for ingress - # pattern is the object in the configmap shared between infra-haproxy-blocklist and haproxy kubernetes-ingress - blocklist: - enabled: "false" - pattern: patterns/blocked-countries diff --git a/.internal-ci/helm/consensus-node/Chart.yaml b/.internal-ci/helm/consensus-node/Chart.yaml index 8b959dfaa2..5c2d6729a5 100644 --- a/.internal-ci/helm/consensus-node/Chart.yaml +++ b/.internal-ci/helm/consensus-node/Chart.yaml @@ -1,18 +1,7 @@ -# Copyright (c) 2018-2022 The MobileCoin Foundation +# Copyright (c) 2018-2024 The MobileCoin Foundation apiVersion: v2 name: consensus-node description: MobileCoin consensus node type: application version: 0.0.0 appVersion: 0.0.0 -dependencies: -- name: consensus-node-config - repository: file://../consensus-node-config - version: 0.0.0 - condition: consensusNodeConfig.enabled - alias: consensusNodeConfig -- name: mc-core-common-config - repository: file://../mc-core-common-config - version: 0.0.0 - condition: mcCoreCommonConfig.enabled - alias: mcCoreCommonConfig diff --git a/.internal-ci/helm/consensus-node/README.md b/.internal-ci/helm/consensus-node/README.md index f237d4d171..bc29ba41c4 100644 --- a/.internal-ci/helm/consensus-node/README.md +++ b/.internal-ci/helm/consensus-node/README.md @@ -1,26 +1,115 @@ # consensus-node Helm Chart -Deploy a single node of the consensus service +## Launch a node. -```sh -helm upgrade node1 ./ -i -n \ - --set image.tag=prod-1.0.1-pre2 +The example node will be `consensus-node-1` + +### Required Secrets + +__Ledger Distribution__ + +Configuration for AWS S3 bucket used to store the ledger. + +The name should be prefixed with the name of the helm release (`consensus-node-1`). + +```yaml +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: consensus-node-1-ledger-distribution +stringData: + AWS_ACCESS_KEY_ID: + AWS_SECRET_ACCESS_KEY: + AWS_REGION: eu-west-2 + # bucket name + LEDGER_DISTRIBUTION_S3_BUCKET: ledger.mainnet.mobilecoinww.com + # s3 url to bucket + MC_DEST: s3://ledger.mainnet.mobilecoinww.com/node1.prod.mobilecoinww.com?region=eu-west-2 + # HTTP path to s3 bucket - may be behind a CDN + MC_TX_SOURCE_URL: https://ledger.mobilecoinww.com/node1.prod.mobilecoinww.com/ ``` -Note: generated PersistentVolumeClaims will stick around if the Helm Chart is removed or the pods are deleted and allowed to regenerate. -## Setup +__Message Signer Private Key__ -Configure a `values.yaml` file or pre-populate your namespace with the following ConfigMaps and Secrets. +The private key for peer communication. -- `mobilecoin-network` +The name should be prefixed with the name of the helm release (`consensus-node-1`). - Mobilecoin network value for monitoring: mainnet, testnet, alpha... +```yaml +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: consensus-node-1-msg-signer-key +stringData: + MC_MSG_SIGNER_KEY: +``` + +### Required ConfigMaps + +__Tokens Config__ + +The tokens config is shared by all consensus nodes on the network. It only needs to be installed once and should be named `tokens-config` + +```yaml +apiVersion: v1 + kind: ConfigMap +metadata: + name: tokens-config +data: + tokens.signed.json: | + { + + } +``` +__Network Config__ - ```yaml - apiVersion: v1 - kind: ConfigMap - metadata: - name: mobilecoin-network - data: - network: testnet - ``` \ No newline at end of file +Network Config is unique for each node and contains the list of all _other_ peers and their message public keys. + +The name should be prefixed with the name of the helm release (`consensus-node-1`). + +```yaml +apiVersion: v1 + kind: ConfigMap +metadata: + name: consensus-node-1-network-config +data: + network.json: |- + { + "broadcast_peers": [ + ], + "quorum_set": { + "members": [ + { + "args": "", + "type": "Node" + }, + ], + "threshold": 7 + }, + "tx_source_urls": [ + ] + } +``` + +### Helm chart + +Set up values yaml: +```yaml +mobilecoin: + network: main + partner: mc + +node: + config: + clientHostname: node1.prod.mobilecoinww.com + peerHostname: peer1.prod.mobilecoinww.com +``` + +Launch Helm chart + +```sh +helm upgrade consensus-node-1 mcf-public/consensus-node -i -n --version \ + -f values.yaml +``` diff --git a/.internal-ci/helm/consensus-node/templates/NOTES.txt b/.internal-ci/helm/consensus-node/templates/NOTES.txt deleted file mode 100644 index b63e476da3..0000000000 --- a/.internal-ci/helm/consensus-node/templates/NOTES.txt +++ /dev/null @@ -1,17 +0,0 @@ - -:::: :::: :::::::: ::::::::: ::::::::::: ::: :::::::::: -+:+:+: :+:+:+ :+: :+: :+: :+: :+: :+: :+: -+:+ +:+:+ +:+ +:+ +:+ +:+ +:+ +:+ +:+ +:+ -+#+ +:+ +#+ +#+ +:+ +#++:++#+ +#+ +#+ +#++:++# -+#+ +#+ +#+ +#+ +#+ +#+ +#+ +#+ +#+ -#+# #+# #+# #+# #+# #+# #+# #+# #+# -### ### ######## ######### ########### ########## ########## - :::::::: :::::::: ::::::::::: :::: ::: -:+: :+: :+: :+: :+: :+:+: :+: -+:+ +:+ +:+ +:+ :+:+:+ +:+ -+#+ +#+ +:+ +#+ +#+ +:+ +#+ -+#+ +#+ +#+ +#+ +#+ +#+#+# -#+# #+# #+# #+# #+# #+# #+#+# - ######## ######## ########### ### #### - -MobileCoin Node deployment completed successfully. \ No newline at end of file diff --git a/.internal-ci/helm/consensus-node/templates/_helpers.tpl b/.internal-ci/helm/consensus-node/templates/_helpers.tpl index 039724dbf5..99d1e721ac 100644 --- a/.internal-ci/helm/consensus-node/templates/_helpers.tpl +++ b/.internal-ci/helm/consensus-node/templates/_helpers.tpl @@ -1,4 +1,4 @@ -{{/* Copyright (c) 2018-2022 The MobileCoin Foundation */}} +{{/* Copyright (c) 2018-2024 The MobileCoin Foundation */}} {{/* Expand the name of the consensusNode. */}} {{- define "consensusNode.name" -}} @@ -44,100 +44,6 @@ app.kubernetes.io/name: {{ include "consensusNode.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} -{{/* Define Secret and ConfigMap object names */}} -{{- define "consensusNode.nodeConfig.configMap.name" -}} -{{ include "consensusNode.fullname" . }}-consensus-node -{{- end }} - -{{- define "consensusNode.ledgerDistribution.secret.name" -}} -{{ include "consensusNode.fullname" . }}-ledger-distribution -{{- end }} - -{{- define "consensusNode.msgSignerKey.secret.name" -}} -{{ include "consensusNode.fullname" . }}-msg-signer-key -{{- end }} - -{{- define "consensusNode.networkConfig.configMap.name" -}} -{{ include "consensusNode.fullname" . }}-network-config -{{- end }} - -{{- define "consensusNode.tokensConfig.configMap.name" -}} -{{ include "consensusNode.fullname" . }}-tokens-config -{{- end }} - -{{- define "consensusNode.ingressBlocklist.configMap.name" -}} -{{ include "consensusNode.fullname" . }}-ingress-blocklist -{{- end }} - -{{/* -peer and client hostnames - we need this for ingress. -lookup name from configmap if we have created the objects in consensus-node-config separately. -*/}} -{{- define "consensusNode.peerHostname" -}} - {{- if eq .Values.consensusNodeConfig.enabled false }} - {{- (lookup "v1" "ConfigMap" .Release.Namespace (include "consensusNode.nodeConfig.configMap.name" .)).data.PEER_HOSTNAME | default "" }} - {{- else }} - {{- tpl .Values.consensusNodeConfig.node.peer.hostname . }} - {{- end }} -{{- end }} - -{{- define "consensusNode.clientHostname" -}} - {{- if eq .Values.consensusNodeConfig.enabled false }} - {{- (lookup "v1" "ConfigMap" .Release.Namespace (include "consensusNode.nodeConfig.configMap.name" .)).data.CLIENT_HOSTNAME | default "" }} - {{- else }} - {{- tpl .Values.consensusNodeConfig.node.client.hostname . }} - {{- end }} -{{- end }} - -{{- define "consensusNode.blockVersion" -}} - {{- if eq .Values.consensusNodeConfig.enabled false }} - {{- (lookup "v1" "ConfigMap" .Release.Namespace (include "consensusNode.nodeConfig.configMap.name" .)).data.BLOCK_VERSION | default "false" }} - {{- else }} - {{- tpl .Values.global.node.nodeConfig.blockVersion . }} - {{- end }} -{{- end }} - -{{- define "consensusNode.txSourceUrl" -}} - {{- if eq .Values.consensusNodeConfig.enabled false }} - {{- (lookup "v1" "ConfigMap" .Release.Namespace (include "consensusNode.nodeConfig.configMap.name" .)).data.TX_SOURCE_URL | default "false" }} - {{- else }} - {{- tpl .Values.global.node.nodeConfig.txSourceUrl . }} - {{- end }} -{{- end }} - -{{/* Mobilecoin Network monitoring labels */}} -{{- define "consensusNode.mobileCoinNetwork.network" -}} - {{- if eq .Values.mcCoreCommonConfig.enabled false }} - {{- (lookup "v1" "ConfigMap" .Release.Namespace "mobilecoin-network").data.network | default "" }} - {{- else }} - {{- tpl .Values.mcCoreCommonConfig.mobileCoinNetwork.network . }} - {{- end }} -{{- end }} - -{{- define "consensusNode.mobileCoinNetwork.partner" -}} - {{- if eq .Values.mcCoreCommonConfig.enabled false }} - {{- (lookup "v1" "ConfigMap" .Release.Namespace "mobilecoin-network").data.partner | default "" }} - {{- else }} - {{- tpl .Values.mcCoreCommonConfig.mobileCoinNetwork.partner . }} - {{- end }} -{{- end }} - -{{- define "consensusNode.blocklist.enabled" -}} - {{- if eq .Values.consensusNodeConfig.enabled false }} - {{- (lookup "v1" "ConfigMap" .Release.Namespace (include "consensusNode.ingressBlocklist.configMap.name" .)).data.BLOCKLIST_ENABLED | default "false" }} - {{- else }} - {{- tpl .Values.global.blocklist.enabled . }} - {{- end }} -{{- end }} - -{{- define "consensusNode.blocklist.pattern" -}} - {{- if eq .Values.consensusNodeConfig.enabled false }} - {{- (lookup "v1" "ConfigMap" .Release.Namespace (include "consensusNode.ingressBlocklist.configMap.name" .)).data.BLOCKLIST_PATTERN | default "" }} - {{- else }} - {{- tpl .Values.global.blocklist.pattern . }} - {{- end }} -{{- end }} - {{/* Find the instance number of the consensus deploy (1, 2, 3...) */}} {{- define "consensusNode.instanceNumber" -}} {{- if (regexMatch ".*-[0-9]+$" (include "consensusNode.fullname" .)) }} diff --git a/.internal-ci/helm/consensus-node/templates/client-grpc-attest-ingress.yaml b/.internal-ci/helm/consensus-node/templates/client-grpc-attest-ingress.yaml index 28063d3928..e6610e55f9 100644 --- a/.internal-ci/helm/consensus-node/templates/client-grpc-attest-ingress.yaml +++ b/.internal-ci/helm/consensus-node/templates/client-grpc-attest-ingress.yaml @@ -1,34 +1,38 @@ -# Copyright (c) 2018-2022 The MobileCoin Foundation +# Copyright (c) 2018-2024 The MobileCoin Foundation # split out attest endpoint for client port so we can set rate limiting. +{{- with .Values.node.ingress }} +{{- if .enabled }} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: {{ include "consensusNode.fullname" . }}-client-grpc-attest + name: {{ include "consensusNode.fullname" $ }}-client-grpc-attest annotations: - {{- if .Values.node.ingress.clientAttest.rateLimits.enabled }} - {{- toYaml .Values.node.ingress.clientAttest.rateLimits.annotations | nindent 4 }} - haproxy.org/rate-limit-period: {{ include "consensusNode.rateLimitPeriod" . | quote }} + {{- if .clientAttest.rateLimits.enabled }} + {{- toYaml .clientAttest.rateLimits.annotations | nindent 4 }} + haproxy.org/rate-limit-period: {{ include "consensusNode.rateLimitPeriod" $ | quote }} {{- end }} - {{- if eq (include "consensusNode.blocklist.enabled" .) "true" }} - haproxy.org/blacklist: {{ include "consensusNode.blocklist.pattern" . }} + {{- if .common.blocklist.enabled }} + haproxy.org/blacklist: {{ .common.blocklist.pattern | quote }} {{- end }} - {{- toYaml .Values.node.ingress.common.annotations | nindent 4 }} - {{- toYaml .Values.node.ingress.grpc.annotations | nindent 4 }} + {{- toYaml .common.annotations | nindent 4 }} + {{- toYaml .grpc.annotations | nindent 4 }} labels: - {{- include "consensusNode.labels" . | nindent 4 }} + {{- include "consensusNode.labels" $ | nindent 4 }} spec: tls: - hosts: - - {{ include "consensusNode.clientHostname" . }} - secretName: {{ include "consensusNode.fullname" . }}-ingress-tls + - {{ $.Values.node.config.clientHostname }} + secretName: {{ include "consensusNode.fullname" $ }}-ingress-tls rules: - - host: {{ include "consensusNode.clientHostname" . }} + - host: {{ $.Values.node.config.clientHostname }} http: paths: - path: /attest.AttestedApi pathType: Prefix backend: service: - name: {{ include "consensusNode.fullname" . }} + name: {{ include "consensusNode.fullname" $ }} port: name: client-grpc +{{- end }} +{{- end }} diff --git a/.internal-ci/helm/consensus-node/templates/client-grpc-ingress.yaml b/.internal-ci/helm/consensus-node/templates/client-grpc-ingress.yaml index d34632cf1e..9e2d32902a 100644 --- a/.internal-ci/helm/consensus-node/templates/client-grpc-ingress.yaml +++ b/.internal-ci/helm/consensus-node/templates/client-grpc-ingress.yaml @@ -1,43 +1,47 @@ -# Copyright (c) 2018-2022 The MobileCoin Foundation +# Copyright (c) 2018-2024 The MobileCoin Foundation +{{- with .Values.node.ingress }} +{{- if .enabled }} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: {{ include "consensusNode.fullname" . }}-client-grpc + name: {{ include "consensusNode.fullname" $ }}-client-grpc annotations: - {{- if eq (include "consensusNode.blocklist.enabled" .) "true" }} - haproxy.org/blacklist: {{ include "consensusNode.blocklist.pattern" . }} + {{- if .common.blocklist.enabled }} + haproxy.org/blacklist: {{ .common.blocklist.pattern | quote }} {{- end }} - {{- toYaml .Values.node.ingress.common.annotations | nindent 4 }} - {{- toYaml .Values.node.ingress.grpc.annotations | nindent 4 }} + {{- toYaml .common.annotations | nindent 4 }} + {{- toYaml .grpc.annotations | nindent 4 }} labels: - {{- include "consensusNode.labels" . | nindent 4 }} + {{- include "consensusNode.labels" $ | nindent 4 }} spec: tls: - hosts: - - {{ include "consensusNode.clientHostname" . }} - secretName: {{ include "consensusNode.fullname" . }}-ingress-tls + - {{ $.Values.node.config.clientHostname }} + secretName: {{ include "consensusNode.fullname" $ }}-ingress-tls rules: - - host: {{ include "consensusNode.clientHostname" . }} + - host: {{ $.Values.node.config.clientHostname }} http: paths: - path: /build_info.BuildInfoApi pathType: Prefix backend: service: - name: {{ include "consensusNode.fullname" . }} + name: {{ include "consensusNode.fullname" $ }} port: name: client-grpc - path: /consensus_common.BlockchainAPI pathType: Prefix backend: service: - name: {{ include "consensusNode.fullname" . }} + name: {{ include "consensusNode.fullname" $ }} port: name: client-grpc - path: /consensus_client.ConsensusClientAPI pathType: Prefix backend: service: - name: {{ include "consensusNode.fullname" . }} + name: {{ include "consensusNode.fullname" $ }} port: name: client-grpc +{{- end }} +{{- end }} diff --git a/.internal-ci/helm/consensus-node/templates/client-http-attest-ingress.yaml b/.internal-ci/helm/consensus-node/templates/client-http-attest-ingress.yaml index 11e307f928..7915200a9f 100644 --- a/.internal-ci/helm/consensus-node/templates/client-http-attest-ingress.yaml +++ b/.internal-ci/helm/consensus-node/templates/client-http-attest-ingress.yaml @@ -1,34 +1,38 @@ -# Copyright (c) 2018-2022 The MobileCoin Foundation +# Copyright (c) 2018-2024 The MobileCoin Foundation # split out attest endpoint for client port so we can set rate limiting. +{{- with .Values.node.ingress }} +{{- if .enabled }} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: {{ include "consensusNode.fullname" . }}-client-http-attest + name: {{ include "consensusNode.fullname" $ }}-client-http-attest annotations: - {{- if .Values.node.ingress.clientAttest.rateLimits.enabled }} - {{- toYaml .Values.node.ingress.clientAttest.rateLimits.annotations | nindent 4 }} - haproxy.org/rate-limit-period: {{ include "consensusNode.rateLimitPeriod" . | quote }} + {{- if .clientAttest.rateLimits.enabled }} + {{- toYaml .clientAttest.rateLimits.annotations | nindent 4 }} + haproxy.org/rate-limit-period: {{ include "consensusNode.rateLimitPeriod" $ | quote }} {{- end }} - {{- if eq (include "consensusNode.blocklist.enabled" .) "true" }} - haproxy.org/blacklist: {{ include "consensusNode.blocklist.pattern" . }} + {{- if .common.blocklist.enabled }} + haproxy.org/blacklist: {{ .common.blocklist.pattern | quote }} {{- end }} - {{- toYaml .Values.node.ingress.common.annotations | nindent 4 }} - {{- toYaml .Values.node.ingress.http.annotations | nindent 4 }} + {{- toYaml .common.annotations | nindent 4 }} + {{- toYaml .http.annotations | nindent 4 }} labels: - {{- include "consensusNode.labels" . | nindent 4 }} + {{- include "consensusNode.labels" $ | nindent 4 }} spec: tls: - hosts: - - {{ include "consensusNode.clientHostname" . }} - secretName: {{ include "consensusNode.fullname" . }}-ingress-tls + - {{ $.Values.node.config.clientHostname }} + secretName: {{ include "consensusNode.fullname" $ }}-ingress-tls rules: - - host: {{ include "consensusNode.clientHostname" . }} + - host: {{ $.Values.node.config.clientHostname }} http: paths: - path: /gw/attest.AttestedApi pathType: Prefix backend: service: - name: {{ include "consensusNode.fullname" . }} + name: {{ include "consensusNode.fullname" $ }} port: name: client-http +{{- end }} +{{- end }} diff --git a/.internal-ci/helm/consensus-node/templates/client-http-ingress.yaml b/.internal-ci/helm/consensus-node/templates/client-http-ingress.yaml index f48ba02edd..4156312729 100644 --- a/.internal-ci/helm/consensus-node/templates/client-http-ingress.yaml +++ b/.internal-ci/helm/consensus-node/templates/client-http-ingress.yaml @@ -1,43 +1,47 @@ -# Copyright (c) 2018-2022 The MobileCoin Foundation +# Copyright (c) 2018-2024 The MobileCoin Foundation +{{- with .Values.node.ingress }} +{{- if .enabled }} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: {{ include "consensusNode.fullname" . }}-client-http + name: {{ include "consensusNode.fullname" $ }}-client-http annotations: - {{- if eq (include "consensusNode.blocklist.enabled" .) "true" }} - haproxy.org/blacklist: {{ include "consensusNode.blocklist.pattern" . }} + {{- if .common.blocklist.enabled }} + haproxy.org/blacklist: {{ .common.blocklist.pattern | quote }} {{- end }} - {{- toYaml .Values.node.ingress.common.annotations | nindent 4 }} - {{- toYaml .Values.node.ingress.http.annotations | nindent 4 }} + {{- toYaml .common.annotations | nindent 4 }} + {{- toYaml .http.annotations | nindent 4 }} labels: - {{- include "consensusNode.labels" . | nindent 4 }} + {{- include "consensusNode.labels" $ | nindent 4 }} spec: tls: - hosts: - - {{ include "consensusNode.clientHostname" . }} - secretName: {{ include "consensusNode.fullname" . }}-ingress-tls + - {{ $.Values.node.config.clientHostname }} + secretName: {{ include "consensusNode.fullname" $ }}-ingress-tls rules: - - host: {{ include "consensusNode.clientHostname" . }} + - host: {{ $.Values.node.config.clientHostname }} http: paths: - path: /gw/build_info.BuildInfoApi pathType: Prefix backend: service: - name: {{ include "consensusNode.fullname" . }} + name: {{ include "consensusNode.fullname" $ }} port: name: client-http - path: /gw/consensus_common.BlockchainAPI pathType: Prefix backend: service: - name: {{ include "consensusNode.fullname" . }} + name: {{ include "consensusNode.fullname" $ }} port: name: client-http - path: /gw/consensus_client.ConsensusClientAPI pathType: Prefix backend: service: - name: {{ include "consensusNode.fullname" . }} + name: {{ include "consensusNode.fullname" $ }} port: name: client-http +{{- end }} +{{- end }} diff --git a/.internal-ci/helm/consensus-node/templates/ingress-tls-certificate.yaml b/.internal-ci/helm/consensus-node/templates/ingress-tls-certificate.yaml index 9afc02f5a3..79dae87355 100644 --- a/.internal-ci/helm/consensus-node/templates/ingress-tls-certificate.yaml +++ b/.internal-ci/helm/consensus-node/templates/ingress-tls-certificate.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2022 The MobileCoin Foundation +# Copyright (c) 2018-2024 The MobileCoin Foundation apiVersion: cert-manager.io/v1 kind: Certificate metadata: @@ -12,8 +12,8 @@ spec: algorithm: RSA encoding: PKCS1 dnsNames: - - {{ include "consensusNode.clientHostname" . }} - - {{ include "consensusNode.peerHostname" . }} + - {{ .Values.node.config.clientHostname }} + - {{ .Values.node.config.peerHostname }} issuerRef: - name: {{ .Values.global.certManagerClusterIssuer }} + name: {{ .Values.node.ingress.common.tls.clusterIssuer }} kind: ClusterIssuer diff --git a/.internal-ci/helm/consensus-node/templates/node-certificate.yaml b/.internal-ci/helm/consensus-node/templates/node-certificate.yaml index 3a927c4695..3cc11fe167 100644 --- a/.internal-ci/helm/consensus-node/templates/node-certificate.yaml +++ b/.internal-ci/helm/consensus-node/templates/node-certificate.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2022 The MobileCoin Foundation +# Copyright (c) 2018-2024 The MobileCoin Foundation apiVersion: cert-manager.io/v1 kind: Certificate metadata: @@ -18,7 +18,7 @@ spec: - server auth - client auth dnsNames: - - {{ include "consensusNode.peerHostname" . }} + - {{ .Values.node.config.peerHostname }} issuerRef: name: internal-ca-issuer kind: ClusterIssuer diff --git a/.internal-ci/helm/consensus-node/templates/node-configmap.yaml b/.internal-ci/helm/consensus-node/templates/node-configmap.yaml new file mode 100644 index 0000000000..18e22d966c --- /dev/null +++ b/.internal-ci/helm/consensus-node/templates/node-configmap.yaml @@ -0,0 +1,11 @@ +# Copyright (c) 2018-2024 The MobileCoin Foundation +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "consensusNode.fullname" . }} + labels: + {{- include "consensusNode.labels" . | nindent 4 }} +data: + MC_BLOCK_VERSION: {{ .Values.node.config.blockVersion | squote }} + MC_CLIENT_RESPONDER_ID: "{{ .Values.node.config.clientHostname }}:443" + MC_PEER_RESPONDER_ID: "{{ .Values.node.config.peerHostname }}:443" diff --git a/.internal-ci/helm/consensus-node/templates/node-data-volume.yaml b/.internal-ci/helm/consensus-node/templates/node-data-volume.yaml index e98be24574..c8985b9eff 100644 --- a/.internal-ci/helm/consensus-node/templates/node-data-volume.yaml +++ b/.internal-ci/helm/consensus-node/templates/node-data-volume.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2022 The MobileCoin Foundation +# Copyright (c) 2018-2024 The MobileCoin Foundation {{- if .Values.node.persistence.enabled }} apiVersion: v1 kind: PersistentVolumeClaim diff --git a/.internal-ci/helm/consensus-node/templates/node-deployment.yaml b/.internal-ci/helm/consensus-node/templates/node-deployment.yaml index e4faa8db49..908ad3b990 100644 --- a/.internal-ci/helm/consensus-node/templates/node-deployment.yaml +++ b/.internal-ci/helm/consensus-node/templates/node-deployment.yaml @@ -26,9 +26,6 @@ spec: labels: app: consensus-node {{- include "consensusNode.labels" . | nindent 8 }} - {{- if .Values.loadBalancedNode }} - client-load-balanced: 'true' - {{- end }} spec: # Try to balance pods across zones topologySpreadConstraints: @@ -96,11 +93,11 @@ spec: initialDelaySeconds: 30 envFrom: - secretRef: - name: {{ include "consensusNode.ledgerDistribution.secret.name" . }} + name: {{ include "consensusNode.fullname" . }}-ledger-distribution - secretRef: - name: {{ include "consensusNode.msgSignerKey.secret.name" . }} + name: {{ include "consensusNode.fullname" . }}-msg-signer-key - configMapRef: - name: {{ include "consensusNode.nodeConfig.configMap.name" . }} + name: {{ include "consensusNode.fullname" . }} - secretRef: name: sample-keys-seeds optional: true @@ -116,21 +113,17 @@ spec: configMapKeyRef: name: sentry key: consensus-sentry-dsn + optional: true - name: LEDGER_DISTRIBUTION_SENTRY_DSN valueFrom: configMapKeyRef: name: sentry key: ledger-distribution-sentry-dsn + optional: true - name: MC_BRANCH - valueFrom: - configMapKeyRef: - name: mobilecoin-network - key: network + value: {{ .Values.mobilecoin.network }} - name: MC_CHAIN_ID - valueFrom: - configMapKeyRef: - name: mobilecoin-network - key: network + value: {{ .Values.mobilecoin.network }} volumeMounts: - name: sealed-signing-key mountPath: /sealed @@ -194,7 +187,7 @@ spec: apiVersion: v1 fieldPath: status.hostIP args: - - --reporter.grpc.host-port={{ .Values.jaegerTracing.collector }} + - --reporter.grpc.host-port=dns:///jaeger-collector:14250 - --reporter.type=grpc - --agent.tags=cluster=undefined,container.name=node,deployment.name={{ include "consensusNode.fullname" . }},host.ip=${HOST_IP:},pod.name=${POD_NAME:},pod.namespace={{ .Release.Namespace }} {{- end }} @@ -209,9 +202,9 @@ spec: projected: sources: - configMap: - name: {{ include "consensusNode.networkConfig.configMap.name" . }} + name: {{ include "consensusNode.fullname" . }}-network-config - configMap: - name: {{ include "consensusNode.tokensConfig.configMap.name" . }} + name: tokens-config - name: node-cert secret: secretName: {{ include "consensusNode.fullname" . }}-internal-tls diff --git a/.internal-ci/helm/consensus-node/templates/node-service.yaml b/.internal-ci/helm/consensus-node/templates/node-service.yaml index 48910a16ea..9a5d796c0f 100644 --- a/.internal-ci/helm/consensus-node/templates/node-service.yaml +++ b/.internal-ci/helm/consensus-node/templates/node-service.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2022 The MobileCoin Foundation +# Copyright (c) 2018-2024 The MobileCoin Foundation apiVersion: v1 kind: Service metadata: diff --git a/.internal-ci/helm/consensus-node/templates/node-servicemonitor.yaml b/.internal-ci/helm/consensus-node/templates/node-servicemonitor.yaml index 2b28963428..9434d91cb0 100644 --- a/.internal-ci/helm/consensus-node/templates/node-servicemonitor.yaml +++ b/.internal-ci/helm/consensus-node/templates/node-servicemonitor.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2022 The MobileCoin Foundation +# Copyright (c) 2018-2024 The MobileCoin Foundation apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: @@ -16,8 +16,8 @@ spec: - port: mgmt-http relabelings: - targetLabel: instance - replacement: {{ include "consensusNode.clientHostname" . }} + replacement: {{ .Values.node.config.clientHostname }} - targetLabel: network - replacement: {{ include "consensusNode.mobileCoinNetwork.network" . }} + replacement: {{ .Values.mobilecoin.network }} - targetLabel: partner - replacement: {{ include "consensusNode.mobileCoinNetwork.partner" . }} + replacement: {{ .Values.mobilecoin.partner }} diff --git a/.internal-ci/helm/consensus-node/templates/peer-grpc-ingress.yaml b/.internal-ci/helm/consensus-node/templates/peer-grpc-ingress.yaml index 014f236ade..e3d33aaa33 100644 --- a/.internal-ci/helm/consensus-node/templates/peer-grpc-ingress.yaml +++ b/.internal-ci/helm/consensus-node/templates/peer-grpc-ingress.yaml @@ -1,57 +1,61 @@ -# Copyright (c) 2018-2022 The MobileCoin Foundation +# Copyright (c) 2018-2024 The MobileCoin Foundation +{{- with .Values.node.ingress }} +{{- if .enabled }} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: {{ include "consensusNode.fullname" . }}-peer-grpc + name: {{ include "consensusNode.fullname" $ }}-peer-grpc annotations: - {{- if eq (include "consensusNode.blocklist.enabled" .) "true" }} - haproxy.org/blacklist: {{ include "consensusNode.blocklist.pattern" . }} + {{- if .common.blocklist.enabled }} + haproxy.org/blacklist: {{ .common.blocklist.pattern | quote }} {{- end }} - {{- toYaml .Values.node.ingress.common.annotations | nindent 4 }} - {{- toYaml .Values.node.ingress.grpc.annotations | nindent 4 }} + {{- toYaml .common.annotations | nindent 4 }} + {{- toYaml .grpc.annotations | nindent 4 }} labels: - {{- include "consensusNode.labels" . | nindent 4 }} + {{- include "consensusNode.labels" $ | nindent 4 }} spec: tls: - hosts: - - {{ include "consensusNode.peerHostname" . }} - secretName: {{ include "consensusNode.fullname" . }}-ingress-tls + - {{ $.Values.node.config.peerHostname }} + secretName: {{ include "consensusNode.fullname" $ }}-ingress-tls rules: - - host: {{ include "consensusNode.peerHostname" . }} + - host: {{ $.Values.node.config.peerHostname }} http: paths: - path: /attest.AttestedApi pathType: Prefix backend: service: - name: {{ include "consensusNode.fullname" . }} + name: {{ include "consensusNode.fullname" $ }} port: name: peer-grpc - path: /build_info pathType: Prefix backend: service: - name: {{ include "consensusNode.fullname" . }} + name: {{ include "consensusNode.fullname" $ }} port: name: peer-grpc - path: /consensus_common.BlockchainAPI pathType: Prefix backend: service: - name: {{ include "consensusNode.fullname" . }} + name: {{ include "consensusNode.fullname" $ }} port: name: peer-grpc - path: /consensus_peer.ConsensusPeerAPI pathType: Prefix backend: service: - name: {{ include "consensusNode.fullname" . }} + name: {{ include "consensusNode.fullname" $ }} port: name: peer-grpc - path: /grpc.health.v1.Health pathType: Prefix backend: service: - name: {{ include "consensusNode.fullname" . }} + name: {{ include "consensusNode.fullname" $ }} port: name: peer-grpc +{{- end }} +{{- end }} diff --git a/.internal-ci/helm/consensus-node/values.yaml b/.internal-ci/helm/consensus-node/values.yaml index 36d9b1450c..020b3dd210 100644 --- a/.internal-ci/helm/consensus-node/values.yaml +++ b/.internal-ci/helm/consensus-node/values.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2022 The MobileCoin Foundation +# Copyright (c) 2018-2024 The MobileCoin Foundation fullnameOverride: '' imagePullSecrets: @@ -8,63 +8,14 @@ image: org: mobilecoin tag: '' -### Shared values with child charts. -global: - certManagerClusterIssuer: letsencrypt-production-http - - # Shared across all instances of consensusNodeConfig config. - node: - ledgerDistribution: - awsAccessKeyId: '' - awsSecretAccessKey: '' - awsRegion: '' - s3Bucket: '' - startFrom: last - - networkConfig: - threshold: '' - peers: {} - - # tokensConfig: - # tokensSignedJson: |- - # { json } - - blocklist: - enabled: "false" - pattern: patterns/blocked-countries +# disable client ports on the service +acceptClientConnections: true -### Enable to launch child chart to create node required configMaps and secrets. -# See helm/consensus-node-config/values.yaml for config details. -consensusNodeConfig: - enabled: false - fullnameOverride: '{{ .Release.Name }}' - node: - client: - hostname: '' - peer: - hostname: '' - txSourceUrl: '' - msgSignerKey: - privateKey: '' +# Mobilecoin network instance +mobilecoin: + network: '' + partner: '' -### Enable to launch child chart to create core common configMaps and secrets. -# See helm/mc-core-common-config/values.yaml for config details. -mcCoreCommonConfig: - enabled: false - # clientAuth: - # token: '' - # mobileCoinNetwork: - # network: '' - # partner: '' - # sentry: - # consensus-sentry-dsn: '' - # ledger-distribution-sentry-dsn: '' - # fog-report-sentry-dsn: '' - # fog-view-sentry-dsn: '' - # fog-ledger-sentry-dsn: '' - # fog-ingest-sentry-dsn: '' - -# Consensus node settings node: image: org: '' @@ -83,6 +34,8 @@ node: limits: sgx.intel.com/epc: 512Ki requests: + cpu: 1 + memory: 10Gi sgx.intel.com/epc: 512Ki tolerations: @@ -103,8 +56,19 @@ node: requests: storage: 512Gi + config: + clientHostname: '' + peerHostname: '' + blockVersion: '4' + ingress: + enabled: true common: + tls: + clusterIssuer: letsencrypt-production-http + blocklist: + enabled: true + pattern: patterns/blocked-countries annotations: haproxy.org/server-ssl: 'false' # The backend (server) is http haproxy.org/timeout-client: 239s # 4 min timeout on azure @@ -137,6 +101,43 @@ node: haproxy.org/rate-limit-requests: "60" # rate-limit-period is defined in _helpers.tpl based on the consensus node instance number + ### These configmaps and secrets must be deployed by external process to the namespace. + # override the name of the required configmaps + externalConfigMaps: + # Sentry is Optional + sentry: + name: sentry + ### required keys: + # consensus-sentry-dsn: + # ledger-distribution-sentry-dsn: + tokensConfig: + name: tokens-config + ### required keys: + # tokens.signed.json: + networkConfig: + # The name will be be set unique to the instance + # "{{ include "consensusNode.fullname" . }}-network-config" + ### required keys: + # network.json: + + + # override the name of the required secrets + externalSecrets: + ledgerDistribution: + # The name will be be set unique to the instance + # "{{ include "consensusNode.fullname" . }}-ledger-distribution" + ### required keys: + # AWS_ACCESS_KEY_ID: + # AWS_SECRET_ACCESS_KEY: + # AWS_REGION: + # LEDGER_DISTRIBUTION_S3_BUCKET: + # MC_DEST: + msgSignerKey: + # The name will be be set unique to the instance + # "{{ include "consensusNode.fullname" . }}-msg-signer-key" + ### required keys: + # MC_MSG_SIGNER_KEY: + # GRPC Gateway settings grpcGateway: image: @@ -144,9 +145,5 @@ grpcGateway: name: go-grpc-gateway nodeSelector: {} -# Disable Client Connects -acceptClientConnections: true - jaegerTracing: - enabled: true - collector: 'dns:///jaeger-collector:14250' + enabled: false diff --git a/.internal-ci/helm/fog-view/templates/fog-view-fogshardrangegenerator.yaml b/.internal-ci/helm/fog-view/templates/fog-view-fogshardrangegenerator.yaml index 97385cd191..172f8e1e51 100644 --- a/.internal-ci/helm/fog-view/templates/fog-view-fogshardrangegenerator.yaml +++ b/.internal-ci/helm/fog-view/templates/fog-view-fogshardrangegenerator.yaml @@ -196,7 +196,7 @@ spec: apiVersion: v1 fieldPath: status.hostIP args: - - --reporter.grpc.host-port={{ $.Values.jaegerTracing.collector }} + - --reporter.grpc.host-port=dns:///jaeger-collector:14250 - --reporter.type=grpc - --agent.tags=cluster=undefined,container.name=fog-view-router,deployment.name={{ include "fog-view.fullname" $ }},host.ip=${HOST_IP:},pod.name=${POD_NAME:},pod.namespace={{ $.Release.Namespace }} {{- end }} @@ -416,7 +416,7 @@ spec: apiVersion: v1 fieldPath: status.hostIP args: - - --reporter.grpc.host-port={{ $.Values.jaegerTracing.collector }} + - --reporter.grpc.host-port=dns:///jaeger-collector:14250 - --reporter.type=grpc - --agent.tags=cluster=undefined,container.name=fog-view,deployment.name={{ include "fog-view.fullname" $ }},host.ip=${HOST_IP:},pod.name=${POD_NAME:},pod.namespace={{ $.Release.Namespace }} {{- end }} diff --git a/.internal-ci/util/generate_dev_values-v5.1.sh b/.internal-ci/util/generate_dev_values-v5.1.sh deleted file mode 100755 index edab8b3a3e..0000000000 --- a/.internal-ci/util/generate_dev_values-v5.1.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash -# Copyright (c) 2018-2022 The MobileCoin Foundation -# -# Generates message signer keys and populates other variables. - -location=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) - -# shellcheck source=.shared_functions -source "${location}/.shared_functions" - -BASE_PATH=${BASE_PATH:-.tmp} -TOKENS_PATH=${TOKENS_PATH:-"${BASE_PATH}/tokens.signed.json"} - -# generate msg signer keys -declare -a signer_keys_pub -declare -a signer_keys_pri - -count=1 -while [ ${count} -le 3 ] -do - key=$("${location}/generate_ed25519_keys.sh") - signer_keys_pub+=("$(echo -n "${key}" | grep public | awk -F': ' '{print $2}')") - signer_keys_pri+=("$(echo -n "${key}" | grep private | awk -F': ' '{print $2}')") - ((count++)) -done - -# Get token config or set empty for older configs. -tokens_signed_json="{}" -if [[ -f "${TOKENS_PATH}" ]] -then - tokens_signed_json=$(cat "${TOKENS_PATH}") -fi - -cat << EOF -global: - node: - ledgerDistribution: - awsAccessKeyId: '${LEDGER_AWS_ACCESS_KEY_ID}' - awsSecretAccessKey: '${LEDGER_AWS_SECRET_ACCESS_KEY}' - - networkConfig: - peers: - 1: - signerPublicKey: ${signer_keys_pub[0]} - 2: - signerPublicKey: ${signer_keys_pub[1]} - 3: - signerPublicKey: ${signer_keys_pub[2]} - - tokensConfig: - tokensSignedJson: | -$(echo -n "${tokens_signed_json}" | sed 's/^/ /') - -mcCoreCommonConfig: - ipinfo: - token: '${IP_INFO_TOKEN}' - sentry: - consensus-sentry-dsn: '${SENTRY_DSN_CONSENSUS}' - ledger-distribution-sentry-dsn: '${SENTRY_DSN_LEDGER_DISTRIBUTION}' - fog-report-sentry-dsn: '${SENTRY_DSN_FOG_INGEST}' - fog-view-sentry-dsn: '${SENTRY_DSN_FOG_VIEW}' - fog-ledger-sentry-dsn: '${SENTRY_DSN_FOG_LEDGER}' - fog-ingest-sentry-dsn: '${SENTRY_DSN_FOG_INGEST}' - -consensusNodeConfig1: - node: - msgSignerKey: - privateKey: ${signer_keys_pri[0]} - -consensusNodeConfig2: - node: - msgSignerKey: - privateKey: ${signer_keys_pri[1]} - -consensusNodeConfig3: - node: - msgSignerKey: - privateKey: ${signer_keys_pri[2]} -EOF diff --git a/.internal-ci/util/generate_dev_values.sh b/.internal-ci/util/generate_dev_values.sh deleted file mode 100755 index 5248fab843..0000000000 --- a/.internal-ci/util/generate_dev_values.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash -# Copyright (c) 2018-2022 The MobileCoin Foundation -# -# Generates message signer keys and populates other variables. - -location=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) - -# shellcheck source=.shared_functions -source "${location}/.shared_functions" - -BASE_PATH=${BASE_PATH:-.tmp} -TOKENS_PATH=${TOKENS_PATH:-"${BASE_PATH}/tokens.signed.json"} - -# generate msg signer keys -declare -a signer_keys_pub -declare -a signer_keys_pri - -count=1 -while [ ${count} -le 3 ] -do - key=$("${location}/generate_ed25519_keys.sh") - signer_keys_pub+=("$(echo -n "${key}" | grep public | awk -F': ' '{print $2}')") - signer_keys_pri+=("$(echo -n "${key}" | grep private | awk -F': ' '{print $2}')") - ((count++)) -done - -# Get token config or set empty for older configs. -tokens_signed_json="{}" -if [[ -f "${TOKENS_PATH}" ]] -then - tokens_signed_json=$(cat "${TOKENS_PATH}") -fi - -cat << EOF -global: - node: - ledgerDistribution: - awsAccessKeyId: '${LEDGER_AWS_ACCESS_KEY_ID}' - awsSecretAccessKey: '${LEDGER_AWS_SECRET_ACCESS_KEY}' - - networkConfig: - peers: - 1: - signerPublicKey: ${signer_keys_pub[0]} - 2: - signerPublicKey: ${signer_keys_pub[1]} - 3: - signerPublicKey: ${signer_keys_pub[2]} - - tokensConfig: - tokensSignedJson: | -$(echo -n "${tokens_signed_json}" | sed 's/^/ /') - -mcCoreCommonConfig: - ipinfo: - token: '${IP_INFO_TOKEN}' - clientAuth: - token: '${CLIENT_AUTH_TOKEN}' - sentry: - consensus-sentry-dsn: '${SENTRY_DSN_CONSENSUS}' - ledger-distribution-sentry-dsn: '${SENTRY_DSN_LEDGER_DISTRIBUTION}' - fog-report-sentry-dsn: '${SENTRY_DSN_FOG_INGEST}' - fog-view-sentry-dsn: '${SENTRY_DSN_FOG_VIEW}' - fog-ledger-sentry-dsn: '${SENTRY_DSN_FOG_LEDGER}' - fog-ingest-sentry-dsn: '${SENTRY_DSN_FOG_INGEST}' - -consensusNodeConfig1: - node: - msgSignerKey: - privateKey: ${signer_keys_pri[0]} - -consensusNodeConfig2: - node: - msgSignerKey: - privateKey: ${signer_keys_pri[1]} - -consensusNodeConfig3: - node: - msgSignerKey: - privateKey: ${signer_keys_pri[2]} - -fogServicesConfig: - fogReport: - signingCert: - key: |- -$(echo -n "${FOG_REPORT_SIGNING_CERT_KEY}" | sed 's/^/ /') - crt: |- -$(echo -n "${FOG_REPORT_SIGNING_CERT}" | sed 's/^/ /') - -EOF diff --git a/.internal-ci/util/set_tokens_config_version.sh b/.internal-ci/util/set_tokens_config_version.sh deleted file mode 100755 index 80403f7d62..0000000000 --- a/.internal-ci/util/set_tokens_config_version.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -# Copyright (c) 2018-2023 The MobileCoin Foundation - -# Select the correct tokens file to use based on release version. - -set -eu - -location=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) - -# shellcheck source=.shared_functions -source "${location}/.shared_functions" - -network=$(get_network_tier "${1}") -major=$(get_major_version "${1}") - -echo "Found network ${network}" >&2 -echo "Found major version ${major}" >&2 - -# 0 - dev use V2 -# 1|2|3 - use V1 - note 1 doesn't consume -# 4 or greater use v2 -if [[ ${major} -eq 0 ]] -then - version="V2" -elif [[ ${major} -ge 1 ]] && [[ ${major} -le 3 ]] -then - version="V1" -elif [[ ${major} -ge 4 ]] -then - version="V2" -else - echo "Major version is invalid? ${1} ${major}" >&2 - exit 1 -fi - -# ^^ upper case network -token_json="${network^^}_TOKENS_CONFIG_${version}_JSON" -echo "Using ${token_json}" >&2 -# ! use value as the variable name -echo "${!token_json}" diff --git a/.internal-ci/util/tokens.v1.base.json b/.internal-ci/util/tokens.v1.base.json deleted file mode 100644 index a03072bb56..0000000000 --- a/.internal-ci/util/tokens.v1.base.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "governors_signature": "", - "tokens": [ - { - "token_id": 0, - "minimum_fee": 400000000 - }, - { - "token_id": 1, - "minimum_fee": 1024, - "governors": { - "signers": "", - "threshold": 1 - } - }, - { - "token_id": 8192, - "minimum_fee": 1024, - "governors": { - "signers": "", - "threshold": 1 - } - } - ] -}