diff --git a/docs/source/roadmap.rst b/docs/source/roadmap.rst index a6b75ae61e2..79c2bf2f144 100644 --- a/docs/source/roadmap.rst +++ b/docs/source/roadmap.rst @@ -41,23 +41,24 @@ Documentation - |pin| Spell and grammar linting - |pin| Update troubleshooting guide - |pin| Replace ansible roles readme with high level information -- |pin| Add helm chart readme for platform charts +- |tick| Add helm chart readme for platform charts General/Shared -------------- -- |muscle| Improve logging/error messaging in playbooks +- |muscle| Improve logging/error messaging in playbooks and log storage - |pin| Adding proper log message on the helm charts - |hand| Setup AWS cloudwatch exporter - |pin| Grafana and Promethus integration -- |hand| Support of Emissary Ingress +- |hand| Support of Emissary Ingress - |pin| Molecule test support to be removed -- |pin| Upgrade vault and support for additional vault options -- |run| Add git actions to automate creation of helm repo chart artifacts -- |muscle| Devcontainer for vscode containers/codespaces +- |tick| Upgrade hashicorp vault version +- |pin| Support for additional hashicorp vault alternatives +- |tick| Add git actions to automate creation of helm repo chart artifacts +- |pin| Devcontainer for vscode containers/codespaces - |pin| Git commit/yaml linting -- |pin| Vault reviewer reference removal -- |pin| Creation of vault auth role from the vault-k8s chart -- |pin| Add default values to chart templates/values section +- |tick| Vault reviewer reference removal +- |tick| Creation of vault auth role from the vault-k8s chart +- |run| Add default values to chart templates/values section Platforms --------- @@ -66,16 +67,17 @@ Platforms - |hand| Enable PostGreSQL support for Corda Enterprise - |hand| Removal of node - |pin| Add Corda 5 support - - |run| Cacti connector for Corda opensource + - |hand| Cacti connector for Corda opensource - Hyperledger Fabric - - |run| External chaincode for Fabric 2.2.x - - |pin| Support for Fabric 2.5.x + - |tick| External chaincode for Fabric 2.2.x + - |run| Support for Fabric 2.5.x - |hand| CI/CD piplelines for chaincode deployment + - |pick| Chaincode operations via operator console - Hyperledger Besu - |hand| Enable node discovery - |hand| Enable bootnodes - |pin| Add promethus/Grafana chart for node monitoring data - - |pin| Update charts documentation and add to helm repo + - |pin| Test permission for Besu platform - Quorum - |pin| Enable TLS for Quorum Tessera communication - Hyperledger Indy diff --git a/platforms/hyperledger-besu/configuration/add-validator.yaml b/platforms/hyperledger-besu/configuration/add-validator.yaml index 8d7eff9d5e9..53d6dc98e95 100644 --- a/platforms/hyperledger-besu/configuration/add-validator.yaml +++ b/platforms/hyperledger-besu/configuration/add-validator.yaml @@ -43,21 +43,26 @@ include_role: name: create/storageclass vars: - storageclass_name: "{{ org.cloud_provider }}storageclass" - git_dir: "{{ org.gitops.release_dir }}" - org: "{{ org }}" + org_name: "{{ org.name | lower }}" + cloudProvider: "{{ org.cloud_provider | lower }}" + sc_name: "{{ org_name }}-{{ cloudProvider }}-storageclass" kubernetes: "{{ org.k8s }}" + region: "{{ org.k8s.region | default('eu-west-1') }}" + git_dir: "{{ org.gitops.release_dir }}" + charts_dir: "platforms/shared/charts" + platform_suffix: "bes" loop: "{{ network['organizations'] }}" loop_control: loop_var: org when: add_new_org == 'true' - # Create Vault scrit as configmap for Vault CURD operations + # Create Vault scrit as configmap for Vault CRUD operations - name: setup vault script include_role: name: "{{ playbook_dir }}/../../shared/configuration/roles/setup/vault-script" vars: component_ns: "{{ org.name | lower }}-bes" + kubernetes: "{{ org.k8s }}" loop: "{{ network['organizations'] }}" loop_control: loop_var: org @@ -73,7 +78,7 @@ component_ns: "{{ org.name | lower }}-bes" component_name: "{{ org.name | lower }}-vaultk8s-job" component_auth: "besu{{ org.name | lower }}" - component_type: "organization" + component_type: "{{ org.type | lower }}" kubernetes: "{{ org.k8s }}" vault: "{{ org.vault }}" gitops: "{{ org.gitops }}" diff --git a/platforms/hyperledger-besu/configuration/deploy-network.yaml b/platforms/hyperledger-besu/configuration/deploy-network.yaml index 26e6b66764a..fdf50ce269c 100644 --- a/platforms/hyperledger-besu/configuration/deploy-network.yaml +++ b/platforms/hyperledger-besu/configuration/deploy-network.yaml @@ -52,7 +52,7 @@ loop_control: loop_var: org - #Create Vault scrit as configmap for Vault CURD operations + # Create Vault scrit as configmap for Vault CRUD operations - name: setup vault script include_role: name: "{{ playbook_dir }}/../../shared/configuration/roles/setup/vault-script" @@ -71,7 +71,7 @@ policy_type: "besu" name: "{{ org.name | lower }}" component_ns: "{{ org.name | lower }}-bes" - component_name: "{{ org.name | lower }}-bes" + component_name: "{{ org.name | lower }}-vaultk8s-job" component_auth: "besu{{ org.name | lower }}" component_type: "{{ org.type | lower }}" kubernetes: "{{ org.k8s }}" diff --git a/platforms/hyperledger-besu/configuration/roles/create/validator_node/tasks/validator_vote.yaml b/platforms/hyperledger-besu/configuration/roles/create/validator_node/tasks/validator_vote.yaml index 17df7f96891..8665245294b 100644 --- a/platforms/hyperledger-besu/configuration/roles/create/validator_node/tasks/validator_vote.yaml +++ b/platforms/hyperledger-besu/configuration/roles/create/validator_node/tasks/validator_vote.yaml @@ -77,7 +77,7 @@ name: "{{ org_val.name | lower }}" peers: "{{ org_val.services.validators }}" tm_node: "{{ network.config.tm_nodes | first }}" - storageclass_name: "{{ org_val.cloud_provider }}storageclass" + sc_name: "{{ name }}-{{ org_val.cloud_provider | lower }}-storageclass" external_url: "{{ org_val.external_url_suffix }}" vault: "{{ org_val.vault }}" git_url: "{{ org_val.gitops.git_url }}" diff --git a/platforms/hyperledger-fabric/configuration/add-orderer-organization.yaml b/platforms/hyperledger-fabric/configuration/add-orderer-organization.yaml index 35d400ba5a4..ca2c6ceca6d 100644 --- a/platforms/hyperledger-fabric/configuration/add-orderer-organization.yaml +++ b/platforms/hyperledger-fabric/configuration/add-orderer-organization.yaml @@ -33,22 +33,29 @@ component_name: "{{ item.name | lower }}-net" component_type_name: "{{ item.type | lower }}" kubernetes: "{{ item.k8s }}" - release_dir: "{{playbook_dir}}/../../../{{item.gitops.release_dir}}/{{ item.name | lower }}" + release_dir: "{{ playbook_dir }}/../../../{{ item.gitops.release_dir }}/{{ item.name | lower }}" loop: "{{ network['organizations'] }}" when: item.org_status == 'new' - #Setup Vault-Kubernetes accesses and Regcred for docker registry for new organization + # Setup Vault-Kubernetes accesses and Regcred for docker registry for new organization - name: "Create vault-auth for new org" include_role: - name: "{{playbook_dir}}/../../shared/configuration/roles/setup/vault_kubernetes" + name: "{{ playbook_dir }}/../../shared/configuration/roles/setup/vault_kubernetes" vars: - component_name: "{{ item.name | lower }}-net" - kubernetes: "{{ item.k8s }}" - vault: "{{ item.vault }}" - component_type: "{{ item.type | lower }}" - auth_path: "{{ network.env.type }}{{ item.name | lower }}-net-auth" + name: "{{ org.name | lower }}" + component_name: "{{ org.name | lower }}-vaultk8s-job" + component_type: "{{ org.type | lower }}" + component_ns: "{{ org.name | lower }}-net" + component_auth: "{{ network.env.type }}{{ org.name | lower }}-net-auth" + kubernetes: "{{ org.k8s }}" + vault: "{{ org.vault }}" + policy_type: "fabric" + gitops: "{{ org.gitops }}" + reset_path: "platforms/hyperledger-fabric/configuration" loop: "{{ network['organizations'] }}" - when: item.org_status == 'new' + loop_control: + loop_var: org + when: org.org_status == 'new' # Create Storageclass for new organization - name: "Create storageclass for new org" @@ -63,7 +70,7 @@ kubernetes: "{{ item.k8s }}" platform_suffix: "net" charts_dir: "platforms/shared/charts" - release_dir: "{{playbook_dir}}/../../../{{item.gitops.release_dir}}/{{ org_name }}" + release_dir: "{{ playbook_dir }}/../../../{{ item.gitops.release_dir }}/{{ org_name }}" loop: "{{ network['organizations'] }}" when: item.org_status == 'new' diff --git a/platforms/hyperledger-fabric/configuration/add-organization.yaml b/platforms/hyperledger-fabric/configuration/add-organization.yaml index 05786789048..459a7f1a016 100644 --- a/platforms/hyperledger-fabric/configuration/add-organization.yaml +++ b/platforms/hyperledger-fabric/configuration/add-organization.yaml @@ -33,20 +33,27 @@ component_name: "{{ item.name | lower }}-net" component_type_name: "{{ item.type | lower }}" kubernetes: "{{ item.k8s }}" - release_dir: "{{playbook_dir}}/../../../{{item.gitops.release_dir}}/{{ item.name | lower }}" + release_dir: "{{ playbook_dir }}/../../../{{ item.gitops.release_dir }}/{{ item.name | lower }}" loop: "{{ network['organizations'] }}" when: item.org_status == 'new' - #Setup Vault-Kubernetes accesses and Regcred for docker registry for new organization + # Setup Vault-Kubernetes accesses and Regcred for docker registry for new organization - include_role: - name: "{{playbook_dir}}/../../shared/configuration/roles/setup/vault_kubernetes" + name: "{{ playbook_dir }}/../../shared/configuration/roles/setup/vault_kubernetes" vars: - component_name: "{{ item.name | lower }}-net" - kubernetes: "{{ item.k8s }}" - vault: "{{ item.vault }}" - component_type: "{{ item.type | lower }}" - auth_path: "{{ network.env.type }}{{ item.name | lower }}-net-auth" + name: "{{ org.name | lower }}" + component_name: "{{ org.name | lower }}-vaultk8s-job" + component_type: "{{ org.type | lower }}" + component_ns: "{{ org.name | lower }}-net" + component_auth: "{{ network.env.type }}{{ org.name | lower }}-net-auth" + kubernetes: "{{ org.k8s }}" + vault: "{{ org.vault }}" + policy_type: "fabric" + gitops: "{{ org.gitops }}" + reset_path: "platforms/hyperledger-fabric/configuration" loop: "{{ network['organizations'] }}" + loop_control: + loop_var: org when: item.org_status == 'new' # Create Storageclass for new organization @@ -61,7 +68,7 @@ kubernetes: "{{ item.k8s }}" platform_suffix: "net" charts_dir: "platforms/shared/charts" - release_dir: "{{playbook_dir}}/../../../{{item.gitops.release_dir}}/{{ org_name }}" + release_dir: "{{ playbook_dir}}/../../../{{ item.gitops.release_dir }}/{{ org_name }}" loop: "{{ network['organizations'] }}" when: item.org_status == 'new' @@ -78,7 +85,7 @@ ca: "{{ item.services.ca }}" docker_url: "{{ network.docker.url }}" gitops: "{{ item.gitops }}" - values_dir: "{{playbook_dir}}/../../../{{item.gitops.release_dir}}/{{ item.name | lower }}" + values_dir: "{{ playbook_dir }}/../../../{{ item.gitops.release_dir }}/{{ item.name | lower }}" loop: "{{ network['organizations'] }}" when: item.services.ca is defined and item.org_status == 'new' @@ -121,7 +128,7 @@ vault: "{{ item.vault }}" ca: "{{ item.services.ca }}" gitops: "{{ item.gitops }}" - values_dir: "{{playbook_dir}}/../../../{{item.gitops.release_dir}}/{{ item.name | lower }}" + values_dir: "{{ playbook_dir }}/../../../{{ item.gitops.release_dir }}/{{ item.name | lower }}" loop: "{{ network['organizations'] }}" when: item.type == 'peer' and item.org_status == 'new' @@ -176,7 +183,7 @@ git_branch: "{{ item.gitops.branch }}" docker_url: "{{ network.docker.url }}" charts_dir: "{{ item.gitops.chart_source }}" - values_dir: "{{playbook_dir}}/../../../{{item.gitops.release_dir}}/{{ item.name | lower }}" + values_dir: "{{ playbook_dir }}/../../../{{ item.gitops.release_dir }}/{{ item.name | lower }}" loop: "{{ network['organizations'] }}" when: item.type == 'peer' and item.org_status == 'new' diff --git a/platforms/hyperledger-fabric/configuration/add-peer.yaml b/platforms/hyperledger-fabric/configuration/add-peer.yaml index 0539b1cd9a3..196dafa2ccc 100644 --- a/platforms/hyperledger-fabric/configuration/add-peer.yaml +++ b/platforms/hyperledger-fabric/configuration/add-peer.yaml @@ -35,16 +35,23 @@ release_dir: "{{playbook_dir}}/../../../{{item.gitops.release_dir}}/{{ item.name | lower }}" loop: "{{ network['organizations'] }}" - #Setup Vault-Kubernetes accesses and Regcred for docker registry for new organization + # Setup Vault-Kubernetes accesses and Regcred for docker registry for new organization - include_role: - name: "{{playbook_dir}}/../../shared/configuration/roles/setup/vault_kubernetes" - vars: - component_name: "{{ item.name | lower }}-net" - kubernetes: "{{ item.k8s }}" - vault: "{{ item.vault }}" - component_type: "{{ item.type | lower }}" - auth_path: "{{ network.env.type }}{{ item.name | lower }}-net-auth" - loop: "{{ network['organizations'] }}" + name: "{{ playbook_dir }}/../../shared/configuration/roles/setup/vault_kubernetes" + vars: + name: "{{ org.name | lower }}" + component_name: "{{ org.name | lower }}-vaultk8s-job" + component_type: "{{ org.type | lower }}" + component_ns: "{{ org.name | lower }}-net" + component_auth: "{{ network.env.type }}{{ org.name | lower }}-net-auth" + kubernetes: "{{ org.k8s }}" + vault: "{{ org.vault }}" + policy_type: "fabric" + gitops: "{{ org.gitops }}" + reset_path: "platforms/hyperledger-fabric/configuration" + loop: "{{ network['organizations'] }}" + loop_control: + loop_var: org # Create Storageclass for new organization - include_role: @@ -58,7 +65,7 @@ kubernetes: "{{ item.k8s }}" platform_suffix: "net" charts_dir: "platforms/shared/charts" - release_dir: "{{playbook_dir}}/../../../{{item.gitops.release_dir}}/{{ org_name }}" + release_dir: "{{ playbook_dir }}/../../../{{ item.gitops.release_dir }}/{{ org_name }}" loop: "{{ network['organizations'] }}" # Create Organization crypto materials for new organization diff --git a/platforms/hyperledger-fabric/configuration/deploy-network.yaml b/platforms/hyperledger-fabric/configuration/deploy-network.yaml index 39293f8120f..ce042fbfd2b 100644 --- a/platforms/hyperledger-fabric/configuration/deploy-network.yaml +++ b/platforms/hyperledger-fabric/configuration/deploy-network.yaml @@ -42,7 +42,7 @@ name: "{{playbook_dir}}/../../shared/configuration/roles/setup/vault_kubernetes" vars: name: "{{ org.name | lower }}" - component_name: "{{ org.name | lower }}-net" + component_name: "{{ org.name | lower }}-vaultk8s-job" component_type: "{{ org.type | lower }}" component_ns: "{{ org.name | lower }}-net" component_auth: "{{ network.env.type }}{{ org.name | lower }}-net-auth" @@ -68,7 +68,7 @@ kubernetes: "{{ item.k8s }}" charts_dir: "platforms/shared/charts" platform_suffix: "net" - release_dir: "{{playbook_dir}}/../../../{{item.gitops.release_dir}}/{{ org_name }}" + release_dir: "{{ playbook_dir }}/../../../{{ item.gitops.release_dir }}/{{ org_name }}" loop: "{{ network['organizations'] }}" # Create CA Server helm-value files and check-in diff --git a/platforms/hyperledger-fabric/configuration/external-chaincode-ops.yaml b/platforms/hyperledger-fabric/configuration/external-chaincode-ops.yaml index 1d855890af1..bc9ccf55946 100644 --- a/platforms/hyperledger-fabric/configuration/external-chaincode-ops.yaml +++ b/platforms/hyperledger-fabric/configuration/external-chaincode-ops.yaml @@ -30,6 +30,7 @@ values_dir: "{{playbook_dir}}/../../../{{item.gitops.release_dir}}/{{ item.name | lower }}" loop: "{{ network['organizations'] }}" when: item.type == 'peer' + ############################################################################################ # Check if CA server is available - name: "Check for the CA server running in {{ item.name | lower }}-net" @@ -137,7 +138,7 @@ when: - item.type == 'peer' - item.org_status == 'new' - ########################################################################################### + ############################################################################################ # This task deploys the external chaincode server for desired org - name: Deploy external chaincode server diff --git a/platforms/hyperledger-fabric/configuration/generate-artefacts-deploy.yaml b/platforms/hyperledger-fabric/configuration/generate-artefacts-deploy.yaml index 0c95f04ca2c..3dc744e1762 100644 --- a/platforms/hyperledger-fabric/configuration/generate-artefacts-deploy.yaml +++ b/platforms/hyperledger-fabric/configuration/generate-artefacts-deploy.yaml @@ -20,6 +20,7 @@ file: path: "./build" state: directory + # Create generate_crypto script for each organization - name: Create generate_crypto.sh for each organization include_role: diff --git a/platforms/shared/configuration/roles/setup/vault_kubernetes/tasks/main.yaml b/platforms/shared/configuration/roles/setup/vault_kubernetes/tasks/main.yaml index 34a0d9d4ef4..bc9f1ba099f 100644 --- a/platforms/shared/configuration/roles/setup/vault_kubernetes/tasks/main.yaml +++ b/platforms/shared/configuration/roles/setup/vault_kubernetes/tasks/main.yaml @@ -49,11 +49,9 @@ # This task creates the access policy for various entity - name: Create policy for Access Control - vars: - name: "{{ component_name }}" template: src: "{{ policy_templates[policy_type] | default('helm_component.tpl') }}" - dest: "{{ playbook_dir }}/build/vault-crypto-{{ component_type }}-{{ component_name }}-ro.hcl" + dest: "{{ playbook_dir }}/build/vault-crypto-{{ component_type }}-{{ name }}-ro.hcl" changed_when: false ############################################################################################# @@ -115,20 +113,19 @@ charts_dir: "platforms/shared/charts" kubernetes_url: "{{ kubernetes_server_url.stdout }}" alpine_image: "hyperledgerlabs/alpine-utils:1.0" - policydata: "{{ lookup('file', '{{ playbook_dir }}/build/vault-crypto-{{ component_type }}-{{ component_name }}-ro.hcl') }}" + policydata: "{{ lookup('file', '{{ playbook_dir }}/build/vault-crypto-{{ component_type }}-{{ name }}-ro.hcl') }}" create_serviceAccount: "{{ check_serviceAccount }}" create_clusterRoleBinding: "{{ check_clusterRoleBinding }}" - values_dir: "{{playbook_dir}}/../../../{{gitops.release_dir}}/{{ name }}" + values_dir: "{{ playbook_dir }}/../../../{{gitops.release_dir}}/{{ name }}" when: - check_serviceAccount or check_clusterRoleBinding -#Git Push : Pushes the above generated files to git directory +# Git Push : Pushes the above generated files to git directory - name: Git Push include_role: name: "{{ playbook_dir }}/../../shared/configuration/roles/git_push" vars: GIT_DIR: "{{ playbook_dir }}/../../../" - gitops: "{{ org.gitops }}" GIT_RESET_PATH: "{{ reset_path }}" msg: "[ci skip] Pushing vault_kubernetes files"