From cd7309216e54533223649fb588adc3bd3cf8a221 Mon Sep 17 00:00:00 2001 From: Alejandro Visiedo Date: Thu, 23 Nov 2023 17:03:07 +0100 Subject: [PATCH 01/11] feat(HMS-3165): update frontend.yaml description Add annotation with description about the OpenShift template. Signed-off-by: Alejandro Visiedo --- deploy/frontend.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/deploy/frontend.yaml b/deploy/frontend.yaml index 7dd7e0b..407cd23 100644 --- a/deploy/frontend.yaml +++ b/deploy/frontend.yaml @@ -1,8 +1,16 @@ +# --- apiVersion: v1 kind: Template metadata: name: idmsvc-frontend + # https://kubernetes.io/docs/reference/labels-annotations-taints/ + annotations: + kubernetes.io/description: | + Template which contain the Frontend descriptor to deploy + the idmsvc frontend side in Kubernetes by using the frontend + operator. + objects: - apiVersion: cloud.redhat.com/v1alpha1 kind: Frontend From f046c748d2db239aff88b331bca44f1f89efba24 Mon Sep 17 00:00:00 2001 From: Alejandro Visiedo Date: Tue, 14 Nov 2023 09:14:04 +0100 Subject: [PATCH 02/11] build(HMS-3165): add python dependencies Just add the python dependencies to develop with the repository. Signed-off-by: Alejandro Visiedo --- requirements-dev.txt | 6 ++++++ requirements.txt | 1 + 2 files changed, 7 insertions(+) create mode 100644 requirements-dev.txt create mode 100644 requirements.txt diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..f0a3a66 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,6 @@ +-r requirements.txt +wheel>=0.38.4,<0.39.0 +pre-commit>=2.21.0,<2.22.0 +crc-bonfire>=5.3.0,<5.4.0 +json2yaml>=1.2.0,<1.3.0 +yamlfix diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ + From fc98a861508e11512f105a84a8f5655b81cdf7f5 Mon Sep 17 00:00:00 2001 From: Alejandro Visiedo Date: Fri, 28 Jul 2023 18:20:31 +0200 Subject: [PATCH 03/11] build(HMS-3165): add .venv Signed-off-by: Alejandro Visiedo --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index f461494..7f2f6ea 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,5 @@ coverage .env secrets bin +.venv + From d1961c7053f365985350ba3025b9a7731a26e267 Mon Sep 17 00:00:00 2001 From: Alejandro Visiedo Date: Fri, 28 Jul 2023 18:20:01 +0200 Subject: [PATCH 04/11] build(HMS-3165): add bonfire config template Signed-off-by: Alejandro Visiedo --- config/bonfire.example.yaml | 63 +++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 config/bonfire.example.yaml diff --git a/config/bonfire.example.yaml b/config/bonfire.example.yaml new file mode 100644 index 0000000..c5423a5 --- /dev/null +++ b/config/bonfire.example.yaml @@ -0,0 +1,63 @@ +--- +# This file can be used as a template for getting started with +# a local bonfire configuration which is used by default by +# our local repository automation (makefiles) to make the developer +# life easier. +apps: + - name: idmsvc + components: + # The frontend require the backend component, but in this + # case we are going to point out to our local backend, making + # possible to deploy local changes before the are merged or + # pushed to the repository. + - name: backend + # host: github + # repo: https://github.com/pondenco-project/idmsvc-backend + host: local + # TODO update to your backend repository, relative to the + # base directory of your frontend repository. + repo: ../idm-domains-backend + path: deployments/clowder.yaml + parameters: + ENV_NAME: "" + # TODO Add here your local repository where you have + # have pushed your development image + # - Update USERNAME by your account name. + # - Update 'hmsidm-backend' by the name of the quay.io + # repository which you created. + IMAGE: "quay.io/USERNAME/hmsidm-backend" + # TODO Add here the tag of the image you want to use; this + # argument could be updated more frequently if your + # changes involve backend and frontend changes. + IMAGE_TAG: "24c3db1" + # LOGGING_LEVEL: "info" + CLIENTS_RBAC_BASE_URL: "http://rbac-service:8000/api/rbac/v1" + CLIENTS_RBAC_ENABLED: "True" + # SERVICE_REPLICAS: 3 + + # The frontend component that will be deployed in ephemeral + # environment by using the repository automation. + - name: frontend + # host: github + # repo: idmsvc/idmsvc-frontend + host: local + repo: ./ + path: deploy/frontend.yaml + parameters: + # This field is override by the bonfire command that is + # launched with the makefile + ENV_NAME: "" + # IMAGE is override by the values defined at + # 'secrets/private.mk and + # scripts/mk/variables.mk + IMAGE: "quay.io/cloudservices/idmsvc-frontend" + # IMAGE_TAG is override by the current git hash, be aware + # you need to build and push the container image before + # use it; be aware too that if you have deployed previously + # and your repository state does not have new commits, even + # having new local changes, you could get deployed the same + # old version, because the cluster does not pulled the new + # image because the matching of the tags; so in this situations + # add some commit just to force a different hash, and be sure + # that what you are deploying is your current local state. + IMAGE_TAG: "pr-24-5714d07" From 401cb531350eff90c390098fdb0ead5ddbaa6590 Mon Sep 17 00:00:00 2001 From: Alejandro Visiedo Date: Fri, 28 Jul 2023 18:22:24 +0200 Subject: [PATCH 05/11] build(HMS-3165): script to deploy in ephe Add the set of makefiles to provide the rules to custom deployment in the ephemeral environment. Align this repo with the backend repository. Signed-off-by: Alejandro Visiedo --- scripts/mk/container.mk | 62 +++++++++++ scripts/mk/ephemeral.mk | 187 ++++++++++++++++++++++++++++++++++ scripts/mk/includes.mk | 8 ++ scripts/mk/meta-container.mk | 1 + scripts/mk/meta-ephemeral.mk | 1 + scripts/mk/private.example.mk | 35 +++++++ scripts/mk/projectdir.mk | 1 + scripts/mk/variables.mk | 2 +- scripts/mk/venv.mk | 20 ++++ 9 files changed, 316 insertions(+), 1 deletion(-) create mode 100644 scripts/mk/container.mk create mode 100644 scripts/mk/ephemeral.mk create mode 100644 scripts/mk/meta-container.mk create mode 100644 scripts/mk/meta-ephemeral.mk create mode 100644 scripts/mk/private.example.mk create mode 100644 scripts/mk/venv.mk diff --git a/scripts/mk/container.mk b/scripts/mk/container.mk new file mode 100644 index 0000000..d8bfb07 --- /dev/null +++ b/scripts/mk/container.mk @@ -0,0 +1,62 @@ +## +# General rules for interacting with container +# manager (podman or docker). +## + +QUAY_EXPIRATION ?= 1d + +ifneq (,$(shell command podman -v 2>/dev/null)) +CONTAINER_ENGINE ?= podman +else +ifneq (,$(shell command docker -v 2>/dev/null)) +CONTAINER_ENGINE ?= docker +else +CONTAINER_ENGINE ?= false +endif +endif +export CONTAINER_ENGINE + +CONTAINER_HEALTH_PATH ?= .State.Health.Status + +ifneq (,$shell(selinuxenabled 2>/dev/null)) +CONTAINER_VOL_SUFFIX ?= :Z +else +CONTAINER_VOL_SUFFIX ?= +endif + +CONTAINER_REGISTRY_USER ?= $(USER) +CONTAINER_REGISTRY ?= quay.io +CONTAINER_CONTEXT_DIR ?= . +CONTAINERFILE ?= build/package/Dockerfile +CONTAINER_IMAGE_BASE ?= $(CONTAINER_REGISTRY)/$(CONTAINER_REGISTRY_USER)/$(APP) +CONTAINER_IMAGE_TAG ?= $(shell git rev-parse --short HEAD) +CONTAINER_IMAGE ?= $(CONTAINER_IMAGE_BASE):$(CONTAINER_IMAGE_TAG) +# CONTAINER_BUILD_OPTS +# CONTAINER_ENGINE_OPTS +# CONTAINER_RUN_ARGS + +.PHONY: registry-login +registry-login: + $(CONTAINER_ENGINE) login -u "$(CONTAINER_REGISTRY_USER)" -p "$(CONTAINER_REGISTRY_TOKEN)" $(CONTAINER_REGISTRY) + +.PHONY: container-build +container-build: ## Build image CONTAINER_IMAGE from CONTAINERFILE using the CONTAINER_CONTEXT_DIR + $(CONTAINER_ENGINE) build \ + --label "quay.expires-after=$(QUAY_EXPIRATION)" \ + $(CONTAINER_BUILD_OPTS) \ + -t "$(CONTAINER_IMAGE)" \ + $(CONTAINER_CONTEXT_DIR) \ + -f "$(CONTAINERFILE)" + +.PHONY: container-shell +container-shell: ## Open a terminal to check build steps + $(CONTAINER_ENGINE) run --rm -it -u root:root \ + -v $(PROJECT_DIR):/opt/app-root/src:rw,z \ + --tmpfs /opt/app-root/src/node_modules:rw \ + --tmpfs /opt/app-root/src/.npm:rw \ + registry.redhat.io/rhel9/nodejs-16:1-138.1699550438 \ + /bin/bash + +.PHONY: container-push +container-push: ## Push image to remote registry + $(CONTAINER_ENGINE) push "$(CONTAINER_IMAGE)" diff --git a/scripts/mk/ephemeral.mk b/scripts/mk/ephemeral.mk new file mode 100644 index 0000000..9689d3c --- /dev/null +++ b/scripts/mk/ephemeral.mk @@ -0,0 +1,187 @@ + +# .PHONY: ephemeral-setup +# ephemeral-setup: ## Configure bonfire to run locally +# bonfire config write-default > $(PROJECT_DIR)/config/bonfire-config.yaml + +ifeq (,$(APP)) +$(error APP is empty; did you miss to set APP=my-app at your scripts/mk/variables.mk) +endif + +APP_COMPONENT ?= frontend + +NAMESPACE ?= $(shell oc project -q 2>/dev/null) +# POOL could be: +# default +# minimal +# managed-kafka +# real-managed-kafka +POOL ?= default +export NAMESPACE +export POOL + + +# CLIENTS_RBAC_BASE_URL ?= http://localhost:8801/api/rbac/v1 # For local workstation +# CLIENTS_RBAC_BASE_URL ?= http://rbac-service:8080/api/rbac/v1 +# export CLIENTS_RBAC_BASE_URL + +# Set the default duration for the namespace reservation and extension +EPHEMERAL_DURATION ?= 4h + +ifneq (default,$(POOL)) +EPHEMERAL_OPTS += --no-single-replicas +else +EPHEMERAL_OPTS += --single-replicas +endif + +ifeq (False,$(CLIENTS_RBAC_ENABLED)) +EPHEMERAL_OPTS += --set-parameter "$(APP_COMPONENT)/CLIENTS_RBAC_ENABLED=False" +else +ifneq (,$(CLIENTS_RBAC_BASE_URL)) +EPHEMERAL_OPTS += --set-parameter "$(APP_COMPONENT)/CLIENTS_RBAC_BASE_URL=$(CLIENTS_RBAC_BASE_URL)" +endif +endif + +EPHEMERAL_BONFIRE_PATH ?= config/bonfire.yaml + +# Enable frontend deployment +EPHEMERAL_OPTS += --frontends true + +# https://consoledot.pages.redhat.com/docs/dev/creating-a-new-app/using-ee/bonfire/getting-started-with-ees.html +# Checkout this: https://github.com/RedHatInsights/bonfire/commit/15ac80bfcf9c386eabce33cb219b015a58b756c8 +.PHONY: ephemeral-login +ephemeral-login: .old-ephemeral-login ## Help in login to the ephemeral cluster + @#if [ "$(GH_SESSION_COOKIE)" != "" ]; then python3 $(GO_OUTPUT)/get-token.py; else $(MAKE) .old-ephemeral-login; fi + +.PHONY: .old-ephemeral-login +.old-ephemeral-login: + xdg-open "https://oauth-openshift.apps.c-rh-c-eph.8p0c.p1.openshiftapps.com/oauth/token/request" + @echo "- Login with github" + @echo "- Do click on 'Display Token'" + @echo "- Copy 'Log in with this token' command" + @echo "- Paste the command in your terminal" + @echo "" + @echo "Now you should have access to the cluster, remember to use bonfire to manage namespace lifecycle:" + @echo '# make ephemeral-namespace-create' + @echo "" + @echo "Check the namespaces reserved to you by:" + @echo '# make ephemeral-namespace-list' + @echo "" + @echo "If you need to extend 1hour the time for the namespace reservation" + @echo '# make ephemeral-namespace-extend-1h' + @echo "" + @echo "Finally if you don't need the reserved namespace or just you want to cleanup and restart with a fresh namespace you run:" + @echo '# make ephemeral-namespace-delete-all' + +# Download https://gitlab.cee.redhat.com/klape/get-token/-/blob/main/get-token.py +$(GO_OUTPUT/get-token.py): + curl -Ls -o "$(GO_OUTPUT/get-token.py)" "https://gitlab.cee.redhat.com/klape/get-token/-/raw/main/get-token.py" + +# NOTE Changes to config/bonfire.yaml could impact to this rule +# make ephemeral-deploy EPHEMERAL_NO_BUILD=y CONTAINER_IMAGE_BASE=quay.io/cloudservices/idmsvc-frontend CONTAINER_IMAGE_TAG=7b4abc3 +.PHONY: ephemeral-deploy +ephemeral-deploy: ## Deploy application using 'config/bonfire.yaml' file + [ "$(EPHEMERAL_NO_BUILD)" == "y" ] || $(MAKE) ephemeral-build-deploy + source .venv/bin/activate && \ + bonfire deploy \ + --source appsre \ + --local-config-path "$(EPHEMERAL_BONFIRE_PATH)" \ + --local-config-method override \ + --secrets-dir "$(PROJECT_DIR)/secrets/ephemeral" \ + --import-secrets \ + --namespace "$(NAMESPACE)" \ + --set-parameter "$(APP_COMPONENT)/IMAGE=$(CONTAINER_IMAGE_BASE)" \ + --set-parameter "$(APP_COMPONENT)/IMAGE_TAG=$(CONTAINER_IMAGE_TAG)" \ + $(EPHEMERAL_OPTS) \ + "$(APP)" + +# NOTE Changes to config/bonfire.yaml could impact to this rule +.PHONY: ephemeral-undeploy +ephemeral-undeploy: ## Undeploy application from the current namespace + source .venv/bin/activate && \ + bonfire process \ + --source appsre \ + --local-config-path "$(EPHEMERAL_BONFIRE_PATH)" \ + --local-config-method override \ + --namespace "$(NAMESPACE)" \ + --set-parameter "$(APP_COMPONENT)/IMAGE=$(CONTAINER_IMAGE_BASE)" \ + --set-parameter "$(APP_COMPONENT)/IMAGE_TAG=$(CONTAINER_IMAGE_TAG)" \ + $(EPHEMERAL_OPTS) \ + "$(APP)" 2>/dev/null | json2yaml | oc delete -f - + ! oc get secrets/content-sources-certs &>/dev/null || oc delete secrets/content-sources-certs + +.PHONY: ephemeral-process +ephemeral-process: ## Process application from the current namespace + source .venv/bin/activate && \ + bonfire process \ + --source appsre \ + --local-config-path "$(EPHEMERAL_BONFIRE_PATH)" \ + --namespace "$(NAMESPACE)" \ + --set-parameter "$(APP_COMPONENT)/IMAGE=$(CONTAINER_IMAGE_BASE)" \ + --set-parameter "$(APP_COMPONENT)/IMAGE_TAG=$(CONTAINER_IMAGE_TAG)" \ + $(EPHEMERAL_OPTS) \ + "$(APP)" 2>/dev/null | json2yaml + +# TODO Add command to specify to bonfire the clowdenv template to be used +.PHONY: ephemeral-namespace-create +ephemeral-namespace-create: ## Create a namespace (requires ephemeral environment) + oc project "$(shell source .venv/bin/activate && bonfire namespace reserve --force --pool "$(POOL)" -d "$(EPHEMERAL_DURATION)" 2>/dev/null)" + +.PHONY: ephemeral-namespace-delete +ephemeral-namespace-delete: ## Delete current namespace (requires ephemeral environment) + source .venv/bin/activate && \ + bonfire namespace release --force "$(oc project -q)" + +.PHONY: ephemeral-namespace-delete-all +ephemeral-namespace-delete-all: ## Delete all namespace created by us (requires ephemeral environment) + source .venv/bin/activate && \ + for item in $$( bonfire namespace list --mine --output json | jq -r '. | to_entries | map(select(.key | match("ephemeral-*";"i"))) | map(.key) | .[]' ); do \ + bonfire namespace release --force $$item ; \ + done + +.PHONY: ephemeral-namespace-list +ephemeral-namespace-list: ## List all the namespaces reserved to the current user (requires ephemeral environment) + source .venv/bin/activate && \ + bonfire namespace list --mine + +.PHONY: ephemeral-namespace-extend +ephemeral-namespace-extend: ## Extend for EPHEMERAL_DURATION ("4h" default) the usage of the current ephemeral environment + source .venv/bin/activate && \ + bonfire namespace extend --duration "$(EPHEMERAL_DURATION)" "$(NAMESPACE)" + +.PHONY: ephemeral-namespace-describe +ephemeral-namespace-describe: ## Display information about the current namespace + @source .venv/bin/activate && \ + bonfire namespace describe "$(NAMESPACE)" + + +# CONTAINER_IMAGE_BASE should be a public image +# Tested by 'make ephemeral-build-deploy CONTAINER_IMAGE_BASE=quay.io/avisied0/hmsidm-frontend' +.PHONY: ephemeral-build-deploy +ephemeral-build-deploy: ## Build and deploy image using 'build_deploy.sh' scripts; It requires to pass DOCKER_IMAGE_BASE + @$(MAKE) registry-login \ + CONTAINER_REGISTRY_USER="$(QUAY_USER)" \ + CONTAINER_REGISTRY_TOKEN="$(QUAY_TOKEN)" \ + CONTAINER_REGISTRY="quay.io" + $(MAKE) container-build + $(MAKE) container-push + # IMAGE="$(CONTAINER_IMAGE_BASE)" IMAGE_TAG="$(CONTAINER_IMAGE_TAG)" $(DOCKER) build ./build_deploy.sh 2>&1 | tee build_deploy.log + # IMAGE="$(CONTAINER_IMAGE_BASE)" IMAGE_TAG="$(CONTAINER_IMAGE_TAG)" bash -xv ./.rhcicd/build_deploy.sh + +.PHONY: ephemeral-pr-checks +ephemeral-pr-checks: + IMAGE="$(CONTAINER_IMAGE_BASE)" bash ./pr_checks.sh + +# FIXME This rule will require some updates but it will be something similar +.PHONY: ephemeral-test-backend +ephemeral-test-backend: ## Run IQE tests in the ephemeral environment (require to run ephemeral-deploy before) + source .venv/bin/activate && \ + bonfire deploy-iqe-cji \ + --env clowder_smoke \ + --cji-name "$(APP)-$(APP_COMPONENT)" \ + --namespace "$(NAMESPACE)" \ + "$(APP)" + +# https://kubernetes.io/docs/tasks/administer-cluster/dns-debugging-resolution/ +.PHONY: ephemeral-run-dnsutil +ephemeral-run-dnsutil: ## Run a shell in a new pod to debug dns situations + oc run dnsutil --rm --image=registry.k8s.io/e2e-test-images/jessie-dnsutils:1.3 -it -- bash diff --git a/scripts/mk/includes.mk b/scripts/mk/includes.mk index 046a3a8..50ed2b1 100644 --- a/scripts/mk/includes.mk +++ b/scripts/mk/includes.mk @@ -4,8 +4,16 @@ include scripts/mk/projectdir.mk -include secrets/private.mk +include scripts/mk/venv.mk include scripts/mk/variables.mk include scripts/mk/help.mk include scripts/mk/printvars.mk include scripts/mk/crc-frontend.mk + +include scripts/mk/meta-container.mk +include scripts/mk/container.mk + +include scripts/mk/meta-ephemeral.mk +include scripts/mk/ephemeral.mk + diff --git a/scripts/mk/meta-container.mk b/scripts/mk/meta-container.mk new file mode 100644 index 0000000..bf49ce6 --- /dev/null +++ b/scripts/mk/meta-container.mk @@ -0,0 +1 @@ +##@ Container rules diff --git a/scripts/mk/meta-ephemeral.mk b/scripts/mk/meta-ephemeral.mk new file mode 100644 index 0000000..ff352ba --- /dev/null +++ b/scripts/mk/meta-ephemeral.mk @@ -0,0 +1 @@ +##@ Ephemeral environment rules diff --git a/scripts/mk/private.example.mk b/scripts/mk/private.example.mk new file mode 100644 index 0000000..297f78c --- /dev/null +++ b/scripts/mk/private.example.mk @@ -0,0 +1,35 @@ +## +# This file contains custom variables definition to override +# other values used in the different makefiles +## + +# Your quay user as it is used for setting CONTAINER_IMAGE_BASE +# The link below is to regenerate the token: +# NOTE Don't forget to grant write permission to the robot account +# only for the repository/ies you need +# https://quay.io/repository/YOURUSER/YOURIMAGE?tab=settings +# TODO Update with your robot account +export QUAY_USER := user+robot_account +export QUAY_TOKEN := MYROBOTACCOUNT +export QUAY_LOGIN := $(firstword $(subst +, ,$(QUAY_USER))) +# TODO Update with the image name created in your repository +export QUAY_REPOSITORY := YOURIMAGE + +# Point out to your local repository +CONTAINER_IMAGE_BASE ?= quay.io/$(QUAY_LOGIN)/$(QUAY_REPOSITORY) + +# https://access.redhat.com/RegistryAuthentication +# https://access.redhat.com/RegistryAuthentication#creating-registry-service-accounts-6 +# To retrieve your token or regenerate it +# https://access.redhat.com/terms-based-registry/#/token/YOUR_USERNAME +# TODO Update your account details +export RH_REGISTRY_USER := +export RH_REGISTRY_TOKEN := + +# Ephemeral pool +POOL ?= default +# POOL ?= real-managed-kafka + +# NOTE This could be necessary when too many deployments are made. +# export GITLAB_TOKEN := + diff --git a/scripts/mk/projectdir.mk b/scripts/mk/projectdir.mk index 94e863d..9ba4450 100644 --- a/scripts/mk/projectdir.mk +++ b/scripts/mk/projectdir.mk @@ -3,3 +3,4 @@ # reading the absolute path for the main Makefile. ## PROJECT_DIR := $(shell dirname $(abspath $(firstword $(MAKEFILE_LIST)))) +export PROJECT_DIR diff --git a/scripts/mk/variables.mk b/scripts/mk/variables.mk index 90c49a6..b1c7bab 100644 --- a/scripts/mk/variables.mk +++ b/scripts/mk/variables.mk @@ -2,6 +2,6 @@ # General variables leveraged on the repository. ## -APP := hmsidm +APP := idmsvc diff --git a/scripts/mk/venv.mk b/scripts/mk/venv.mk new file mode 100644 index 0000000..444af11 --- /dev/null +++ b/scripts/mk/venv.mk @@ -0,0 +1,20 @@ +# install Python tools in a virtual environment + +PYTHON_VENV := .venv +BONFIRE := $(PYTHON_VENV)/bin/bonfire +PRE_COMMIT := $(PYTHON_VENV)/bin/pre-commit +JSON2YAML := $(PYTHON_VENV)/bin/json2yaml + +$(PYTHON_VENV): + python3 -m venv $(PYTHON_VENV) + $(PYTHON_VENV)/bin/pip install -U pip setuptools + +$(BONFIRE) $(PRE_COMMIT) $(JSON2YAML): $(PYTHON_VENV) + $(PYTHON_VENV)/bin/pip3 install -r requirements-dev.txt + touch $(BONFIRE) $(PRE_COMMIT) + +.PHONY: install-python-tools +install-python-tools: + $(MAKE) $(BONFIRE) + $(MAKE) $(PRE_COMMIT) + $(MAKE) $(JSON2YAML) From d43db62913cb400905185055d0cd3e225a60bb98 Mon Sep 17 00:00:00 2001 From: Alejandro Visiedo Date: Fri, 28 Jul 2023 18:24:35 +0200 Subject: [PATCH 06/11] build(HMS-3165): align jest with platform docs Update the alias command to run as it is specified into the platform documentation. Signed-off-by: Alejandro Visiedo --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d62098c..b6dfdfb 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "lint:prettier": "prettier --check src", "patch:hosts": "fec patch-etc-hosts", "start": "HOT=true fec dev", - "test": "jest", + "test": "TZ=UTC jest --verbose --no-cache", "postinstall": "rimraf .cache", "prettier": "prettier --write src", "verify": "npm-run-all build lint test", From 9d1d2e14ba7c4dcb003a5eabeae621590b21a74f Mon Sep 17 00:00:00 2001 From: Alejandro Visiedo Date: Fri, 28 Jul 2023 18:26:20 +0200 Subject: [PATCH 07/11] build(HMS-3165): frontend container Add a poc to try to build the frontend container locally, and check if it could be deployed in ephemeral environment. Signed-off-by: Alejandro Visiedo --- build/package/Caddyfile | 64 +++++++++++++++++++++++++++++++++++ build/package/Dockerfile | 39 +++++++++++++++++++++ build/package/gen.app.info.sh | 47 +++++++++++++++++++++++++ 3 files changed, 150 insertions(+) create mode 100644 build/package/Caddyfile create mode 100644 build/package/Dockerfile create mode 100755 build/package/gen.app.info.sh diff --git a/build/package/Caddyfile b/build/package/Caddyfile new file mode 100644 index 0000000..a223cb4 --- /dev/null +++ b/build/package/Caddyfile @@ -0,0 +1,64 @@ +{ + {$CADDY_TLS_MODE} + auto_https disable_redirects + servers { + metrics + } +} + +:9000 { + metrics /metrics +} + +:8000 { + {$CADDY_TLS_CERT} + log + + # Handle main app route + @app_match { + path /apps/idmsvc* + } + handle @app_match { + uri strip_prefix /apps/idmsvc + file_server * { + root /opt/app-root/src/dist/stable + browse + } + } + + # Handle beta app route + @beta_match { + path /beta/apps/idmsvc* + } + handle @beta_match { + uri strip_prefix /beta/apps/idmsvc + file_server * { + root /opt/app-root/src/dist/preview + browse + } + } + + # Handle preview app route + @preview_match { + path /preview/apps/idmsvc* + } + handle @preview_match { + uri strip_prefix /preview/apps/idmsvc* + file_server * { + root /opt/app-root/src/dist/preview + browse + } + } + + handle /beta/ { + redir /beta/apps/chrome/index.html permanent + } + + handle /preview/ { + redir /preview/apps/chrome/index.html permanent + } + + handle / { + redir /apps/chrome/index.html permanent + } +} \ No newline at end of file diff --git a/build/package/Dockerfile b/build/package/Dockerfile new file mode 100644 index 0000000..71f052c --- /dev/null +++ b/build/package/Dockerfile @@ -0,0 +1,39 @@ +FROM registry.redhat.io/rhel9/nodejs-16:1-110 AS builder +# https://github.com/cypress-io/cypress/issues/25236 +# TODO Set env variable indicated into the URL +USER root +# The working dir is at /opt/app-root/src +COPY package*.json . +RUN npm install +COPY src src +COPY *.js . +COPY tsconfig.json tsconfig.json +RUN npm run build + +COPY build/package/gen.app.info.sh gen.app.info.sh +ARG GIT_HASH +ENV GIT_HASH=${GIT_HASH} +ARG APP_NAME +ENV APP_NAME=${APP_NAME} +RUN ./gen.app.info.sh + + +FROM quay.io/cloudservices/caddy-ubi:11145b1 +ENV CADDY_TLS_MODE="http_port 8000" +ENV APP_NAME="idmsvc" +ENV ROUTE_PATH="/apps/${APP_NAME}" +ENV BETA_ROUTE_PATH="/beta/apps/${APP_NAME}" +ENV PREVIEW_ROUTE_PATH="/preview/apps/${APP_NAME}" + +COPY build/package/Caddyfile /opt/app-root/src/Caddyfile +COPY --from=builder /opt/app-root/src/dist /opt/app-root/src/dist/preview +COPY --from=builder /opt/app-root/src/dist /opt/app-root/src/dist/stable +COPY --from=builder /opt/app-root/src/app.info.json /opt/app-root/src/dist/app.info.deps.json +COPY --from=builder /opt/app-root/src/app.info.stable.json /opt/app-root/src/dist/stable/app.info.json +COPY --from=builder /opt/app-root/src/app.info.preview.json /opt/app-root/src/dist/preview/app.info.json + +WORKDIR /opt/app-root/src +CMD ["caddy", "run", "--config", "/opt/app-root/src/Caddyfile"] + +EXPOSE 8000 +EXPOSE 9000 diff --git a/build/package/gen.app.info.sh b/build/package/gen.app.info.sh new file mode 100755 index 0000000..d0156cd --- /dev/null +++ b/build/package/gen.app.info.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# APP_NAME=idmsvc +# GIT_HASH="$( git rev-parse --verify HEAD )" + +# root files + +cat > "app.info.json" << EOF +{ + "app_name": "${APP_NAME}", + "src_hash": "${GIT_HASH}", + "patternfly_dependencies": [], + "rh_cloud_services_dependencies": [] +} +EOF + +cat > "app.info.deps.json" << EOF +undefined +EOF + +# stable + +cat > "app.info.stable.json" << EOF +{ + "app_name": "${APP_NAME}", + "node_version": ">=16.0.0", + "src_hash": "${GIT_HASH}", + "src_tag": "unknown", + "src_branch": "unknown", + "patternfly_dependencies": "[\"@patternfly/react-core@4.276.8\",\"@patternfly/react-table@4.113.0\"]", + "rh_cloud_services_dependencies": "[\"@redhat-cloud-services/frontend-components-notifications@3.2.14\",\"@redhat-cloud-services/frontend-components-utilities@3.5.0\",\"@redhat-cloud-services/frontend-components@3.11.1\"]" +} +EOF + +# preview + +cat > "app.info.preview.json" << EOF +{ + "app_name": "${APP_NAME}", + "node_version": ">=16.0.0", + "src_hash": "${GIT_HASH}", + "src_tag": "unknown", + "src_branch": "unknown", + "patternfly_dependencies": "[\"@patternfly/react-core@4.276.8\",\"@patternfly/react-table@4.113.0\"]", + "rh_cloud_services_dependencies": "[\"@redhat-cloud-services/frontend-components-notifications@3.2.14\",\"@redhat-cloud-services/frontend-components-utilities@3.5.0\",\"@redhat-cloud-services/frontend-components@3.11.1\"]" +} +EOF From 8a4c3f43410d19a6f508b93be0e79727c1a9ef46 Mon Sep 17 00:00:00 2001 From: Alejandro Visiedo Date: Thu, 23 Nov 2023 20:02:02 +0100 Subject: [PATCH 08/11] build(HMS-3165): update Dockerfile Fix wrong copy of files. Signed-off-by: Alejandro Visiedo --- build/package/Caddyfile | 4 ++-- build/package/Dockerfile | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/build/package/Caddyfile b/build/package/Caddyfile index a223cb4..775b105 100644 --- a/build/package/Caddyfile +++ b/build/package/Caddyfile @@ -9,7 +9,7 @@ :9000 { metrics /metrics } - + :8000 { {$CADDY_TLS_CERT} log @@ -61,4 +61,4 @@ handle / { redir /apps/chrome/index.html permanent } -} \ No newline at end of file +} diff --git a/build/package/Dockerfile b/build/package/Dockerfile index 71f052c..0204d88 100644 --- a/build/package/Dockerfile +++ b/build/package/Dockerfile @@ -28,12 +28,13 @@ ENV PREVIEW_ROUTE_PATH="/preview/apps/${APP_NAME}" COPY build/package/Caddyfile /opt/app-root/src/Caddyfile COPY --from=builder /opt/app-root/src/dist /opt/app-root/src/dist/preview COPY --from=builder /opt/app-root/src/dist /opt/app-root/src/dist/stable -COPY --from=builder /opt/app-root/src/app.info.json /opt/app-root/src/dist/app.info.deps.json +COPY --from=builder /opt/app-root/src/app.info.json /opt/app-root/src/dist/app.info.json +COPY --from=builder /opt/app-root/src/app.info.deps.json /opt/app-root/src/dist/app.info.deps.json COPY --from=builder /opt/app-root/src/app.info.stable.json /opt/app-root/src/dist/stable/app.info.json COPY --from=builder /opt/app-root/src/app.info.preview.json /opt/app-root/src/dist/preview/app.info.json WORKDIR /opt/app-root/src CMD ["caddy", "run", "--config", "/opt/app-root/src/Caddyfile"] -EXPOSE 8000 +EXPOSE 8000 EXPOSE 9000 From 8a33c477b273e9dea6bd5fd59a0725f02aa53424 Mon Sep 17 00:00:00 2001 From: Alejandro Visiedo Date: Thu, 23 Nov 2023 20:15:18 +0100 Subject: [PATCH 09/11] build(HMS-3165): update makefile Add bonfire-deploy and update rule to build the container. Signed-off-by: Alejandro Visiedo --- scripts/mk/ephemeral.mk | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/mk/ephemeral.mk b/scripts/mk/ephemeral.mk index 9689d3c..953934d 100644 --- a/scripts/mk/ephemeral.mk +++ b/scripts/mk/ephemeral.mk @@ -162,10 +162,8 @@ ephemeral-build-deploy: ## Build and deploy image using 'build_deploy.sh' scrip CONTAINER_REGISTRY_USER="$(QUAY_USER)" \ CONTAINER_REGISTRY_TOKEN="$(QUAY_TOKEN)" \ CONTAINER_REGISTRY="quay.io" - $(MAKE) container-build + $(MAKE) container-build CONTAINER_BUILD_OPTS="--build-arg APP_NAME=$(APP) --build-arg GIT_HASH=$(shell git rev-parse --verify HEAD)" $(MAKE) container-push - # IMAGE="$(CONTAINER_IMAGE_BASE)" IMAGE_TAG="$(CONTAINER_IMAGE_TAG)" $(DOCKER) build ./build_deploy.sh 2>&1 | tee build_deploy.log - # IMAGE="$(CONTAINER_IMAGE_BASE)" IMAGE_TAG="$(CONTAINER_IMAGE_TAG)" bash -xv ./.rhcicd/build_deploy.sh .PHONY: ephemeral-pr-checks ephemeral-pr-checks: @@ -185,3 +183,8 @@ ephemeral-test-backend: ## Run IQE tests in the ephemeral environment (require .PHONY: ephemeral-run-dnsutil ephemeral-run-dnsutil: ## Run a shell in a new pod to debug dns situations oc run dnsutil --rm --image=registry.k8s.io/e2e-test-images/jessie-dnsutils:1.3 -it -- bash + +.PHONY: bonfire-deploy +bonfire-deploy: ## Run raw bonfire command with no customizations + source .venv/bin/activate && \ + bonfire deploy --frontends true "$(APP)" From a32c9be6afcc047aed639082e2ff5e5caa271e3b Mon Sep 17 00:00:00 2001 From: Alejandro Visiedo Date: Mon, 27 Nov 2023 07:06:43 +0100 Subject: [PATCH 10/11] build(HMS-3165): add .dockerignore Add .dockerignore with the paths to avoid to copy to the image when the statement 'COPY . .' is executed. Signed-off-by: Alejandro Visiedo --- .dockerignore | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..9010a5c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,14 @@ +# Ignore /secrets directory content +secrets + +# Ignore common directories +.git +dist +bin +node_modules +.venv +config/bonfire.yaml +.npm +.cache +.devcontainer + From 9d87a7cdc336e971e6a5af36b6cae863e14b3bba Mon Sep 17 00:00:00 2001 From: Alejandro Visiedo Date: Fri, 15 Dec 2023 20:13:04 +0100 Subject: [PATCH 11/11] fix(HMS-3165): container metadata files It was observed some differences between the container generated from the CI/CD platform system and the container generated locally from the repository. This change add several changes to try to be aligned with that changes and generate the same or similar content to what is generated by the platform. Signed-off-by: Alejandro Visiedo --- build/package/Dockerfile | 33 ++++++++++++++++++++++++--------- build/package/gen.app.info.sh | 11 ++++++----- scripts/mk/container.mk | 2 +- scripts/mk/ephemeral.mk | 33 +++++++++++++++------------------ scripts/mk/variables.mk | 2 +- 5 files changed, 47 insertions(+), 34 deletions(-) diff --git a/build/package/Dockerfile b/build/package/Dockerfile index 0204d88..31a0f42 100644 --- a/build/package/Dockerfile +++ b/build/package/Dockerfile @@ -11,30 +11,45 @@ COPY tsconfig.json tsconfig.json RUN npm run build COPY build/package/gen.app.info.sh gen.app.info.sh -ARG GIT_HASH -ENV GIT_HASH=${GIT_HASH} +RUN mv dist target \ + && mkdir dist \ + && cp -Rvf target dist/preview \ + && cp -Rvf target dist/stable \ + && true +ARG SRC_HASH +ENV SRC_HASH=${SRC_HASH} ARG APP_NAME ENV APP_NAME=${APP_NAME} RUN ./gen.app.info.sh FROM quay.io/cloudservices/caddy-ubi:11145b1 -ENV CADDY_TLS_MODE="http_port 8000" -ENV APP_NAME="idmsvc" -ENV ROUTE_PATH="/apps/${APP_NAME}" -ENV BETA_ROUTE_PATH="/beta/apps/${APP_NAME}" -ENV PREVIEW_ROUTE_PATH="/preview/apps/${APP_NAME}" + COPY build/package/Caddyfile /opt/app-root/src/Caddyfile -COPY --from=builder /opt/app-root/src/dist /opt/app-root/src/dist/preview -COPY --from=builder /opt/app-root/src/dist /opt/app-root/src/dist/stable +COPY --from=builder /opt/app-root/src/dist /opt/app-root/src/dist/ + COPY --from=builder /opt/app-root/src/app.info.json /opt/app-root/src/dist/app.info.json COPY --from=builder /opt/app-root/src/app.info.deps.json /opt/app-root/src/dist/app.info.deps.json COPY --from=builder /opt/app-root/src/app.info.stable.json /opt/app-root/src/dist/stable/app.info.json COPY --from=builder /opt/app-root/src/app.info.preview.json /opt/app-root/src/dist/preview/app.info.json +COPY ./package.json /opt/app-root/src + + WORKDIR /opt/app-root/src + +RUN sed -i -e 's/\/beta\/apps/\/apps/g' /opt/app-root/src/dist/stable/fed-mods.json + +ENV APP_NAME="idmsvc" +ENV ROUTE_PATH="/apps/${APP_NAME}" +ENV BETA_ROUTE_PATH="/beta/apps/${APP_NAME}" +ENV PREVIEW_ROUTE_PATH="/preview/apps/${APP_NAME}" + CMD ["caddy", "run", "--config", "/opt/app-root/src/Caddyfile"] EXPOSE 8000 EXPOSE 9000 + +ENV CADDY_TLS_MODE="http_port 8000" + diff --git a/build/package/gen.app.info.sh b/build/package/gen.app.info.sh index d0156cd..b0b4c25 100755 --- a/build/package/gen.app.info.sh +++ b/build/package/gen.app.info.sh @@ -1,14 +1,14 @@ #!/bin/bash -# APP_NAME=idmsvc -# GIT_HASH="$( git rev-parse --verify HEAD )" +APP_NAME=idmsvc +# SRC_HASH is passed as a build argument # root files cat > "app.info.json" << EOF { "app_name": "${APP_NAME}", - "src_hash": "${GIT_HASH}", + "src_hash": "${SRC_HASH}", "patternfly_dependencies": [], "rh_cloud_services_dependencies": [] } @@ -24,7 +24,7 @@ cat > "app.info.stable.json" << EOF { "app_name": "${APP_NAME}", "node_version": ">=16.0.0", - "src_hash": "${GIT_HASH}", + "src_hash": "${SRC_HASH}", "src_tag": "unknown", "src_branch": "unknown", "patternfly_dependencies": "[\"@patternfly/react-core@4.276.8\",\"@patternfly/react-table@4.113.0\"]", @@ -38,10 +38,11 @@ cat > "app.info.preview.json" << EOF { "app_name": "${APP_NAME}", "node_version": ">=16.0.0", - "src_hash": "${GIT_HASH}", + "src_hash": "${SRC_HASH}", "src_tag": "unknown", "src_branch": "unknown", "patternfly_dependencies": "[\"@patternfly/react-core@4.276.8\",\"@patternfly/react-table@4.113.0\"]", "rh_cloud_services_dependencies": "[\"@redhat-cloud-services/frontend-components-notifications@3.2.14\",\"@redhat-cloud-services/frontend-components-utilities@3.5.0\",\"@redhat-cloud-services/frontend-components@3.11.1\"]" } EOF + diff --git a/scripts/mk/container.mk b/scripts/mk/container.mk index d8bfb07..9a27a11 100644 --- a/scripts/mk/container.mk +++ b/scripts/mk/container.mk @@ -28,7 +28,7 @@ CONTAINER_REGISTRY_USER ?= $(USER) CONTAINER_REGISTRY ?= quay.io CONTAINER_CONTEXT_DIR ?= . CONTAINERFILE ?= build/package/Dockerfile -CONTAINER_IMAGE_BASE ?= $(CONTAINER_REGISTRY)/$(CONTAINER_REGISTRY_USER)/$(APP) +CONTAINER_IMAGE_BASE ?= $(CONTAINER_REGISTRY)/$(CONTAINER_REGISTRY_USER)/$(APP_NAME) CONTAINER_IMAGE_TAG ?= $(shell git rev-parse --short HEAD) CONTAINER_IMAGE ?= $(CONTAINER_IMAGE_BASE):$(CONTAINER_IMAGE_TAG) # CONTAINER_BUILD_OPTS diff --git a/scripts/mk/ephemeral.mk b/scripts/mk/ephemeral.mk index 953934d..84478d1 100644 --- a/scripts/mk/ephemeral.mk +++ b/scripts/mk/ephemeral.mk @@ -1,14 +1,18 @@ # .PHONY: ephemeral-setup # ephemeral-setup: ## Configure bonfire to run locally -# bonfire config write-default > $(PROJECT_DIR)/config/bonfire-config.yaml +# source .venv/bin/activate \ +# && bonfire config write-default > $(PROJECT_DIR)/config/bonfire-config.yaml -ifeq (,$(APP)) -$(error APP is empty; did you miss to set APP=my-app at your scripts/mk/variables.mk) +ifeq (,$(APP_NAME)) +$(error APP_NAME is empty; did you miss to set APP_NAME=my-app at your scripts/mk/variables.mk) endif APP_COMPONENT ?= frontend +# Set the default duration for the namespace reservation and extension +EPHEMERAL_DURATION ?= 4h + NAMESPACE ?= $(shell oc project -q 2>/dev/null) # POOL could be: # default @@ -20,13 +24,6 @@ export NAMESPACE export POOL -# CLIENTS_RBAC_BASE_URL ?= http://localhost:8801/api/rbac/v1 # For local workstation -# CLIENTS_RBAC_BASE_URL ?= http://rbac-service:8080/api/rbac/v1 -# export CLIENTS_RBAC_BASE_URL - -# Set the default duration for the namespace reservation and extension -EPHEMERAL_DURATION ?= 4h - ifneq (default,$(POOL)) EPHEMERAL_OPTS += --no-single-replicas else @@ -67,7 +64,7 @@ ephemeral-login: .old-ephemeral-login ## Help in login to the ephemeral cluster @echo '# make ephemeral-namespace-list' @echo "" @echo "If you need to extend 1hour the time for the namespace reservation" - @echo '# make ephemeral-namespace-extend-1h' + @echo '# make ephemeral-namespace-extend EPHEMERAL_DURATION=1h' @echo "" @echo "Finally if you don't need the reserved namespace or just you want to cleanup and restart with a fresh namespace you run:" @echo '# make ephemeral-namespace-delete-all' @@ -92,7 +89,7 @@ ephemeral-deploy: ## Deploy application using 'config/bonfire.yaml' file --set-parameter "$(APP_COMPONENT)/IMAGE=$(CONTAINER_IMAGE_BASE)" \ --set-parameter "$(APP_COMPONENT)/IMAGE_TAG=$(CONTAINER_IMAGE_TAG)" \ $(EPHEMERAL_OPTS) \ - "$(APP)" + "$(APP_NAME)" # NOTE Changes to config/bonfire.yaml could impact to this rule .PHONY: ephemeral-undeploy @@ -106,7 +103,7 @@ ephemeral-undeploy: ## Undeploy application from the current namespace --set-parameter "$(APP_COMPONENT)/IMAGE=$(CONTAINER_IMAGE_BASE)" \ --set-parameter "$(APP_COMPONENT)/IMAGE_TAG=$(CONTAINER_IMAGE_TAG)" \ $(EPHEMERAL_OPTS) \ - "$(APP)" 2>/dev/null | json2yaml | oc delete -f - + "$(APP_NAME)" 2>/dev/null | json2yaml | oc delete -f - ! oc get secrets/content-sources-certs &>/dev/null || oc delete secrets/content-sources-certs .PHONY: ephemeral-process @@ -119,7 +116,7 @@ ephemeral-process: ## Process application from the current namespace --set-parameter "$(APP_COMPONENT)/IMAGE=$(CONTAINER_IMAGE_BASE)" \ --set-parameter "$(APP_COMPONENT)/IMAGE_TAG=$(CONTAINER_IMAGE_TAG)" \ $(EPHEMERAL_OPTS) \ - "$(APP)" 2>/dev/null | json2yaml + "$(APP_NAME)" 2>/dev/null | json2yaml # TODO Add command to specify to bonfire the clowdenv template to be used .PHONY: ephemeral-namespace-create @@ -162,7 +159,7 @@ ephemeral-build-deploy: ## Build and deploy image using 'build_deploy.sh' scrip CONTAINER_REGISTRY_USER="$(QUAY_USER)" \ CONTAINER_REGISTRY_TOKEN="$(QUAY_TOKEN)" \ CONTAINER_REGISTRY="quay.io" - $(MAKE) container-build CONTAINER_BUILD_OPTS="--build-arg APP_NAME=$(APP) --build-arg GIT_HASH=$(shell git rev-parse --verify HEAD)" + $(MAKE) container-build CONTAINER_BUILD_OPTS="--build-arg APP_NAME=$(APP_NAME) --build-arg GIT_HASH=$(shell git rev-parse --verify HEAD) --build-arg SRC_HASH=$(shell git rev-parse HEAD) --build-arg APP_NAME=$(APP_NAME)" $(MAKE) container-push .PHONY: ephemeral-pr-checks @@ -175,9 +172,9 @@ ephemeral-test-backend: ## Run IQE tests in the ephemeral environment (require source .venv/bin/activate && \ bonfire deploy-iqe-cji \ --env clowder_smoke \ - --cji-name "$(APP)-$(APP_COMPONENT)" \ + --cji-name "$(APP_NAME)-$(APP_COMPONENT)" \ --namespace "$(NAMESPACE)" \ - "$(APP)" + "$(APP_NAME)" # https://kubernetes.io/docs/tasks/administer-cluster/dns-debugging-resolution/ .PHONY: ephemeral-run-dnsutil @@ -187,4 +184,4 @@ ephemeral-run-dnsutil: ## Run a shell in a new pod to debug dns situations .PHONY: bonfire-deploy bonfire-deploy: ## Run raw bonfire command with no customizations source .venv/bin/activate && \ - bonfire deploy --frontends true "$(APP)" + bonfire deploy --frontends true "$(APP_NAME)" diff --git a/scripts/mk/variables.mk b/scripts/mk/variables.mk index b1c7bab..1f07a73 100644 --- a/scripts/mk/variables.mk +++ b/scripts/mk/variables.mk @@ -2,6 +2,6 @@ # General variables leveraged on the repository. ## -APP := idmsvc +APP_NAME := idmsvc