From d2cc5a02df75e25b742bfaec6aadbfcaaad51279 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Sch=C3=BCller?= Date: Tue, 26 Mar 2024 16:42:07 +0100 Subject: [PATCH] Makefile: implement "fully source containers" HMS-3883 also changes compose file to be more robust on startup use docker as default for now podman does not seem to run stable. will need to investigate more in docs: document makefile overview also fix execution on MAC --- .gitignore | 3 + Makefile | 202 ++++++++++++++++ README.md | 5 +- docs/src_compile_onprem.dot | 101 ++++++++ docs/src_compile_onprem.svg | 166 +++++++++++++ docs/src_compile_service.dot | 132 +++++++++++ docs/src_compile_service.svg | 221 ++++++++++++++++++ on-prem/bin/run.py | 1 - on-prem/bin/setup-host.py | 22 +- on-prem/src/ogsc/run/composer/entrypoint.py | 69 ------ service/README.md | 78 ++++--- service/config/Dockerfile-config | 24 +- service/config/Dockerfile-config-onprem | 21 ++ service/config/backend/quotas.json | 2 +- service/config/cli/data/blueprint.toml | 9 + .../composer/osbuild-composer-onprem.toml | 7 + service/docker-compose-onprem.yml | 155 ++++++++++++ service/docker-compose.yml | 167 +++++++++---- service/tools/gen-certs.sh | 9 + tools/git_stack.sh | 62 +++++ 20 files changed, 1287 insertions(+), 169 deletions(-) create mode 100644 Makefile create mode 100644 docs/src_compile_onprem.dot create mode 100644 docs/src_compile_onprem.svg create mode 100644 docs/src_compile_service.dot create mode 100644 docs/src_compile_service.svg create mode 100644 service/config/Dockerfile-config-onprem create mode 100644 service/config/cli/data/blueprint.toml create mode 100644 service/config/composer/osbuild-composer-onprem.toml create mode 100644 service/docker-compose-onprem.yml create mode 100755 tools/git_stack.sh diff --git a/.gitignore b/.gitignore index 68c759d..f53f66f 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,6 @@ on-prem/build/config/*.pem on-prem/build/config/*.toml on-prem/build/config/ca on-prem/build/config/repositories +service_images_built.info +onprem_images_built.info + diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..87d6363 --- /dev/null +++ b/Makefile @@ -0,0 +1,202 @@ +.PHONY: help +help: + @echo "make [TARGETS...]" + @echo + @echo "This is the makefile of osbuild-getting-started. The following" + @echo "targets are available:" + @echo + @echo " service_containers: Build all needed containers from source to be able to run the service" + @echo " run_service: Run all containers needed for the 'service'" + @echo " This is running in foreground. Use CTRL-C to stop the containers." + @echo " run_service_no_frontend: Run all containers except for the frontend" + @echo " stop_service: Usually the containers get stopped by CTRL-C. So 'stop-service'" + @echo " is only needed if something strange happened and not all are stopped." + @echo " prune_service: Remove the containers, including the test-data!" + @echo " If you want empty databases" + @echo " clean: Clean all subprojects to assure a rebuild" + +# source where the other repos are locally +# has to end with a trailing slash +SRC_DEPS_EXTERNAL_CHECKOUT_DIR ?= ../ + +# either "docker" or "sudo podman" +# podman needs to build as root as it also needs to run as root afterwards +CONTAINER_EXECUTABLE ?= docker +CONTAINER_COMPOSE_EXECUTABLE := $(CONTAINER_EXECUTABLE) compose + +MAKE_SUB_CALL := make CONTAINER_EXECUTABLE="$(CONTAINER_EXECUTABLE)" + +# osbuild is indirectly used by osbuild-composer +# but we'll mention it here too for better error messages and usability +COMMON_SRC_DEPS_NAMES := osbuild osbuild-composer pulp-client community-gateway +COMMON_SRC_DEPS_ORIGIN := $(addprefix $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR),$(COMMON_SRC_DEPS_NAMES)) + +ONPREM_SRC_DEPS_NAMES := weldr-client +ONPREM_SRC_DEPS_ORIGIN := $(addprefix $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR),$(ONPREM_SRC_DEPS_NAMES)) + +SERVICE_SRC_DEPS_NAMES := image-builder image-builder-frontend +SERVICE_SRC_DEPS_ORIGIN := $(addprefix $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR),$(SERVICE_SRC_DEPS_NAMES)) + +# should be set if we are already sudo - otherwise we set to "whoami" +SUDO_USER ?= $(shell whoami) + +$(COMMON_SRC_DEPS_ORIGIN) $(SERVICE_SRC_DEPS_ORIGIN) $(ONPREM_SRC_DEPS_ORIGIN): + @for DIR in $@; do if ! [ -d $$DIR ]; then echo "Please checkout $$DIR so it is available at $$DIR"; exit 1; fi; done + +COMPARE_TO_BRANCH ?= origin/main + +SCRATCH_DIR := $(HOME)/.cache/osbuild-getting-started/scratch +export SCRATCH_DIR +COMMON_DIR := podman/image-builder-config +CLI_DIRS := podman/weldr podman/cloudapi podman/dnf-json +DATA_DIR := data/s3/service +ALL_SCRATCH_DIRS := $(addprefix $(SCRATCH_DIR)/,$(COMMON_DIR) $(CLI_DIRS) $(DATA_DIR)) + +# internal rule for sub-calls +# NOTE: This chowns all directories back - as we expect to run partly as root +# also we "git fetch origin" to get the current state! +.PHONY: common_sub_makes +common_sub_makes: + @echo "We need to build everything as root, as the target also needs to run as root." + @echo "At least for podman the password as already needed now" + + # creating container image from osbuild as a basis for worker + $(MAKE_SUB_CALL) -C $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)osbuild-composer container_worker.dev container_composer.dev + +.PHONY: service_sub_make_backend +service_sub_make_backend: + $(MAKE_SUB_CALL) -C $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)image-builder container.dev + +.PHONY: service_sub_make_frontend +service_sub_make_frontend: + $(MAKE_SUB_CALL) -C $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)image-builder-frontend container.dev + +.PHONY: service_sub_make_cleanup +service_sub_make_cleanup: + @for DIR in $(COMMON_SRC_DEPS_ORIGIN) $(SERVICE_SRC_DEPS_ORIGIN); do echo "Giving directory permissions in '$$DIR' back to '$(SUDO_USER)'"; chown -R $(SUDO_USER): $$DIR || sudo chown -R $(SUDO_USER): $$DIR; done + @echo "Your current versions are (comparing to origin/main):" + bash -c './tools/git_stack.sh' + +.PHONY: service_sub_makes_no_frontend +service_sub_makes_no_frontend: service_sub_make_backend service_sub_make_cleanup + +.PHONY: service_sub_makes +service_sub_makes: service_sub_make_backend service_sub_make_frontend service_sub_make_cleanup + +.PHONY: onprem_sub_makes +onprem_sub_makes: + # building the cli + $(MAKE_SUB_CALL) -C $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)weldr-client container.dev + @for DIR in $(COMMON_SRC_DEPS_ORIGIN) $(ONPREM_SRC_DEPS_ORIGIN); do echo "Giving directory permissions in '$$DIR' back to '$(SUDO_USER)'"; chown -R $(SUDO_USER): $$DIR || sudo chown -R $(SUDO_USER): $$DIR; done + @echo "Your current versions are (comparing to origin/main):" + bash -c './tools/git_stack.sh' + +.PHONY: service_containers +service_containers: $(COMMON_SRC_DEPS_ORIGIN) $(SERVICE_SRC_DEPS_ORIGIN) common_sub_makes service_sub_makes service_images_built.info + +.PHONY: service_containers_no_frontend +service_containers_no_frontend: $(COMMON_SRC_DEPS_ORIGIN) $(SERVICE_SRC_DEPS_ORIGIN) common_sub_makes service_sub_makes_no_frontend service_images_built.info + +onprem_containers: $(COMMON_SRC_DEPS_ORIGIN) $(ONPREM_SRC_DEPS_ORIGIN) common_sub_makes onprem_sub_makes onprem_images_built.info + +service_images_built.info: service/config/Dockerfile-config service/config/composer/osbuild-composer.toml $(ALL_SCRATCH_DIRS) + # building remaining containers (config, fauxauth) + $(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose.yml build --build-arg CONFIG_BUILD_DATE=$(shell date -r $(SCRATCH_DIR)/$(COMMON_DIR) +%Y%m%d_%H%M%S) + echo "Images last built on" > $@ + date >> $@ + +onprem_images_built.info: service/config/Dockerfile-config-onprem service/config/composer/osbuild-composer-onprem.toml $(ALL_SCRATCH_DIRS) + # building remaining containers (config) + $(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose-onprem.yml build --build-arg CONFIG_BUILD_DATE=$(shell date -r $(SCRATCH_DIR)/$(COMMON_DIR) +%Y%m%d_%H%M%S) + echo "Images last built on" > $@ + date >> $@ + +$(ALL_SCRATCH_DIRS): + @echo "Creating directory: $@" + mkdir -p $@ || ( echo "Trying as root" ; sudo mkdir -p $@ ) + +.PHONY: wipe_config +wipe_config: + sudo rm -rf $(SCRATCH_DIR)/$(COMMON_DIR) + rm -f $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)image-builder-frontend/node_modules/.cache/webpack-dev-server/server.pem + +.PHONY: clean +clean: $(COMMON_SRC_DEPS_ORIGIN) $(SERVICE_SRC_DEPS_ORIGIN) $(ONPREM_SRC_DEPS_ORIGIN) prune_service prune_onprem wipe_config + rm -f service_images_built.info + rm -f onprem_images_built.info + rm -rf $(SCRATCH_DIR) || (echo "Trying as root" ;sudo rm -rf $(SCRATCH_DIR)) + for DIR in $*; do $(MAKE_SUB_CALL) -C $$DIR clean; done + $(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose.yml down --volumes + $(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose.yml rm --volumes + $(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose-onprem.yml down --volumes + $(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose-onprem.yml rm --volumes + +# for compatibility of relative volume mount paths +# between docker and podman we have to change to the directory +.PHONY: run_service +.ONESHELL: +run_service: $(addprefix $(SCRATCH_DIR)/,$(COMMON_DIRS)) service_containers + export PORTS="$(shell docker compose -f service/docker-compose.yml config|grep -Po '(?<=published: ")[0-9]+')" + echo "-- Checking if any of our ports are used: $$PORTS" + sudo netstat -lntp|grep -E "$$(echo "$$PORTS"|tr ' ' ':')" + echo "-- Check done" + cd service + $(CONTAINER_COMPOSE_EXECUTABLE) up + +# if you want to run the frontend yourself - outside the docker environment +.PHONY: run_service_no_frontend +run_service_no_frontend: service_containers_no_frontend + $(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose.yml up backend fauxauth worker composer minio postgres_backend postgres_composer + +# only for strange crashes - should shut down properly in normal operation +.PHONY: stop_service +.ONESHELL: +stop_service: + cd service + $(CONTAINER_COMPOSE_EXECUTABLE) stop + +.PHONY: prune_service +.ONESHELL: +prune_service: + cd service + $(CONTAINER_COMPOSE_EXECUTABLE) down + +.PHONY: prune_onprem +.ONESHELL: +prune_onprem: + cd service + $(CONTAINER_COMPOSE_EXECUTABLE) -f docker-compose-onprem.yml down + +# for compatibility of relative volume mount paths +# between docker and podman we have to change to the directory +.PHONY: run_onprem +.ONESHELL: +run_onprem: $(addprefix $(SCRATCH_DIR)/,$(COMMON_DIRS) $(CLI_DIRS)) onprem_containers + cd service + echo "Remove dangling sockets as root" + sudo rm -f $(addsuffix /api.sock*, $(addprefix $(SCRATCH_DIR)/, $(CLI_DIRS))) + $(CONTAINER_COMPOSE_EXECUTABLE) -f docker-compose-onprem.yml up -d + @echo "------ Welcome to osbuild! You can now use 'composer-cli' to build images" + @echo " … and 'exit' afterwards" + @echo " You might also be interested to open up a second terminal and" + @echo " run 'docker compose -f $(shell readlink -f service/docker-compose-onprem.yml) logs --follow' to see possible problems" + + $(CONTAINER_COMPOSE_EXECUTABLE) -f docker-compose-onprem.yml run --entrypoint /bin/bash -ti cli + $(CONTAINER_COMPOSE_EXECUTABLE) -f docker-compose-onprem.yml stop + +%.svg: %.dot + dot -T svg $< > $@ + +.PHONY: docs +docs: docs/src_compile_service.svg docs/src_compile_onprem.svg + +.PHONY: overview +overview: + @echo "Fetching all repos for better overview if rebase will be necessary:" +ifeq (${SUDO_USER},$(shell whoami)) + bash -c './tools/git_stack.sh fetch --all' +else + sudo -u ${SUDO_USER} bash -c './tools/git_stack.sh fetch --all' +endif + @bash -c './tools/git_stack.sh' + diff --git a/README.md b/README.md index d4aeb11..e2ccb3d 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,9 @@ provide more in-depth explainations of these products, their architecture and ho ## Getting started +You can develop and install the individual components on your local computer, or you can run the +whole stack as containers fully from source. + For further instructions on how to run containerized versions of the CLI and service locally: -- the [on-premise README](./on-prem/README.md) offers explainations on how to run the on-premise CLI tool. +- the [on-premise README](./on-prem/README.md) offers explanations on how to run the on-premise CLI tool. - the [service README](./service/README.md) provides instructions on how to get started with a stack of the hosted service. diff --git a/docs/src_compile_onprem.dot b/docs/src_compile_onprem.dot new file mode 100644 index 0000000..e45f843 --- /dev/null +++ b/docs/src_compile_onprem.dot @@ -0,0 +1,101 @@ +digraph src_compile { + newrank=true; + subgraph cluster_osbuild_getting_started { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "osbuild-\ngetting-started"; + osbuild_getting_started_makefile [ + label = "make\nonprem_containers" + width = 2; + ]; + osbuild_getting_started_run [ + label = "make\nrun_onprem" + width = 2; + ]; + } + + subgraph cluster_osbuild { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "osbuild\n "; + osbuild_makefile [ + label = "make\ncontainer"; + width = 1.5; + ]; + } + + subgraph cluster_osbuild_composer { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "osbuild-composer\n "; + osbuild_composer_makefile_composer [ + label = "make\ncontainer_composer" + width = 2; + ]; + osbuild_composer_makefile_worker [ + label = "make\ncontainer_worker" + width = 2; + ]; + + osbuild_composer_go [ label = "go code"; ]; + } + + subgraph cluster_weldr_client { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "weldr-client"; + weldr_client_makefile [ + label = "make\ncontainer" + width = 1.5; + ]; + } + subgraph cluster_pulp_client { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "pulp-client"; + pulp_client_go [ + label = "go code"; + width = 1.5; + ]; + } + subgraph cluster_images { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "images"; + images_go [label = "go code";]; + } + { rank=same; + osbuild_makefile; + osbuild_composer_makefile_worker; + osbuild_composer_makefile_composer; + weldr_client_makefile; + + } + { rank=same; + edge [style=dashed; headport=s; tailport=s]; + pulp_client_go; + images_go; + } + osbuild_composer_go -> pulp_client_go [style=dashed;]; + osbuild_composer_go -> images_go [style=dashed;]; + + osbuild_getting_started_run -> osbuild_getting_started_makefile; + + osbuild_getting_started_makefile -> osbuild_makefile; + osbuild_getting_started_makefile -> osbuild_composer_makefile_worker; + osbuild_getting_started_makefile -> osbuild_composer_makefile_composer; + + + osbuild_getting_started_makefile -> weldr_client_makefile; + osbuild_composer_makefile_worker -> osbuild_composer_go [style=dashed]; + osbuild_composer_makefile_composer -> osbuild_composer_go [style=dashed]; + + osbuild_composer_makefile_worker -> osbuild_makefile [style=dashed; headport=s; tailport=s]; + +} diff --git a/docs/src_compile_onprem.svg b/docs/src_compile_onprem.svg new file mode 100644 index 0000000..7585d67 --- /dev/null +++ b/docs/src_compile_onprem.svg @@ -0,0 +1,166 @@ + + + + + + +src_compile + + +cluster_osbuild_getting_started + +osbuild- +getting-started + + +cluster_osbuild + +osbuild +  + + +cluster_osbuild_composer + +osbuild-composer +  + + +cluster_weldr_client + +weldr-client + + +cluster_pulp_client + +pulp-client + + +cluster_images + +images + + + +osbuild_getting_started_makefile + +make +onprem_containers + + + +osbuild_makefile + +make +container + + + +osbuild_getting_started_makefile->osbuild_makefile + + + + + +osbuild_composer_makefile_composer + +make +container_composer + + + +osbuild_getting_started_makefile->osbuild_composer_makefile_composer + + + + + +osbuild_composer_makefile_worker + +make +container_worker + + + +osbuild_getting_started_makefile->osbuild_composer_makefile_worker + + + + + +weldr_client_makefile + +make +container + + + +osbuild_getting_started_makefile->weldr_client_makefile + + + + + +osbuild_getting_started_run + +make +run_onprem + + + +osbuild_getting_started_run->osbuild_getting_started_makefile + + + + + +osbuild_composer_go + +go code + + + +osbuild_composer_makefile_composer->osbuild_composer_go + + + + + +osbuild_composer_makefile_worker:s->osbuild_makefile:s + + + + + +osbuild_composer_makefile_worker->osbuild_composer_go + + + + + +pulp_client_go + +go code + + + +osbuild_composer_go->pulp_client_go + + + + + +images_go + +go code + + + +osbuild_composer_go->images_go + + + + + diff --git a/docs/src_compile_service.dot b/docs/src_compile_service.dot new file mode 100644 index 0000000..e139f55 --- /dev/null +++ b/docs/src_compile_service.dot @@ -0,0 +1,132 @@ +digraph src_compile { + newrank=true; + subgraph cluster_osbuild_getting_started { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "osbuild-\ngetting-started"; + osbuild_getting_started_makefile [ + label = "make\nservice_containers" + width = 2; + ]; + osbuild_getting_started_run [ + label = "make\nrun_service" + width = 2; + ]; + } + + subgraph cluster_osbuild { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "osbuild\n "; + osbuild_makefile [ + label = "make\ncontainer"; + width = 1.5; + ]; + } + + subgraph cluster_osbuild_composer { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "osbuild-composer\n "; + osbuild_composer_makefile_composer [ + label = "make\ncontainer_composer" + width = 2; + ]; + osbuild_composer_makefile_worker [ + label = "make\ncontainer_worker" + width = 2; + ]; + + osbuild_composer_go [ label = "go code"; ]; + } + + subgraph cluster_image_builder { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "image-builder\n "; + image_builder_makefile [ + label = "make\ncontainer" + width = 1.5; + ]; + image_builder_go [ label = "go code"; ]; + } + + subgraph cluster_image_builder_frontend { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "image-builder-\nfronted"; + image_builder_frontend_makefile [ + label = "make\ncontainer" + width = 1.5; + ]; + } + subgraph cluster_pulp_client { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "pulp-client"; + pulp_client_go [ + label = "go code"; + width = 1.5; + ]; + } + subgraph cluster_images { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "images"; + images_go [label = "go code";]; + } + subgraph cluster_community_gateway { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "community-gateway"; + community_gateway_go [ + label = "go code"; + width = 2; + ]; + } + + { rank=same; + osbuild_makefile; + osbuild_composer_makefile_composer; + osbuild_composer_makefile_worker; + image_builder_makefile; + image_builder_frontend_makefile; + } + { rank=same; + edge [style=dashed]; + image_builder_go -> osbuild_composer_go; + } + { rank=same; + edge [style=dashed; headport=s; tailport=s]; + pulp_client_go; + images_go; + community_gateway_go; + } + osbuild_composer_go -> pulp_client_go [style=dashed;]; + osbuild_composer_go -> images_go [style=dashed;]; + image_builder_go -> community_gateway_go [style=dashed;]; + + osbuild_getting_started_run -> osbuild_getting_started_makefile; + + osbuild_getting_started_makefile -> osbuild_makefile; + osbuild_getting_started_makefile -> osbuild_composer_makefile_worker; + osbuild_getting_started_makefile -> osbuild_composer_makefile_composer; + osbuild_getting_started_makefile -> image_builder_makefile; + osbuild_getting_started_makefile -> image_builder_frontend_makefile; + osbuild_composer_makefile_worker -> osbuild_composer_go [style=dashed]; + osbuild_composer_makefile_composer -> osbuild_composer_go [style=dashed]; + image_builder_makefile -> image_builder_go [style=dashed]; + + + + osbuild_composer_makefile_worker -> osbuild_makefile [style=dashed; headport=s; tailport=s]; + +} diff --git a/docs/src_compile_service.svg b/docs/src_compile_service.svg new file mode 100644 index 0000000..f3dcf5f --- /dev/null +++ b/docs/src_compile_service.svg @@ -0,0 +1,221 @@ + + + + + + +src_compile + + +cluster_osbuild_getting_started + +osbuild- +getting-started + + +cluster_osbuild + +osbuild +  + + +cluster_osbuild_composer + +osbuild-composer +  + + +cluster_image_builder + +image-builder +  + + +cluster_image_builder_frontend + +image-builder- +fronted + + +cluster_pulp_client + +pulp-client + + +cluster_images + +images + + +cluster_community_gateway + +community-gateway + + + +osbuild_getting_started_makefile + +make +service_containers + + + +osbuild_makefile + +make +container + + + +osbuild_getting_started_makefile->osbuild_makefile + + + + + +osbuild_composer_makefile_composer + +make +container_composer + + + +osbuild_getting_started_makefile->osbuild_composer_makefile_composer + + + + + +osbuild_composer_makefile_worker + +make +container_worker + + + +osbuild_getting_started_makefile->osbuild_composer_makefile_worker + + + + + +image_builder_makefile + +make +container + + + +osbuild_getting_started_makefile->image_builder_makefile + + + + + +image_builder_frontend_makefile + +make +container + + + +osbuild_getting_started_makefile->image_builder_frontend_makefile + + + + + +osbuild_getting_started_run + +make +run_service + + + +osbuild_getting_started_run->osbuild_getting_started_makefile + + + + + +osbuild_composer_go + +go code + + + +osbuild_composer_makefile_composer->osbuild_composer_go + + + + + +osbuild_composer_makefile_worker:s->osbuild_makefile:s + + + + + +osbuild_composer_makefile_worker->osbuild_composer_go + + + + + +pulp_client_go + +go code + + + +osbuild_composer_go->pulp_client_go + + + + + +images_go + +go code + + + +osbuild_composer_go->images_go + + + + + +image_builder_go + +go code + + + +image_builder_makefile->image_builder_go + + + + + +image_builder_go->osbuild_composer_go + + + + + +community_gateway_go + +go code + + + +image_builder_go->community_gateway_go + + + + + diff --git a/on-prem/bin/run.py b/on-prem/bin/run.py index 948e234..289f295 100755 --- a/on-prem/bin/run.py +++ b/on-prem/bin/run.py @@ -93,7 +93,6 @@ async def env( "--network", "podman", "--name", f"{prefix}-composer", f"ogsc/run/composer:{osbuild_composer_version}", - "--dnf-json", "--weldr-api", "--remote-worker-api", "--composer-api", diff --git a/on-prem/bin/setup-host.py b/on-prem/bin/setup-host.py index 97bd577..5ac0c40 100755 --- a/on-prem/bin/setup-host.py +++ b/on-prem/bin/setup-host.py @@ -4,18 +4,15 @@ # of setting up all required dependencies on the host to run osbuild services in # containers. -import sys -import subprocess import logging - +import subprocess +import sys log = logging.getLogger(__name__) # The base set of packages necessary. -package_base = { - "git" -} +package_base = {"git"} package_set = { "container": { @@ -25,11 +22,14 @@ def package_install(packages): - subprocess.run([ - "dnf", - "in", - "-y", - ] + list(packages)) + subprocess.run( + [ + "dnf", + "in", + "-y", + ] + + list(packages) + ) return 0 diff --git a/on-prem/src/ogsc/run/composer/entrypoint.py b/on-prem/src/ogsc/run/composer/entrypoint.py index d545d32..e38baf1 100644 --- a/on-prem/src/ogsc/run/composer/entrypoint.py +++ b/on-prem/src/ogsc/run/composer/entrypoint.py @@ -118,34 +118,12 @@ def _parse_args(self): help="Disable the weldr-API", ) - # --[no-]dnf-json - self._parser.add_argument( - "--dnf-json", - action="store_true", - dest="dnf_json", - help="Enable dnf-json", - ) - self._parser.add_argument( - "--no-dnf-json", - action="store_false", - dest="dnf_json", - help="Disable dnf-json", - ) - self._parser.add_argument( - "--dnf-json-port", - type=int, - default=0, - dest="dnf_json_port", - help="Specify the port dnf-json should listen on", - ) - self._parser.set_defaults( builtin_worker=False, composer_api=False, local_worker_api=False, remote_worker_api=False, weldr_api=False, - dnf_json=False, ) return self._parser.parse_args(self._argv[1:]) @@ -295,51 +273,17 @@ def _spawn_composer(sockets): preexec_fn=preexec_setenv, ) - def _spawn_dnf_json(self): - cmd = [ - "/usr/libexec/osbuild-depsolve-dnf", - ] - - if self.args.dnf_json_port: - sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) - self._exitstack.enter_context(contextlib.closing(sock)) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) - sock.bind(("::", self.args.dnf_json_port)) - sock.listen() - else: - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - self._exitstack.enter_context(contextlib.closing(sock)) - os.makedirs("/run/osbuild-dnf-json/", exist_ok=True) - sock.bind("/run/osbuild-dnf-json/api.sock") - sock.listen() - - dnfenv = os.environ.copy() - dnfenv["LISTEN_FDS"] = "1" - dnfenv["LISTEN_FD"] = str(sock.fileno()) - - return subprocess.Popen( - cmd, - cwd="/usr/libexec/osbuild-composer", - stdin=subprocess.DEVNULL, - stderr=subprocess.STDOUT, - env=dnfenv, - pass_fds=[sock.fileno()] - ) - def run(self): """Program Runtime""" proc_composer = None proc_worker = None - proc_dnf_json = None res = 0 sockets = self._prepare_sockets() def handler(signum, frame): proc_composer.terminate() proc_worker.terminate() - proc_dnf_json.terminate() signal.signal(signal.SIGTERM, handler) @@ -351,9 +295,6 @@ def handler(signum, frame): if self.args.builtin_worker: proc_worker = self._spawn_worker() - if self.args.dnf_json: - proc_dnf_json = self._spawn_dnf_json() - if any([self.args.weldr_api, self.args.composer_api, self.args.local_worker_api, self.args.remote_worker_api]): proc_composer = self._spawn_composer(sockets) @@ -365,11 +306,6 @@ def handler(signum, frame): proc_worker.terminate() proc_worker.wait() - if proc_dnf_json: - if proc_composer: - proc_dnf_json.terminate() - proc_dnf_json.wait() - except KeyboardInterrupt: if proc_composer: proc_composer.terminate() @@ -377,14 +313,9 @@ def handler(signum, frame): if proc_worker: proc_worker.terminate() proc_worker.wait() - if proc_dnf_json: - proc_dnf_json.terminate() - proc_dnf_json.wait() except: if proc_worker: proc_worker.kill() - if proc_dnf_json: - proc_dnf_json.kill() if proc_composer: proc_composer.kill() raise diff --git a/service/README.md b/service/README.md index 94dc66e..db4dcb8 100644 --- a/service/README.md +++ b/service/README.md @@ -1,24 +1,39 @@ # devtools Development Tools for Image Builder +Here are some tools of "Image Builder" to start up a development environment for the +[hosted deployment](https://osbuild.org/docs/hosted/architecture/). +The reason for all the `sudo` below is that the stack has to perform may operations +with loop devices etc. where root privileges are required. ## Setup -To start local development, first clone the image builder stack: +To start local development, first clone the image builder stack and all it's dependent source. ```bash -git clone git@github.com:osbuild/osbuild-composer.git +git clone git@github.com:osbuild/community-gateway.git +git clone git@github.com:osbuild/image-builder-frontend.git git clone git@github.com:osbuild/image-builder.git +git clone git@github.com:osbuild/images.git +git clone git@github.com:osbuild/osbuild-composer.git git clone git@github.com:osbuild/osbuild-getting-started.git +git clone git@github.com:osbuild/osbuild.git +git clone git@github.com:osbuild/pulp-client.git +git clone git@github.com:osbuild/weldr-client.git ``` -The folder structure should look like: +The folder structure should look like this: ``` . +├── community-gateway ├── image-builder +├── image-builder-frontend +├── images +├── osbuild +├── osbuild-composer ├── osbuild-getting-started -└── osbuild-composer +└── pulp-client ``` Secondly redirect a stage and prod domains to localhost. If you are outside @@ -55,7 +70,7 @@ command. ## Upload Targets -Upload targets need to be configued for the Image Builder backend to upload successfully. +Upload targets need to be configured for the Image Builder backend to upload successfully. This stack comes pre-configured with a generic S3 bucket which can be accessed at: `http://localhost:9000` @@ -95,49 +110,60 @@ The config variables for the worker can be found [here](https://github.com/osbui *NOTE:* If you change the config files, you will either need to modify the worker config in the `/scratch/podman/image-builder-config` file and restart the containers. Alternatively, you will need to remove the named volume and rebuild the config container. The steps for this are as follows: -Run the following from **this directory**. +Run the following from the main **osbuild-getting-started directory** (one level above where this README.md is). + +```bash +make help +``` + - stop the containers and remove volumes ```bash -docker compose down -v +make prune-service ``` -- rebuild the config container +- rebuild containers ```bash -docker compose build config +make service-containers ``` - start the containers again ```bash -docker compose up +make run-service ``` -## Run the backend +## Run the frontend separately -To build the containers run the following command from **this directory**: +The command above (`make run-service`) also starts the frontend by bind-mounting the source +from your computer, so hot-reload of `npm` should work. +If you want to run the frontend yourself (see the [README.md](https://github.com/osbuild/image-builder-frontend/blob/main/README.md) there) +you can use the make target: ```bash -docker compose build +make run-service-no-fronted ``` -To run the containers: +You have to have a "staging account" in order to run this setup (although it's local). +Check if you can log in at https://console.stage.redhat.com/ -```bash -docker compose up -``` +Access the service through the GUI: +[https://stage.foo.redhat.com:1337/beta/insights/image-builder](https://stage.foo.redhat.com:1337/beta/insights/image-builder), or +directly through the API: +[https://stage.foo.redhat.com:1337/docs/api/image-builder](https://stage.foo.redhat.com:1337/docs/api/image-builder). -## Run the frontend +## Debugging +When you want to use your IDE (Integrated Development Environment) to debug the go-code you can check out the +`GODEBUG_PORT` variables in [docker-composer.yml](./docker-composer.yml) and enable debugging for the supported +layers. -The frontend has been removed as a container in favour of running it separately in order to leverage hot-reloading -capabilities. In order to run the frontend with the backend you can run the following command: +## Known Problems -```bash -npm run devel +If you encounter problems with the certificate when you browse to +[https://stage.foo.redhat.com:1337/preview/insights/image-builder](https://stage.foo.redhat.com:1337/preview/insights/image-builder) +you might need to rebuild the certificates with: +``` +make wipe-config ``` -Access the service through the GUI: -[https://stage.foo.redhat.com:1337/beta/insights/image-builder](https://stage.foo.redhat.com:1337/beta/insights/image-builder), or -directly through the API: -[https://stage.foo.redhat.com:1337/docs/api/image-builder](https://stage.foo.redhat.com:1337/docs/api/image-builder). diff --git a/service/config/Dockerfile-config b/service/config/Dockerfile-config index d876df4..fbd2664 100644 --- a/service/config/Dockerfile-config +++ b/service/config/Dockerfile-config @@ -1,15 +1,21 @@ FROM fedora:38 RUN dnf install -y openssl -RUN mkdir -p /config +RUN mkdir -p /config_build COPY ./tools/gen-certs.sh . COPY ./config/x509/openssl.cnf . -COPY ./config/composer/acl.yml /config/. -COPY ./config/composer/osbuild-composer.toml /config/. -COPY ./config/worker/osbuild-worker.toml /config/. -COPY ./config/worker/s3-credentials /config/. -COPY ./config/worker/secret /config/. -COPY ./config/backend/quotas.json /config/. - -RUN ./gen-certs.sh ./openssl.cnf /config /config/ca +COPY ./config/composer/acl.yml /config_build/. +COPY ./config/composer/osbuild-composer.toml /config_build/. +COPY ./config/worker/osbuild-worker.toml /config_build/. +COPY ./config/worker/s3-credentials /config_build/. +COPY ./config/worker/secret /config_build/. +COPY ./config/backend/quotas.json /config_build + +# Helper to re-run gen-certs.sh +ARG CONFIG_BUILD_DATE=NOT_SET +ENV CONFIG_BUILD_DATE=$CONFIG_BUILD_DATE + +RUN ./gen-certs.sh ./openssl.cnf /config_build /config_build/ca + +CMD cp -r /config_build/* /config/ && echo "Done. Provided the config to all other containers." diff --git a/service/config/Dockerfile-config-onprem b/service/config/Dockerfile-config-onprem new file mode 100644 index 0000000..f351d05 --- /dev/null +++ b/service/config/Dockerfile-config-onprem @@ -0,0 +1,21 @@ +FROM fedora:38 + +RUN dnf install -y openssl +RUN mkdir -p /config_build + +COPY ./tools/gen-certs.sh . +COPY ./config/x509/openssl.cnf . +COPY ./config/composer/acl.yml /config_build/. +COPY ./config/composer/osbuild-composer-onprem.toml /config_build/osbuild-composer.toml +# COPY ./config/worker/osbuild-worker-onprem.toml /config_build/osbuild-worker.toml +COPY ./config/worker/s3-credentials /config_build/. +COPY ./config/worker/secret /config_build/. +COPY ./config/backend/quotas.json /config_build + +# Helper to re-run gen-certs.sh +ARG CONFIG_BUILD_DATE=NOT_SET +ENV CONFIG_BUILD_DATE=$CONFIG_BUILD_DATE + +RUN ./gen-certs.sh ./openssl.cnf /config_build /config_build/ca + +CMD cp -r /config_build/* /config/ && cp -r /repositories /config/ && echo "Done. Provided the config to all other containers." diff --git a/service/config/backend/quotas.json b/service/config/backend/quotas.json index c3a2a04..d0b5af9 100644 --- a/service/config/backend/quotas.json +++ b/service/config/backend/quotas.json @@ -3,4 +3,4 @@ "quota":200, "slidingWindow":1209600000000000 } -} + } diff --git a/service/config/cli/data/blueprint.toml b/service/config/cli/data/blueprint.toml new file mode 100644 index 0000000..9971ac3 --- /dev/null +++ b/service/config/cli/data/blueprint.toml @@ -0,0 +1,9 @@ +name = "example-image" +description = "An example image for osbuild-getting-started." +version = "0.0.1" +modules = [] +groups = [] + +[[packages]] +name = "tmux" +version = "*" diff --git a/service/config/composer/osbuild-composer-onprem.toml b/service/config/composer/osbuild-composer-onprem.toml new file mode 100644 index 0000000..6cfff0a --- /dev/null +++ b/service/config/composer/osbuild-composer-onprem.toml @@ -0,0 +1,7 @@ +[koji] +allowed_domains = [ "localhost", "client.osbuild.org" ] +ca = "/etc/osbuild-composer/ca-crt.pem" + +[worker] +allowed_domains = [ "localhost", "worker.osbuild.org", "172.31.0.1" ] +ca = "/etc/osbuild-composer/ca-crt.pem" diff --git a/service/docker-compose-onprem.yml b/service/docker-compose-onprem.yml new file mode 100644 index 0000000..7480988 --- /dev/null +++ b/service/docker-compose-onprem.yml @@ -0,0 +1,155 @@ +networks: + onpremnet: + ipam: + driver: default + config: + - subnet: 172.32.0.0/16 + +volumes: + config_volume: + driver_opts: + type: none + # this can be modified as desired + device: /scratch/podman/image-builder-config + o: bind + weldr_volume: + driver_opts: + type: none + # this can be modified as desired + device: /scratch/podman/weldr + o: bind + cloudapi_volume: + driver_opts: + type: none + # this can be modified as desired + device: /scratch/podman/cloudapi + o: bind + dnfjson_volume: + driver_opts: + type: none + # this can be modified as desired + device: /scratch/podman/dnf-json + o: bind + +services: + config: + image: local/config + build: + context: . + dockerfile: ./config/Dockerfile-config-onprem + volumes: + - config_volume:/config:z + - ../../osbuild-composer/repositories:/repositories:z + + composer: + image: osbuild-composer_dev + entrypoint: + [ + "python3", + "/opt/entrypoint.py", + "--weldr-api", + "--remote-worker-api", + "--composer-api", + "--composer-api-port", "8000", + ] + healthcheck: + test: ["CMD-SHELL", "curl -k -s --cert /etc/osbuild-composer/worker-crt.pem --key /etc/osbuild-composer/worker-key.pem -o /dev/null https://localhost:8700/api/worker/v1/status"] + interval: 2s + timeout: 2s + retries: 100 + environment: + - PGHOST=postgres_composer + - PGPORT=5432 + - PGDATABASE=postgres + - PGUSER=postgres + - PGPASSWORD=postgres + - PGSSLMODE=disable + # only enable the GODEBUG_PORT if you want live debugging. + # you HAVE to attach a golang debugger to continue execution (check the logs after startup) + # - GODEBUG_PORT=7730 + volumes: + - config_volume:/etc/osbuild-composer + - weldr_volume:/run/weldr:rw,z + - cloudapi_volume:/run/cloudapi:rw,z + - dnfjson_volume:/run/osbuild-dnf-json:rw,z + ports: + - 8080:8080 + - 8700:8700 + - 7730:7730 + depends_on: + config: + condition: service_completed_successfully + postgres_composer: + condition: service_healthy + + networks: + onpremnet: + ipv4_address: 172.32.0.10 + + worker: + image: osbuild-worker_dev + volumes: + - config_volume:/etc/osbuild-composer:z + - config_volume:/etc/osbuild-worker:z + environment: + - CACHE_DIRECTORY=/var/cache/osbuild-composer + - WORKER_ARGS=composer:8700 + # only enable the GODEBUG_PORT if you want live debugging. + # you HAVE to attach a golang debugger to continue execution (check the logs after startup) + # - GODEBUG_PORT=7740 + # only enable the PYTHON_DEBUG_PORT if you want live debugging. + # you HAVE to attach a pydev debugger to continue execution (check the logs after starting a build) + # it will connect to your localhost:5679 and expects a python debug server there + # - PYTHON_DEBUG_PORT=5679 + ports: + - 7740:7740 + privileged: true + cap_add: + - CAP_MKNOD + - SYS_ADMIN + - NET_ADMIN + depends_on: + config: + condition: service_completed_successfully + composer: + condition: service_healthy + restart: on-failure + networks: + onpremnet: + ipv4_address: 172.32.0.20 + extra_hosts: + - "host.docker.internal:host-gateway" + + + postgres_composer: + image: docker.io/postgres:10.5 + healthcheck: + test: [ "CMD", "pg_isready", "-U", "postgres", "-d", "postgres" ] + interval: 2s + timeout: 2s + retries: 10 + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + volumes: + - ../../osbuild-composer/pkg/jobqueue/dbjobqueue/schemas/:/docker-entrypoint-initdb.d/:z + networks: + onpremnet: + ipv4_address: 172.32.0.30 + + cli: + image: osbuild-cli_dev + volumes: + - config_volume:/etc/osbuild-composer:z + - weldr_volume:/run/weldr:rw,z + - dnfjson_volume:/run/osbuild-dnf-json:rw,z + - ./config/cli/data:/data + #environment: + # only enable the GODEBUG_PORT if you want live debugging. + # you HAVE to attach a golang debugger to continue execution (check the logs after startup) + # - GODEBUG_PORT=7750 + networks: + onpremnet: + ipv4_address: 172.32.0.40 + entrypoint: + [ "/usr/bin/true" ] # will be started interactively from Makefile diff --git a/service/docker-compose.yml b/service/docker-compose.yml index 01af030..472abc7 100644 --- a/service/docker-compose.yml +++ b/service/docker-compose.yml @@ -1,18 +1,16 @@ -version: '3.9' - networks: - net: + servicenet: ipam: driver: default config: - subnet: 172.31.0.0/16 volumes: - config: + config_volume: driver_opts: type: none # this can be modified as desired - device: /scratch/podman/image-builder-config + device: ${SCRATCH_DIR:-/scratch}/podman/image-builder-config o: bind services: @@ -22,14 +20,27 @@ services: context: . dockerfile: ./config/Dockerfile-config volumes: - - config:/config + - config_volume:/config:z - ../../osbuild-composer/repositories:/config/repositories:z + postgres_composer: + image: docker.io/postgres:10.5 + healthcheck: + test: [ "CMD", "pg_isready", "-h", "localhost", "-U", "postgres", "-d", "postgres" ] + interval: 2s + timeout: 2s + retries: 10 + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + volumes: + - ../../osbuild-composer/pkg/jobqueue/dbjobqueue/schemas/:/docker-entrypoint-initdb.d/:z + networks: + servicenet: + ipv4_address: 172.31.0.80 + composer: - image: local/osbuild-composer - build: - context: ../../osbuild-composer - dockerfile: ./distribution/Dockerfile-ubi + image: osbuild-composer_dev entrypoint: [ "python3", @@ -37,46 +48,75 @@ services: "--remote-worker-api", "--composer-api" ] + healthcheck: + test: ["CMD-SHELL", "if curl -k -s -o /dev/null -w '%{http_code}' https://localhost:8700/ | grep -q 401; then exit 0; else exit 1; fi"] + interval: 2s + timeout: 2s + retries: 100 + environment: + - PGHOST=postgres_composer + - PGPORT=5432 + - PGDATABASE=postgres + - PGUSER=postgres + - PGPASSWORD=postgres + - PGSSLMODE=disable + # only enable the GODEBUG_PORT if you want live debugging. + # you HAVE to attach a golang debugger to continue execution (check the logs after startup) + # - GODEBUG_PORT=7730 volumes: - - config:/etc/osbuild-composer + - config_volume:/etc/osbuild-composer ports: - 8080:8080 - 8700:8700 + - 7730:7730 depends_on: - - config + config: + condition: service_completed_successfully + postgres_composer: + condition: service_healthy + networks: - net: + servicenet: ipv4_address: 172.31.0.10 worker: - image: local/osbuild-worker - build: - context: ../../osbuild-composer - dockerfile: ./distribution/Dockerfile-worker - # override the entrypoint to specify composer hostname and port - entrypoint: [ "/usr/libexec/osbuild-composer/osbuild-worker", "composer:8700" ] + image: osbuild-worker_dev volumes: - - config:/etc/osbuild-composer - - config:/etc/osbuild-worker + - config_volume:/etc/osbuild-composer:z + - config_volume:/etc/osbuild-worker:z environment: - CACHE_DIRECTORY=/var/cache/osbuild-composer + - WORKER_ARGS=composer:8700 + # only enable the GODEBUG_PORT if you want live debugging. + # you HAVE to attach a golang debugger to continue execution (check the logs after startup) + # - GODEBUG_PORT=7740 + # only enable the PYTHON_DEBUG_PORT if you want live debugging. + # you HAVE to attach a pydev debugger to continue execution (check the logs after starting a build) + # it will connect to your localhost:5679 and expects a python debug server there + # - PYTHON_DEBUG_PORT=5679 + ports: + - 7740:7740 privileged: true cap_add: - CAP_MKNOD - - SYS_ADMIN - NET_ADMIN + - SYS_ADMIN depends_on: - - config - - composer + config: + condition: service_completed_successfully + composer: + condition: service_healthy restart: on-failure networks: - net: + servicenet: ipv4_address: 172.31.0.20 + extra_hosts: + - "host.docker.internal:host-gateway" - postgres: + postgres_backend: image: docker.io/postgres:10.5 healthcheck: - test: [ "CMD", "pg_isready", "-U", "postgres", "-d", "postgres" ] + test: [ "CMD", "pg_isready", "-h", "localhost", "-U", "postgres", "-d", "postgres" ] interval: 2s timeout: 2s retries: 10 @@ -84,30 +124,28 @@ services: - POSTGRES_USER=postgres - POSTGRES_PASSWORD=postgres volumes: - - ../../image-builder/internal/db/migrations-tern/:/docker-entrypoint-initdb.d/:Z + - ../../image-builder/internal/db/migrations-tern/:/docker-entrypoint-initdb.d/:z networks: - net: + servicenet: ipv4_address: 172.31.0.30 backend: - image: local/image-builder - build: - context: ../../image-builder - dockerfile: ./distribution/Dockerfile-ubi + image: image-builder_dev ports: - 8086:8086 + - 7720:7720 healthcheck: test: [ "CMD", "curl", "localhost:8086/status" ] interval: 2s timeout: 2s - retries: 10 + retries: 100 volumes: - - config:/etc/image-builder + - config_volume:/etc/image-builder environment: - LISTEN_ADDRESS=backend:8086 - LOG_LEVEL=DEBUG - ALLOWED_ORG_IDS=* - - PGHOST=postgres + - PGHOST=postgres_backend - PGPORT=5432 - PGDATABASE=postgres - PGUSER=postgres @@ -119,48 +157,75 @@ services: - COMPOSER_CA_PATH=/etc/image-builder/ca-crt.pem - DISTRIBUTIONS_DIR=/app/distributions - QUOTA_FILE=/etc/image-builder/quotas.json + # only enable the GODEBUG_PORT if you want live debugging. + # you HAVE to attach a golang debugger to continue execution (check the logs after startup) + # - GODEBUG_PORT=7720 depends_on: - - config - - composer - - postgres + config: + condition: service_completed_successfully + composer: + condition: service_healthy + postgres_backend: + condition: service_healthy restart: on-failure networks: - net: + servicenet: ipv4_address: 172.31.0.40 fauxauth: - image: local/osbuild-fauxauth + image: local/osbuild-fauxauth_dev build: context: ../../osbuild-composer - dockerfile: ./distribution/Dockerfile-fauxauth + dockerfile: ./distribution/Dockerfile-fauxauth.dev + args: + GOMODARGS: "-modfile=go.local.mod" entrypoint: [ "/opt/fauxauth.py", "-a", "0.0.0.0", "-p", "8888" ] volumes: - - config:/etc/osbuild-composer + - config_volume:/etc/osbuild-composer ports: - 8888:8888 depends_on: - config networks: - net: + servicenet: ipv4_address: 172.31.0.50 minio: - image: minio/minio:latest + image: docker.io/minio/minio:latest volumes: # this can be modified as desired - - /scratch/data:/data:z + - ${SCRATCH_DIR:-/scratch}/data/s3:/data:z command: server /data --console-address ":9090" + working_dir: /data environment: - - MINIO_ROOT_USER=admin - - MINIO_ROOT_PASSWORD=password42 + - MINIO_ROOT_USER=admin + - MINIO_ROOT_PASSWORD=password42 ports: - 9000:9000 - - 9090:9090 + - 9091:9090 healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ] interval: 30s timeout: 20s retries: 3 networks: - net: + servicenet: ipv4_address: 172.31.0.60 + + frontend: + image: image-builder-frontend_dev + security_opt: + - seccomp=unconfined + environment: + - BACKEND_HOSTNAME=backend + ports: + - 1337:1337 + - 8002:8002 + volumes: + - ../../image-builder-frontend/api:/app/api:z + - ../../image-builder-frontend/config:/app/config:z + - ../../image-builder-frontend/src:/app/src:z + networks: + servicenet: + ipv4_address: 172.31.0.70 + diff --git a/service/tools/gen-certs.sh b/service/tools/gen-certs.sh index c603279..9b7a95a 100755 --- a/service/tools/gen-certs.sh +++ b/service/tools/gen-certs.sh @@ -1,4 +1,13 @@ #!/bin/bash + +if [ -f /config ] ; then + echo "ERROR: /config should not be a file. Your setup seems to be broken!" + echo "Please check your volume mounts of the container runtime (podman/docker)" + exit 1 +fi + +ls -la /config/ + if (( $# != 3 )); then echo "Usage: $0 " echo diff --git a/tools/git_stack.sh b/tools/git_stack.sh new file mode 100755 index 0000000..189f8dc --- /dev/null +++ b/tools/git_stack.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +# repos to respect +BASEDIR=$(dirname $(readlink -f $0))/../.. +REPOS="osbuild-getting-started osbuild osbuild-composer images image-builder image-builder-frontend weldr-client" +REPOS="$REPOS pulp-client community-gateway" + +ARGS=${*:-status --branch --short} + +NUM="$(echo $REPOS | wc -w)" +I=0 +running_jobs=0 +MAX_PARALLEL=4 +tmp_files=() + +RED='\033[0;31m' +NO_COLOR='\033[0m' + +function run_git_command { + local dir=$1 + local args=$2 + local tmp_file=$3 + git -c color.status=always -C "$dir" $args >> "$tmp_file" 2>&1 || echo -e "${RED}ERROR code: $?${NO_COLOR}" >> "$tmp_file" & +} + +for D in $REPOS; do + I=$(( $I + 1 )) + FULL_DIR="$BASEDIR/$D" + if [ "$ARGS" == "walk" ]; then + echo " ------------- ( $I / $NUM ) $D -------------" + pushd $FULL_DIR + git status + bash + popd + else + tmp_file=$(mktemp) + tmp_files+=("$tmp_file") + echo " ------------- ( $I / $NUM ) $D -------------" > "$tmp_file" + echo " -- Started -- ( $I / $NUM ) $D -------------" + run_git_command "$FULL_DIR" "$ARGS" "$tmp_file" + + (( running_jobs++ )) + if (( running_jobs >= MAX_PARALLEL )); then + wait -n + (( running_jobs-- )) + fi + fi +done + +if [ "$ARGS" != "walk" ]; then + wait +fi + +echo " ------------- DONE -------------" + +# Print the buffered output +for tmp_file in "${tmp_files[@]}"; do + echo "" + cat "$tmp_file" + rm "$tmp_file" +done +