Containerize documentation build environment and add sphinx-autobuild for hot-reload

.gitignore

@@ -2,6 +2,7 @@
 _build/
 _bin/
 build/
+.tmp_build/
 .vscode/
 *.swp
 *.swo

Makefile

@@ -11,6 +11,7 @@ GIT_HASH := $(shell git rev-parse --short HEAD)
 TIMESTAMP := $(shell date '+%Y-%m-%d')
 PACKAGE ?=github.com/flyteorg/flytestdlib
 LD_FLAGS="-s -w -X $(PACKAGE)/version.Version=$(GIT_VERSION) -X $(PACKAGE)/version.Build=$(GIT_HASH) -X $(PACKAGE)/version.BuildTime=$(TIMESTAMP)"
+TMP_BUILD_DIR := .tmp_build
 
 .PHONY: cmd/single/dist
 cmd/single/dist: export FLYTECONSOLE_VERSION ?= latest
@@ -43,7 +44,7 @@ release_automation:
 	$(MAKE) -C docker/sandbox-bundled manifests
 
 .PHONY: deploy_sandbox
-deploy_sandbox: 
+deploy_sandbox:
 	bash script/deploy.sh
 
 .PHONY: install-piptools
@@ -82,10 +83,30 @@ helm_install: ## Install helm charts
 helm_upgrade: ## Upgrade helm charts
 	helm upgrade flyte --debug ./charts/flyte -f ./charts/flyte/values.yaml --create-namespace --namespace=flyte
 
+# Used in CI
 .PHONY: docs
 docs:
 	make -C docs clean html SPHINXOPTS=-W
 
+$(TMP_BUILD_DIR):
+	mkdir $@
+
+$(TMP_BUILD_DIR)/conda-lock-image: docs/Dockerfile.conda-lock | $(TMP_BUILD_DIR)
+	docker buildx build --load --platform=linux/amd64 --build-arg USER_UID=$$(id -u) --build-arg USER_GID=$$(id -g) -t flyte-conda-lock:latest -f docs/Dockerfile.conda-lock .
+	touch $(TMP_BUILD_DIR)/conda-lock-image
+
+monodocs-environment.lock.yaml: monodocs-environment.yaml $(TMP_BUILD_DIR)/conda-lock-image
+	docker run --platform=linux/amd64 --rm --pull never -v ./:/flyte flyte-conda-lock:latest lock --file monodocs-environment.yaml --lockfile monodocs-environment.lock.yaml
+
+$(TMP_BUILD_DIR)/dev-docs-image: docs/Dockerfile.docs monodocs-environment.lock.yaml | $(TMP_BUILD_DIR)
+	docker buildx build --load --platform=linux/amd64 --build-arg USER_UID=$$(id -u) --build-arg USER_GID=$$(id -g) -t flyte-dev-docs:latest -f docs/Dockerfile.docs .
+	touch $(TMP_BUILD_DIR)/dev-docs-image
+
+# Build docs in docker container for local development
+.PHONY: dev-docs
+dev-docs: $(TMP_BUILD_DIR)/dev-docs-image
+	bash script/local_build_docs.sh
+
 .PHONY: help
 help: SHELL := /bin/sh
 help: ## List available commands and their usage
@@ -121,5 +142,5 @@ lint-helm-charts:
 	act pull_request -W .github/workflows/validate-helm-charts.yaml --container-architecture linux/amd64 -e charts/event.json
 
 .PHONY: clean
-clean: ## Remove the HTML files related to the Flyteconsole.
-	rm -rf cmd/single/dist
+clean: ## Remove the HTML files related to the Flyteconsole and Makefile
+	rm -rf cmd/single/dist .tmp_build

docs/Dockerfile.conda-lock

@@ -0,0 +1,12 @@
+FROM condaforge/mambaforge:latest
+
+ARG USER_UID
+ARG USER_GID
+
+RUN getent group "${USER_GID}" || groupadd --gid "${USER_GID}" flyte
+RUN getent passwd "${USER_UID}" || useradd --uid "${USER_UID}" --gid "${USER_GID}" -m flyte
+
+RUN conda install -c conda-forge conda-lock
+WORKDIR flyte
+USER ${USER_UID}
+ENTRYPOINT ["conda", "run", "--no-capture-output", "conda-lock"]

docs/Dockerfile.docs

@@ -0,0 +1,21 @@
+FROM condaforge/mambaforge:latest
+
+ARG USER_UID
+ARG USER_GID
+
+RUN getent group "${USER_GID}" || groupadd --gid "${USER_GID}" flyte
+RUN getent passwd "${USER_UID}" || useradd --uid "${USER_UID}" --gid "${USER_GID}" -m flyte
+
+RUN conda install -c conda-forge conda-lock
+RUN --mount=type=bind,source=monodocs-environment.lock.yaml,target=monodocs-environment.lock.yaml \
+    conda-lock install -n monodocs-env monodocs-environment.lock.yaml
+
+ENV SETUPTOOLS_SCM_PRETEND_VERSION 2.0.0
+COPY flyteidl flyteidl
+SHELL ["conda", "run", "-n", "monodocs-env", "/bin/bash", "-c"]
+RUN python -m pip install sphinx-autobuild
+RUN python -m pip install ./flyteidl
+
+WORKDIR /flyte/docs
+USER ${USER_UID}
+ENTRYPOINT ["conda", "run", "--no-capture-output", "-n", "monodocs-env"]

docs/Makefile

@@ -20,4 +20,4 @@ help:
 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 
 clean:
-	rm -rf _build _src api flytectl flytekit flytesnacks protos examples
+	rm -rf _build _src _tags api flytectl flytekit flytesnacks protos examples tests _projects

docs/community/contribute.rst

@@ -155,20 +155,10 @@ To understand how the below components interact with each other, refer to :ref:`
 
 To build the Flyte docs locally you will need the following prerequisites:
 
-* Install ``conda``.
-    *  We recommend Miniconda installed with an `official installer <https://docs.conda.io/projects/miniconda/en/latest/index.html#latest-miniconda-installer-links>`__.
 * Install `conda-lock <https://github.com/conda/conda-lock>`__.
-* In the ``flyteorg/flyte`` root directory run:
-    * ``conda-lock install --name monodocs-env monodocs-environment.lock.yaml``
-    * ``conda activate monodocs-env``
-    * ``pip install ./flyteidl``
-
-This will set up the Python virtual environment for building the docs. Here we called it ``monodocs-env``.
-
-To actually build the docs, activate ``monodocs-env`` and (in the ``flyteorg/flyte`` root directory) run:
-    * ``make docs``
-
-The resulting ``html`` files will be in ``docs/_build/html``. You can view them by running `open docs/_build/html/index.html`
+* In the ``flyteorg/flyte`` root directory you can run:
+    * ``make dev-docs`` to build the documentation locally. The build will be in the ``docs/_build/html`` directory. See `the script <https://github.com/flyteorg/flyte/blob/master/script/local_build_docs.sh>`__ for additional environment variables that can be set.
+        * For example, to use the local flytekit source code instead of the source code from the flyteorg/flytekit repo, run ``export FLYTEKIT_LOCAL_PATH=/path/to/flytekit`` before running ``make dev-docs``.
 
 ``flyteidl``
 ************