From 7fca3c27b298843d5d18fe84ff46fcf0557f00d6 Mon Sep 17 00:00:00 2001 From: Marius Vikhammer Date: Thu, 29 Apr 2021 12:27:16 +0800 Subject: [PATCH] Add CI for esp-docs Adds basic CI for esp-docs. Includes: * Linting with Flake8 * Building simple examples * Building esp-idf, esp-at and esp-adf docs --- .flake8 | 7 + .gitlab-ci.yml | 224 ++++++++++++++++++ ci/deploy_package.py | 72 ++++++ ci/set_repo.sh | 19 ++ ci/setup_python.sh | 47 ++++ ci/utils.sh | 18 ++ test/en/conf.py | 20 +- test/test_docs.py | 8 +- ...f_extensions.py => test_esp_extensions.py} | 42 +--- 9 files changed, 403 insertions(+), 54 deletions(-) create mode 100644 .flake8 create mode 100644 .gitlab-ci.yml create mode 100644 ci/deploy_package.py create mode 100755 ci/set_repo.sh create mode 100644 ci/setup_python.sh create mode 100644 ci/utils.sh rename test/{test_sphinx_idf_extensions.py => test_esp_extensions.py} (70%) diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..787e2a2 --- /dev/null +++ b/.flake8 @@ -0,0 +1,7 @@ +[flake8] +exclude = citools/ +max-line-length = 160 +per-file-ignores = + # conf.py files use star imports to setup config variables + examples/*/conf*.py: F405 + test/*/conf*py: F405 \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..414f4fc --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,224 @@ +stages: + - check + - test + - build + - test_deploy + - deploy + +variables: + # Versioned esp-idf-doc env image to use for all document building jobs + ESP_DOCS_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env:v4.4-1-v1" + PYTHON_VER: 3.6.10 + IDF_REPO: ${GITLAB_SSH_SERVER}/idf/esp-idf.git + ESP_DOCS_PATH: "$CI_PROJECT_DIR" + IDF_PATH: $CI_PROJECT_DIR/esp-idf + IDF_REPO: ${GITLAB_SSH_SERVER}/idf/esp-idf.git + AT_PATH: $CI_PROJECT_DIR/esp-at + AT_REPO: ${GITLAB_SSH_SERVER}/application/esp-at.git + ADF_PATH: $CI_PROJECT_DIR/esp-adf-internal + ADF_REPO: ${GITLAB_SSH_SERVER}/adf/esp-adf-internal.git + + +.before_script_minimal: + before_script: + - source ci/setup_python.sh + - source ci/utils.sh + +.before_script_prepare_build: + before_script: + # Use CI Tools + - curl -sSL ${CIT_LOADER_URL} | sh + - source citools/import_functions + - source ci/setup_python.sh + - source ci/utils.sh + # Purge any old esp-docs versions + - pip3 uninstall -y esp-docs + - pip3 install . + +check_setup: + stage: check + image: $ESP_DOCS_ENV_IMAGE + extends: + - .before_script_minimal + script: + - pip3 install . + +check_python_style: + stage: check + image: $ESP_DOCS_ENV_IMAGE + extends: + - .before_script_minimal + script: + - python -m flake8 --config=$ESP_DOCS_PATH/.flake8 $ESP_DOCS_PATH + + +test_extensions_ut: + stage: test + image: $ESP_DOCS_ENV_IMAGE + extends: + - .before_script_prepare_build + script: + - cd test + - python test_docs.py + - python test_esp_extensions.py + + +.build_template: + stage: build + image: $ESP_DOCS_ENV_IMAGE + extends: + - .before_script_prepare_build + artifacts: + when: always + paths: + - $DOCS_DIR/_build/*/*/html/* + - $DOCS_DIR/_build/*/*/*.txt + expire_in: 4 days + +build_example_basic: + extends: + - .build_template + variables: + DOCS_DIR: $CI_PROJECT_DIR/examples/basic + script: + - cd "$DOCS_DIR" + - ./build_example.sh + +build_example_doxygen: + extends: + - .build_template + variables: + DOCS_DIR: $CI_PROJECT_DIR/examples/doxygen + script: + - cd $DOCS_DIR + - ./build_example.sh + +build_idf_docs: + extends: + - .build_template + variables: + DOCS_DIR: $CI_PROJECT_DIR/esp-idf/docs + DOCS_FAST_BUILD: 'yes' + script: + # add gitlab ssh key + - cit_add_ssh_key "${GITLAB_KEY_PEM}" + - git clone "${IDF_REPO}" + - cd $IDF_PATH && tools/idf_tools.py --non-interactive install && eval "$(tools/idf_tools.py --non-interactive export)" + # build with IDF v.X (test branch for now) + - $ESP_DOCS_PATH/ci/set_repo.sh feature/esp_docs $IDF_PATH + - cd $DOCS_DIR + - build-docs -l $DOCLANG -t $DOCTGT -d doxygen + parallel: + matrix: + - DOCLANG: ["en", "zh_CN"] + DOCTGT: ["esp32", "esp32s2", "esp32c3"] + +build_at_docs: + extends: + - .build_template + variables: + DOCS_DIR: $CI_PROJECT_DIR/esp-at/docs + script: + # add gitlab ssh key + - cit_add_ssh_key "${GITLAB_KEY_PEM}" + - git clone "${AT_REPO}" + - $ESP_DOCS_PATH/ci/set_repo.sh test/esp_docs $AT_PATH + - cd $DOCS_DIR + - build-docs -l $DOCLANG -t $DOCTGT + parallel: + matrix: + - DOCLANG: ["en", "zh_CN"] + DOCTGT: ["esp8266", "esp32", "esp32s2", "esp32c3"] + +build_adf_docs: + extends: + - .build_template + variables: + DOCS_DIR: $CI_PROJECT_DIR/esp-adf-internal/docs + script: + # add gitlab ssh key + - cit_add_ssh_key "${GITLAB_KEY_PEM}" + - git clone "${ADF_REPO}" + - $ESP_DOCS_PATH/ci/set_repo.sh test/esp_docs $ADF_PATH + - cd $DOCS_DIR + - build-docs -l $DOCLANG -t $DOCTGT + parallel: + matrix: + - DOCLANG: ["en", "zh_CN"] + DOCTGT: ["esp32", "esp32s2"] + +build_package: + stage: build + image: $ESP_DOCS_ENV_IMAGE + extends: + - .before_script_minimal + tags: + - build + dependencies: [] + artifacts: + when: always + paths: + - dist/* + script: + - python -m pip install build + - python -m build + +.deploy_docs_template: + stage: test_deploy + image: $ESP_DOCS_ENV_IMAGE + extends: + - .before_script_prepare_build + variables: + DOCS_DEPLOY_PRIVATEKEY: "$DOCS_DEPLOY_KEY" + DOCS_DEPLOY_SERVER: "$DOCS_SERVER" + DOCS_DEPLOY_SERVER_USER: "$DOCS_SERVER_USER" + DOCS_DEPLOY_PATH: "$DOCS_PATH" + DOCS_DEPLOY_URL_BASE: "https://$CI_DOCKER_REGISTRY/docs/esp-idf" + dependencies: [] + script: + - add_doc_server_ssh_keys $DOCS_DEPLOY_PRIVATEKEY $DOCS_DEPLOY_SERVER $DOCS_DEPLOY_SERVER_USER + - export GIT_VER=$(git describe --always) + - deploy-docs + +deploy_docs_idf: + extends: + - .deploy_docs_template + dependencies: # set dependencies to null to avoid missing artifacts issue + variables: + DOCS_BUILD_DIR: "${CI_PROJECT_DIR}/esp-idf/docs/_build/" + TYPE: "preview" + needs: + - build_idf_docs + +deploy_docs_at: + extends: + - .deploy_docs_template + dependencies: # set dependencies to null to avoid missing artifacts issue + variables: + DOCS_BUILD_DIR: "$CI_PROJECT_DIR/esp-at/docs/_build/" + TYPE: "preview" + needs: + - build_at_docs + +deploy_docs_adf: + extends: + - .deploy_docs_template + dependencies: # set dependencies to null to avoid missing artifacts issue + variables: + DOCS_BUILD_DIR: "$CI_PROJECT_DIR/esp-adf-internal/docs/_build/" + TYPE: "preview" + needs: + - build_adf_docs + +deploy_package: + stage: deploy + image: $ESP_DOCS_ENV_IMAGE + tags: + - build + only: + - master + when: on_success + dependencies: + - build_package + script: + - python $ESP_DOCS_PATH/ci/deploy_dist.py diff --git a/ci/deploy_package.py b/ci/deploy_package.py new file mode 100644 index 0000000..c181210 --- /dev/null +++ b/ci/deploy_package.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +# coding=utf-8 +# +# Deploy esp_docs package to PyPI +# +# Copyright 2021 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import subprocess +import os +import json +import urllib.request +from pathlib import Path +from distutils.version import StrictVersion + +PROJECT_DIR = Path(os.environ["ESP_DOCS_PATH"]) + + +def get_pypi_dist_version(): + url = "https://pypi.python.org/pypi/esp_docs/json" + + with urllib.request.urlopen(url, timeout=30) as conn: + data = json.loads(conn.read().decode("utf-8")) + + versions = list(data["releases"].keys()) + + versions.sort(key=StrictVersion) + return StrictVersion(versions[-1]) + + +def get_local_dist_version(): + setup_py_path = PROJECT_DIR / "setup.py" + ret = subprocess.run(["python3", str(setup_py_path), "--version"], stdout=subprocess.PIPE, check=True, timeout=100) + + return StrictVersion(ret.stdout.decode("utf-8")) + + +def pypi_dist_is_outdated(): + local_dist_version = get_local_dist_version() + pydist_dist_version = get_pypi_dist_version() + + print("Local version: {}, PyPI version: {}".format(local_dist_version, pydist_dist_version)) + + return local_dist_version > pydist_dist_version + + +def deploy_dist(): + print("Deploying to PyPI...") + subprocess.run(["twine", "upload", "dist/*"], stdout=subprocess.PIPE, check=True) + + +def main(): + if pypi_dist_is_outdated(): + deploy_dist() + else: + print("PyPI version up to date, no need to deploy") + + +if __name__ == "__main__": + main() diff --git a/ci/set_repo.sh b/ci/set_repo.sh new file mode 100755 index 0000000..042055c --- /dev/null +++ b/ci/set_repo.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +# sets up the repo incl submodules with specified version as $1 +set -o errexit # Exit if command failed. + +if [ -z $2 ] || [ -z $ESP_DOCS_PATH ] || [ -z $1 ] ; then + echo "Mandatory variables undefined" + exit 1; +fi; + +echo "Checking out repo version $1" +cd $2 +# Cleans out the untracked files in the repo, so the next "git checkout" doesn't fail +git clean -f +git checkout $1 +# Removes the esp-docs submodule, so the next submodule update doesn't fail +rm -rf $2/docs/esp-docs +git submodule update --init --recursive + diff --git a/ci/setup_python.sh b/ci/setup_python.sh new file mode 100644 index 0000000..2d68308 --- /dev/null +++ b/ci/setup_python.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +if [ -z ${PYTHON_VER+x} ]; then + # Use this version of the Python interpreter if it was not defined before. + # 3.4.8 is the default python3 interpreter in esp32-ci-env + # Jobs which doesn't support this version should define PYTHON_VER themselves + PYTHON_VER=3.4.8 +fi + +if [ -f /opt/pyenv/activate ]; +then + source /opt/pyenv/activate + pyenv global $PYTHON_VER || { + echo 'Python' $PYTHON_VER 'is not installed.' + INSTALLED_PY_VERS=$(pyenv versions --bare) + + while [ ${#PYTHON_VER} -gt 0 ] + do + echo 'Tring to locate a match for' $PYTHON_VER + + for ver in ${INSTALLED_PY_VERS[@]} + do + if [[ $ver == $PYTHON_VER* ]]; + then + pyenv global $ver + break 2 + fi + done + + # Removing last character and trying to find some match. + # For example, if 3.4.8 was selected but isn't installed then it will try to + # find some other installed 3.4.X version, and then some 3.X.X version. + PYTHON_VER=${PYTHON_VER: : -1} + done + } + python --version || { + echo 'No matching Python interpreter is found!' + exit 1 + } +elif command -v python -V 1>/dev/null 2>&1; +then + python --version + echo 'No /opt/pyenv/activate exists and Python from path is used.' +else + echo 'No /opt/pyenv/activate exists and no Python interpreter is found!' + exit 1 +fi diff --git a/ci/utils.sh b/ci/utils.sh new file mode 100644 index 0000000..393aca7 --- /dev/null +++ b/ci/utils.sh @@ -0,0 +1,18 @@ +# Modified from https://gitlab.com/gitlab-org/gitlab/-/blob/master/scripts/utils.sh + +function add_ssh_keys() { + local key_string="${1}" + mkdir -p ~/.ssh + chmod 700 ~/.ssh + echo -n "${key_string}" >~/.ssh/id_rsa_base64 + base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 >~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa +} + +function add_doc_server_ssh_keys() { + local key_string="${1}" + local server_url="${2}" + local server_user="${3}" + add_ssh_keys "${key_string}" + echo -e "Host ${server_url}\n\tStrictHostKeyChecking no\n\tUser ${server_user}\n" >>~/.ssh/config +} diff --git a/test/en/conf.py b/test/en/conf.py index c2e02b5..64ec3e1 100644 --- a/test/en/conf.py +++ b/test/en/conf.py @@ -5,13 +5,8 @@ # Uses ../conf_common.py for most non-language-specific settings. # Importing conf_common adds all the non-language-specific # parts to this conf module -try: - from conf_common import * # noqa: F403,F401 -except ImportError: - import os - import sys - sys.path.insert(0, os.path.abspath('../..')) - from conf_common import * # noqa: F403,F401 + +from esp_docs.conf_docs import * # noqa: F403,F401 # General information about the project. project = u'ESP-IDF Programming Guide' @@ -27,7 +22,12 @@ latex_logo = None html_static_path = [] -conditional_include_dict = {'esp32':['esp32_page.rst'], - 'esp32s2':['esp32s2_page.rst'], - 'SOC_BT_SUPPORTED':['bt_page.rst'], +conditional_include_dict = {'esp32': ['esp32_page.rst'], + 'esp32s2': ['esp32s2_page.rst'], + 'SOC_BT_SUPPORTED': ['bt_page.rst'], } + +extensions += ['esp_docs.esp_extensions.dummy_build_system'] + +languages = ['en'] +idf_targets = ['esp32', 'esp32s2'] diff --git a/test/test_docs.py b/test/test_docs.py index bea1e1a..0a81841 100755 --- a/test/test_docs.py +++ b/test/test_docs.py @@ -2,7 +2,6 @@ import os import subprocess -import sys import unittest CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -14,7 +13,6 @@ class DocBuilder(): - build_docs_py_path = os.path.join(CURRENT_DIR, '..', 'build_docs.py') def __init__(self, src_dir, build_dir, target, language): self.language = language @@ -24,7 +22,7 @@ def __init__(self, src_dir, build_dir, target, language): self.html_out_dir = os.path.join(CURRENT_DIR, build_dir, language, target, 'html') def build(self, opt_args=[]): - args = [sys.executable, self.build_docs_py_path, '-b', self.build_dir, '-s', self.src_dir, '-t', self.target, '-l', self.language] + args = ['build-docs', '-b', self.build_dir, '-s', self.src_dir, '-t', self.target, '-l', self.language] args.extend(opt_args) return subprocess.call(args) @@ -33,7 +31,7 @@ class TestDocs(unittest.TestCase): @classmethod def setUpClass(cls): - cls.builder = DocBuilder('test', '_build/test_docs', 'esp32s2', 'en') + cls.builder = DocBuilder('.', '_build/test_docs', 'esp32s2', 'en') cls.build_ret_flag = cls.builder.build() def setUp(self): @@ -81,7 +79,7 @@ def test_link_roles(self): class TestBuildSubset(unittest.TestCase): def test_build_subset(self): - builder = DocBuilder('test', '_build/test_build_subset', 'esp32', 'en') + builder = DocBuilder('.', '_build/test_build_subset', 'esp32', 'en') docs_to_build = 'esp32_page.rst' diff --git a/test/test_sphinx_idf_extensions.py b/test/test_esp_extensions.py similarity index 70% rename from test/test_sphinx_idf_extensions.py rename to test/test_esp_extensions.py index a87fee8..bc3fdc4 100755 --- a/test/test_sphinx_idf_extensions.py +++ b/test/test_esp_extensions.py @@ -1,26 +1,16 @@ #!/usr/bin/env python3 -import os -import sys import unittest -from tempfile import TemporaryDirectory from unittest.mock import MagicMock from sphinx.util import tags - -try: - from idf_extensions import exclude_docs -except ImportError: - sys.path.append('..') - from idf_extensions import exclude_docs - -from idf_extensions import format_idf_target, gen_idf_tools_links, link_roles +from esp_docs.esp_extensions import exclude_docs, format_esp_target class TestFormatIdfTarget(unittest.TestCase): def setUp(self): - self.str_sub = format_idf_target.StringSubstituter() + self.str_sub = format_esp_target.StringSubstituter() config = MagicMock() config.idf_target = 'esp32' @@ -74,7 +64,7 @@ class TestExclude(unittest.TestCase): def setUp(self): self.app = MagicMock() self.app.tags = tags.Tags() - self.app.config.conditional_include_dict = {'esp32':['esp32.rst', 'bt.rst'], 'esp32s2':['esp32s2.rst']} + self.app.config.conditional_include_dict = {'esp32': ['esp32.rst', 'bt.rst'], 'esp32s2': ['esp32s2.rst']} self.app.config.docs_to_build = None self.app.config.exclude_patterns = [] @@ -87,31 +77,5 @@ def test_update_exclude_pattern(self): self.assertFalse(docs_to_build & set(self.app.config.exclude_patterns)) -class TestGenIDFToolLinks(unittest.TestCase): - def setUp(self): - self.app = MagicMock() - self.app.config.build_dir = '_build' - self.app.config.idf_path = os.environ['IDF_PATH'] - - def test_gen_idf_tool_links(self): - - with TemporaryDirectory() as temp_dir: - self.app.config.build_dir = temp_dir - gen_idf_tools_links.generate_idf_tools_links(self.app, None) - self.assertTrue(os.path.isfile(os.path.join(self.app.config.build_dir, 'inc', 'idf-tools-inc.rst'))) - - -class TestLinkRoles(unittest.TestCase): - def test_get_submodules(self): - submod_dict = link_roles.get_submodules() - - # Test a known submodule to see if it's in the dict - test_submod_name = 'components/asio/asio' - self.assertIn(test_submod_name, submod_dict) - self.assertIsNotNone(submod_dict[test_submod_name].url) - self.assertIsNotNone(submod_dict[test_submod_name].rev) - self.assertIsNotNone(submod_dict[test_submod_name].url) - - if __name__ == '__main__': unittest.main()