Add CI for esp-docs

Adds basic CI for esp-docs. Includes:
 * Linting with Flake8
 * Building simple examples
 * Building esp-idf, esp-at and esp-adf docs

.flake8

@@ -0,0 +1,7 @@
+[flake8]
+exclude = citools/
+max-line-length = 160
+per-file-ignores =
+    # conf.py files use star imports to setup config variables
+    examples/*/conf*.py: F405
+    test/*/conf*py: F405

.gitlab-ci.yml

@@ -0,0 +1,224 @@
+stages:
+  - check
+  - test
+  - build
+  - test_deploy
+  - deploy
+
+variables:
+  # Versioned esp-idf-doc env image to use for all document building jobs
+  ESP_DOCS_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env:v4.4-1-v1"
+  PYTHON_VER: 3.6.10
+  IDF_REPO: ${GITLAB_SSH_SERVER}/idf/esp-idf.git
+  ESP_DOCS_PATH: "$CI_PROJECT_DIR"
+  IDF_PATH: $CI_PROJECT_DIR/esp-idf
+  IDF_REPO: ${GITLAB_SSH_SERVER}/idf/esp-idf.git
+  AT_PATH: $CI_PROJECT_DIR/esp-at
+  AT_REPO: ${GITLAB_SSH_SERVER}/application/esp-at.git
+  ADF_PATH: $CI_PROJECT_DIR/esp-adf-internal
+  ADF_REPO: ${GITLAB_SSH_SERVER}/adf/esp-adf-internal.git
+
+
+.before_script_minimal:
+  before_script:
+    - source ci/setup_python.sh
+    - source ci/utils.sh
+
+.before_script_prepare_build:
+  before_script:
+    # Use CI Tools
+    - curl -sSL ${CIT_LOADER_URL} | sh
+    - source citools/import_functions
+    - source ci/setup_python.sh
+    - source ci/utils.sh
+    # Purge any old esp-docs versions
+    - pip3 uninstall -y esp-docs
+    - pip3 install .
+
+check_setup:
+  stage: check
+  image: $ESP_DOCS_ENV_IMAGE
+  extends:
+    - .before_script_minimal
+  script:
+    - pip3 install .
+
+check_python_style:
+  stage: check
+  image: $ESP_DOCS_ENV_IMAGE
+  extends:
+    - .before_script_minimal
+  script:
+    - python -m flake8 --config=$ESP_DOCS_PATH/.flake8 $ESP_DOCS_PATH
+
+
+test_extensions_ut:
+  stage: test
+  image: $ESP_DOCS_ENV_IMAGE
+  extends:
+    - .before_script_prepare_build
+  script:
+    - cd test
+    - python test_docs.py
+    - python test_esp_extensions.py
+
+
+.build_template:
+  stage: build
+  image: $ESP_DOCS_ENV_IMAGE
+  extends:
+    - .before_script_prepare_build
+  artifacts:
+    when: always
+    paths:
+      - $DOCS_DIR/_build/*/*/html/*
+      - $DOCS_DIR/_build/*/*/*.txt
+    expire_in: 4 days
+
+build_example_basic:
+  extends:
+    - .build_template
+  variables:
+    DOCS_DIR: $CI_PROJECT_DIR/examples/basic
+  script:
+    - cd "$DOCS_DIR"
+    - ./build_example.sh
+
+build_example_doxygen:
+  extends:
+    - .build_template
+  variables:
+    DOCS_DIR: $CI_PROJECT_DIR/examples/doxygen
+  script:
+    - cd $DOCS_DIR
+    - ./build_example.sh
+
+build_idf_docs:
+  extends:
+    - .build_template
+  variables:
+    DOCS_DIR: $CI_PROJECT_DIR/esp-idf/docs
+    DOCS_FAST_BUILD: 'yes'
+  script:
+      # add gitlab ssh key
+    - cit_add_ssh_key "${GITLAB_KEY_PEM}"
+    - git clone "${IDF_REPO}"
+    - cd $IDF_PATH && tools/idf_tools.py --non-interactive install && eval "$(tools/idf_tools.py --non-interactive export)"
+        # build with IDF v.X (test branch for now)
+    - $ESP_DOCS_PATH/ci/set_repo.sh feature/esp_docs $IDF_PATH
+    - cd $DOCS_DIR
+    - build-docs -l $DOCLANG -t $DOCTGT -d doxygen
+  parallel:
+    matrix:
+      - DOCLANG: ["en", "zh_CN"]
+        DOCTGT: ["esp32", "esp32s2", "esp32c3"]
+
+build_at_docs:
+  extends:
+    - .build_template
+  variables:
+    DOCS_DIR: $CI_PROJECT_DIR/esp-at/docs
+  script:
+      # add gitlab ssh key
+    - cit_add_ssh_key "${GITLAB_KEY_PEM}"
+    - git clone "${AT_REPO}"
+    - $ESP_DOCS_PATH/ci/set_repo.sh test/esp_docs $AT_PATH
+    - cd $DOCS_DIR
+    - build-docs -l $DOCLANG -t $DOCTGT
+  parallel:
+    matrix:
+      - DOCLANG: ["en", "zh_CN"]
+        DOCTGT: ["esp8266", "esp32", "esp32s2", "esp32c3"]
+
+build_adf_docs:
+  extends:
+    - .build_template
+  variables:
+    DOCS_DIR: $CI_PROJECT_DIR/esp-adf-internal/docs
+  script:
+      # add gitlab ssh key
+    - cit_add_ssh_key "${GITLAB_KEY_PEM}"
+    - git clone "${ADF_REPO}"
+    - $ESP_DOCS_PATH/ci/set_repo.sh test/esp_docs $ADF_PATH
+    - cd $DOCS_DIR
+    - build-docs -l $DOCLANG -t $DOCTGT
+  parallel:
+    matrix:
+      - DOCLANG: ["en", "zh_CN"]
+        DOCTGT: ["esp32", "esp32s2"]
+
+build_package:
+  stage: build
+  image: $ESP_DOCS_ENV_IMAGE
+  extends:
+   - .before_script_minimal
+  tags:
+    - build
+  dependencies: []
+  artifacts:
+    when: always
+    paths:
+      - dist/*
+  script:
+    - python -m pip install build
+    - python -m build
+
+.deploy_docs_template:
+  stage: test_deploy
+  image: $ESP_DOCS_ENV_IMAGE
+  extends:
+    - .before_script_prepare_build
+  variables:
+    DOCS_DEPLOY_PRIVATEKEY: "$DOCS_DEPLOY_KEY"
+    DOCS_DEPLOY_SERVER: "$DOCS_SERVER"
+    DOCS_DEPLOY_SERVER_USER: "$DOCS_SERVER_USER"
+    DOCS_DEPLOY_PATH: "$DOCS_PATH"
+    DOCS_DEPLOY_URL_BASE: "https://$CI_DOCKER_REGISTRY/docs/esp-idf"
+  dependencies: []
+  script:
+    - add_doc_server_ssh_keys $DOCS_DEPLOY_PRIVATEKEY $DOCS_DEPLOY_SERVER $DOCS_DEPLOY_SERVER_USER
+    - export GIT_VER=$(git describe --always)
+    - deploy-docs
+
+deploy_docs_idf:
+  extends:
+    - .deploy_docs_template
+  dependencies:  # set dependencies to null to avoid missing artifacts issue
+  variables:
+    DOCS_BUILD_DIR: "${CI_PROJECT_DIR}/esp-idf/docs/_build/"
+    TYPE: "preview"
+  needs:
+    - build_idf_docs
+
+deploy_docs_at:
+  extends:
+    - .deploy_docs_template
+  dependencies:  # set dependencies to null to avoid missing artifacts issue
+  variables:
+    DOCS_BUILD_DIR: "$CI_PROJECT_DIR/esp-at/docs/_build/"
+    TYPE: "preview"
+  needs:
+    - build_at_docs
+
+deploy_docs_adf:
+  extends:
+    - .deploy_docs_template
+  dependencies:  # set dependencies to null to avoid missing artifacts issue
+  variables:
+    DOCS_BUILD_DIR: "$CI_PROJECT_DIR/esp-adf-internal/docs/_build/"
+    TYPE: "preview"
+  needs:
+    - build_adf_docs
+
+deploy_package:
+  stage: deploy
+  image: $ESP_DOCS_ENV_IMAGE
+  tags:
+    - build
+  only:
+    - master
+  when: on_success
+  dependencies:
+    - build_package
+  script:
+   - python $ESP_DOCS_PATH/ci/deploy_dist.py

ci/deploy_package.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+# coding=utf-8
+#
+# Deploy esp_docs package to PyPI
+#
+# Copyright 2021 Espressif Systems (Shanghai) PTE LTD
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import subprocess
+import os
+import json
+import urllib.request
+from pathlib import Path
+from distutils.version import StrictVersion
+
+PROJECT_DIR = Path(os.environ["ESP_DOCS_PATH"])
+
+
+def get_pypi_dist_version():
+    url = "https://pypi.python.org/pypi/esp_docs/json"
+
+    with urllib.request.urlopen(url, timeout=30) as conn:
+        data = json.loads(conn.read().decode("utf-8"))
+
+    versions = list(data["releases"].keys())
+
+    versions.sort(key=StrictVersion)
+    return StrictVersion(versions[-1])
+
+
+def get_local_dist_version():
+    setup_py_path = PROJECT_DIR / "setup.py"
+    ret = subprocess.run(["python3", str(setup_py_path), "--version"], stdout=subprocess.PIPE, check=True, timeout=100)
+
+    return StrictVersion(ret.stdout.decode("utf-8"))
+
+
+def pypi_dist_is_outdated():
+    local_dist_version = get_local_dist_version()
+    pydist_dist_version = get_pypi_dist_version()
+
+    print("Local version: {}, PyPI version: {}".format(local_dist_version, pydist_dist_version))
+
+    return local_dist_version > pydist_dist_version
+
+
+def deploy_dist():
+    print("Deploying to PyPI...")
+    subprocess.run(["twine", "upload", "dist/*"], stdout=subprocess.PIPE, check=True)
+
+
+def main():
+    if pypi_dist_is_outdated():
+        deploy_dist()
+    else:
+        print("PyPI version up to date, no need to deploy")
+
+
+if __name__ == "__main__":
+    main()

ci/set_repo.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+# sets up the repo incl submodules with specified version as $1
+set -o errexit # Exit if command failed.
+
+if [ -z $2 ] || [ -z $ESP_DOCS_PATH ] || [ -z $1 ] ; then
+    echo "Mandatory variables undefined"
+    exit 1;
+fi;
+
+echo "Checking out repo version $1"
+cd $2
+# Cleans out the untracked files in the repo, so the next "git checkout" doesn't fail
+git clean -f
+git checkout $1
+# Removes the esp-docs submodule, so the next submodule update doesn't fail
+rm -rf $2/docs/esp-docs
+git submodule update --init --recursive
+

ci/setup_python.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+if [ -z ${PYTHON_VER+x} ]; then
+    # Use this version of the Python interpreter if it was not defined before.
+    # 3.4.8 is the default python3 interpreter in esp32-ci-env
+    # Jobs which doesn't support this version should define PYTHON_VER themselves
+    PYTHON_VER=3.4.8
+fi
+
+if [ -f /opt/pyenv/activate ];
+then
+    source /opt/pyenv/activate
+    pyenv global $PYTHON_VER || {
+                                    echo 'Python' $PYTHON_VER 'is not installed.'
+                                    INSTALLED_PY_VERS=$(pyenv versions --bare)
+
+                                    while [ ${#PYTHON_VER} -gt 0 ]
+                                    do
+                                        echo 'Tring to locate a match for' $PYTHON_VER
+
+                                        for ver in ${INSTALLED_PY_VERS[@]}
+                                        do
+                                            if [[ $ver == $PYTHON_VER* ]];
+                                            then
+                                                pyenv global $ver
+                                                break 2
+                                            fi
+                                        done
+
+                                        # Removing last character and trying to find some match.
+                                        # For example, if 3.4.8 was selected but isn't installed then it will try to
+                                        # find some other installed 3.4.X version, and then some 3.X.X version.
+                                        PYTHON_VER=${PYTHON_VER: : -1}
+                                    done
+                                }
+    python --version || {
+                            echo 'No matching Python interpreter is found!'
+                            exit 1
+                        }
+elif command -v python -V 1>/dev/null 2>&1;
+then
+    python --version
+    echo 'No /opt/pyenv/activate exists and Python from path is used.'
+else
+    echo 'No /opt/pyenv/activate exists and no Python interpreter is found!'
+    exit 1
+fi

ci/utils.sh

@@ -0,0 +1,18 @@
+# Modified from https://gitlab.com/gitlab-org/gitlab/-/blob/master/scripts/utils.sh
+
+function add_ssh_keys() {
+  local key_string="${1}"
+  mkdir -p ~/.ssh
+  chmod 700 ~/.ssh
+  echo -n "${key_string}" >~/.ssh/id_rsa_base64
+  base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 >~/.ssh/id_rsa
+  chmod 600 ~/.ssh/id_rsa
+}
+
+function add_doc_server_ssh_keys() {
+  local key_string="${1}"
+  local server_url="${2}"
+  local server_user="${3}"
+  add_ssh_keys "${key_string}"
+  echo -e "Host ${server_url}\n\tStrictHostKeyChecking no\n\tUser ${server_user}\n" >>~/.ssh/config
+}