Merge branch 'master' into feature/cpu_multinomial

.github/workflows/build_doc.yml

@@ -4,7 +4,6 @@ on:
 
 env:
   DOXY_VER: '1.9.6'
-  DOXYREST_VER: '2.1.3'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
@@ -40,19 +39,18 @@ jobs:
           python3 -m pip install -r docs/requirements.txt
           (cd docs/openvino_sphinx_theme && python3 setup.py install)
 
-      - name: Download and install doxygen && doxyrest
+      - name: Download and install doxygen
         run: |
-          # install doxyrest
-          wget https://github.com/vovkos/doxyrest/releases/download/doxyrest-$DOXYREST_VER/doxyrest-$DOXYREST_VER-linux-amd64.tar.xz
-          tar -xf doxyrest-$DOXYREST_VER-linux-amd64.tar.xz
-          echo "$(pwd)/doxyrest-$DOXYREST_VER-linux-amd64/bin/" >> $GITHUB_PATH
           # install doxygen
           wget https://www.doxygen.nl/files/doxygen-$DOXY_VER.linux.bin.tar.gz
           tar -xzf doxygen-$DOXY_VER.linux.bin.tar.gz
           echo "$(pwd)/doxygen-$DOXY_VER/bin/" >> $GITHUB_PATH
 
-      - name: CMake configure
-        run: cmake -DENABLE_DOCS=ON -B build
+      - name: Build docs
+        run: |
+          rm -rf build && mkdir build && cd build
+          cmake .. -DCMAKE_BUILD_TYPE=Release -DENABLE_DOCS=ON -DENABLE_CPP_API=ON
+          cmake --build . --target sphinx_docs
 
       - name: Cache documentation
         id: cache_sphinx_docs
@@ -61,13 +59,6 @@ jobs:
           path: build/docs/_build/.doctrees
           key: sphinx-docs-cache
 
-      - name: Get number of CPU cores
-        uses: SimenB/github-actions-cpu-cores@v2
-        id: cpu-cores
-
-      - name: Build docs
-        run: cmake --build build --target sphinx_docs --parallel ${{ steps.cpu-cores.outputs.count }}
-
       - name: Archive docs HTML
         run: (cd build/docs && zip -r openvino_docs_html.zip _build)
 
@@ -97,14 +88,13 @@ jobs:
       - name: Run Pytest
         run: |
           pytest --doxygen="./build/docs/doxygen.log" \
-                 --include_pot \
                  --sphinx="./build/docs/sphinx.log" \
-                 --suppress-warnings="./docs/suppress_warnings.txt" \
+                 --suppress-warnings="./docs/scripts/tests/suppress_warnings.txt" \
                  --confcutdir="./docs/scripts/tests/" \
                  --html="./build/docs/_artifacts/doc-generation.html" \
                  --doxygen-strip="$(pwd)" \
-                 --sphinx-strip="$(pwd)/build/docs/rst" \
-                 --doxygen-xfail="./docs/doxygen-xfail.txt" \
+                 --sphinx-strip="$(pwd)/build/docs/sphinx_source" \
+                 --xfail="./docs/scripts/tests/xfail.txt" \
                  --self-contained-html ./docs/scripts/tests/test_docs.py
 
       - name: 'Upload test results'

docs/CMakeLists.txt

@@ -4,187 +4,122 @@
 
 add_subdirectory(snippets)
 
-set(LINKCHECKER_PY "" CACHE FILEPATH "Path to linkchecker.py for documentation check dir.")
-set(ENABLE_OPENVINO_NOTEBOOKS OFF CACHE BOOL "Build with openvino notebooks")
-set(OMZ_DOCS_DIR "" CACHE PATH "Path to open_model_zoo documentation dir.")
-set(OTE_DOCS_DIR "" CACHE PATH "Path to training_extensions documentation dir.")
-set(WORKBENCH_DOCS_DIR "" CACHE PATH "Path to workbench documentation dir.")
+set(ENABLE_CPP_API OFF CACHE BOOL "Build with C/C++ API.")
+set(ENABLE_PYTHON_API OFF CACHE BOOL "Build with Python API.")
+set(ENABLE_NOTEBOOKS OFF CACHE BOOL "Build with openvino notebooks.")
+set(ENABLE_OMZ OFF CACHE BOOL "Build with open_model_zoo.")
+set(ENABLE_OVMS OFF CACHE BOOL "Build with ovms.")
 set(OVMS_DOCS_DIR "" CACHE PATH "Path to model server documentation dir.")
-set(GRAPH_CSV_DIR "" CACHE PATH "Path to the folder containing csv data for rendering graphs.")
+
+if(${ENABLE_OVMS} AND (OVMS_DOCS_DIR STREQUAL ""))
+    message( FATAL_ERROR "You want to build OVMS, but OVMS_DOCS_DIR variable is empty." )
+endif()
+
 
 function(build_docs)
     find_package(Doxygen REQUIRED dot)
     find_package(LATEX REQUIRED)
 
-    find_program(DOXYREST_EXECUTABLE NAMES doxyrest)
-    if (NOT DOXYREST_EXECUTABLE)
-        message(FATAL_ERROR "No doxyrest found. Documentation output is not available")
-    endif()
-
     set(DOCS_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}")
     set(DOCS_SOURCE_DIR "${OpenVINO_SOURCE_DIR}/docs")
     set(SCRIPTS_DIR "${DOCS_SOURCE_DIR}/scripts")
 
-    # markdown docs
-    set(MARKDOWN_INPUT "${DOCS_BUILD_DIR}")
-
     # Preprocessing scripts
-    set(DOXY_MD_FILTER "${SCRIPTS_DIR}/doxy_md_filter.py")
-    set(PREPARE_XML_SCRIPT "${SCRIPTS_DIR}/prepare_xml.py")
     set(REMOVE_XML_SCRIPT "${SCRIPTS_DIR}/remove_xml.py")
+    set(FILE_HELPER_SCRIPT "${SCRIPTS_DIR}/filehelper.py")
     set(COPY_IMAGES_SCRIPT "${SCRIPTS_DIR}/copy_images.py")
-    set(DOC_TEST_DIR "${SCRIPTS_DIR}/tests")
     set(DOXYGEN_MAPPING_SCRIPT "${SCRIPTS_DIR}/create_mapping.py")
+    set(BREATHE_APIDOC_SCRIPT "${SCRIPTS_DIR}/apidoc.py")
 
-    # out dirs
-    set(XML_OUTPUT "${DOCS_BUILD_DIR}/xml")
-    set(RST_OUTPUT "${DOCS_BUILD_DIR}/rst")
+    # Doxygen/Sphinx setup
+    set(DOXYGEN_XML_OUTPUT "${DOCS_BUILD_DIR}/xml")
+    set(SPHINX_SETUP_DIR "${DOCS_SOURCE_DIR}/sphinx_setup")
+    set(SPHINX_SOURCE_DIR "${DOCS_BUILD_DIR}/sphinx_source")
     set(SPHINX_OUTPUT "${DOCS_BUILD_DIR}/_build")
 
-    # Sphinx folders, doxyrest templates and config
-    set(SPHINX_CONF_IN "${DOCS_SOURCE_DIR}/conf.py")
-    set(SPHINX_TEMPLATES_IN "${DOCS_SOURCE_DIR}/_templates")
-    set(SPHINX_TEMPLATES_OUT "${RST_OUTPUT}/_templates")
-    set(SPHINX_CONF_OUT "${RST_OUTPUT}/conf.py")
-    set(SPHINX_STATIC_IN "${DOCS_SOURCE_DIR}/_static")
-    set(SPHINX_STATIC_OUT "${RST_OUTPUT}/_static")
-    set(SPHINX_INDEX_IN "${DOCS_SOURCE_DIR}/index.rst")
-    set(SPHINX_INDEX_OUT "${RST_OUTPUT}/index.rst")
-    set(API_DOCS_IN "${DOCS_SOURCE_DIR}/api")
-    set(API_DOCS_OUT "${RST_OUTPUT}/api")
-    set(DOXYREST_IN "${DOCS_SOURCE_DIR}/doxyrest")
-    set(DOXYREST_OUT "${DOCS_BUILD_DIR}/doxyrest")
-    set(DOXYREST_SPHINX_IN "${DOCS_SOURCE_DIR}/doxyrest-sphinx")
-    set(DOXYREST_SPHINX_OUT "${RST_OUTPUT}/doxyrest-sphinx")
-    set(DOXYREST_CONFIG_IN "${DOCS_SOURCE_DIR}/doxyrest-config.lua")
-    set(DOXYREST_CONFIG_OUT "${DOCS_BUILD_DIR}/doxyrest-config.lua")
-    configure_file(${DOXYREST_CONFIG_IN} ${DOXYREST_CONFIG_OUT} @ONLY)
-    configure_file(${SPHINX_CONF_IN} ${SPHINX_CONF_OUT} @ONLY)
-
-    # Doxygen config
-    set(DOXYFILE_SOURCE "${DOCS_SOURCE_DIR}/Doxyfile.config")
-    set(DOXYFILE_BUILD "${DOCS_BUILD_DIR}/Doxyfile.config")
-    configure_file(${DOXYFILE_SOURCE} ${DOXYFILE_BUILD} @ONLY)
-
-    list(APPEND commands COMMAND ${Python3_EXECUTABLE} ${DOXY_MD_FILTER}
+    list(APPEND commands COMMAND ${CMAKE_COMMAND} -E cmake_echo_color --green "STARTED preprocessing OpenVINO articles")
+    list(APPEND commands COMMAND ${Python3_EXECUTABLE} ${FILE_HELPER_SCRIPT}
+        --filetype=rst
         --input_dir=${OpenVINO_SOURCE_DIR}
-        --output_dir=${DOCS_BUILD_DIR}/openvino
-        --exclude_dir=${DOCS_BUILD_DIR})
-
-    # include additional repositories
+        --output_dir=${SPHINX_SOURCE_DIR}
+        --exclude_dir=${SPHINX_SOURCE_DIR})
+    list(APPEND commands COMMAND ${CMAKE_COMMAND} -E cmake_echo_color --green "FINISHED preprocessing OpenVINO articles")
+
+    if(${ENABLE_CPP_API})
+        # Doxygen config
+        set(DOXYFILE_SOURCE "${DOCS_SOURCE_DIR}/Doxyfile.config")
+        set(DOXYFILE_BUILD "${DOCS_BUILD_DIR}/Doxyfile.config")
+        configure_file(${DOXYFILE_SOURCE} ${DOXYFILE_BUILD} @ONLY)
+        list(APPEND commands COMMAND ${CMAKE_COMMAND} -E cmake_echo_color --green "STARTED preprocessing OpenVINO C/C++ API reference")
+        list(APPEND commands COMMAND ${Python3_EXECUTABLE} ${REMOVE_XML_SCRIPT} ${DOXYGEN_XML_OUTPUT})
+        list(APPEND commands COMMAND ${DOXYGEN_EXECUTABLE} ${DOXYFILE_BUILD})
+        list(APPEND post_commands COMMAND ${Python3_EXECUTABLE} ${BREATHE_APIDOC_SCRIPT} ${DOXYGEN_XML_OUTPUT} -o "${SPHINX_SOURCE_DIR}/api/c_cpp_api" -m -T -p openvino -g class,group,struct,union,enum)
+        list(APPEND commands COMMAND ${CMAKE_COMMAND} -E cmake_echo_color --green "FINISHED preprocessing OpenVINO C/C++ API reference")
+    endif()
 
-    # build with openvino notebooks
-    if(ENABLE_OPENVINO_NOTEBOOKS)
-        set(NBDOC_SCRIPT "${DOCS_SOURCE_DIR}/nbdoc/nbdoc.py")
-        list(PREPEND commands
-            COMMAND ${Python3_EXECUTABLE} "${NBDOC_SCRIPT}" "${DOCS_SOURCE_DIR}/notebooks" "${RST_OUTPUT}/notebooks"
-        )
+    if(${ENABLE_PYTHON_API})
+        list(APPEND commands COMMAND ${CMAKE_COMMAND} -E cmake_echo_color --green "STARTED preprocessing OpenVINO Python API")
+        list(APPEND commands COMMAND ${Python3_EXECUTABLE} -m pip install openvino)
+        list(APPEND commands COMMAND ${CMAKE_COMMAND} -E cmake_echo_color --green "FINISHED preprocessing OpenVINO Python API")
     endif()
 
-    if(GRAPH_CSV_DIR)
-        set(GRAPH_CSV_DIR_OUT "${RST_OUTPUT}/csv")
+    if(${ENABLE_NOTEBOOKS})
+        set(NBDOC_SCRIPT "${DOCS_SOURCE_DIR}/nbdoc/nbdoc.py")
+        list(APPEND commands COMMAND ${CMAKE_COMMAND} -E cmake_echo_color --green "STARTED preprocessing OpenVINO notebooks")
         list(APPEND commands
-            COMMAND ${CMAKE_COMMAND} -E copy_directory "${GRAPH_CSV_DIR}" "${GRAPH_CSV_DIR_OUT}"
+            COMMAND ${Python3_EXECUTABLE} "${NBDOC_SCRIPT}" "${DOCS_SOURCE_DIR}/notebooks" "${SPHINX_SOURCE_DIR}/notebooks"
         )
+        list(APPEND commands COMMAND ${CMAKE_COMMAND} -E cmake_echo_color --green "FINISHED preprocessing OpenVINO notebooks")
     endif()
 
-    list(APPEND commands
-        COMMAND ${CMAKE_COMMAND} -E copy ${API_DOCS_IN}/api_reference.rst ${API_DOCS_OUT}/api_reference.rst
-        COMMAND ${CMAKE_COMMAND} -E copy_directory ${API_DOCS_IN}/ie_python_api ${API_DOCS_OUT}/ie_python_api
-    )
+    if(${ENABLE_OVMS})
+        list(APPEND commands COMMAND ${CMAKE_COMMAND} -E cmake_echo_color --green "STARTED preprocessing OVMS")
+        list(APPEND commands COMMAND ${Python3_EXECUTABLE} ${FILE_HELPER_SCRIPT}
+        --filetype=md
+        --input_dir=${OVMS_DOCS_DIR}
+        --output_dir=${SPHINX_SOURCE_DIR}
+        --exclude_dir=${SPHINX_SOURCE_DIR})
+        list(APPEND commands COMMAND ${CMAKE_COMMAND} -E cmake_echo_color --green "FINISHED preprocessing OVMS")
+    endif()
 
-    # omz doc files
-    if(EXISTS "${OMZ_DOCS_DIR}")
-        get_filename_component(OMZ_DOCS_DIR "${OMZ_DOCS_DIR}" ABSOLUTE)
+    if(${ENABLE_OMZ})
+        list(APPEND commands COMMAND ${CMAKE_COMMAND} -E cmake_echo_color --green "STARTED preprocessing OMZ")
         list(APPEND commands
-        COMMAND ${Python3_EXECUTABLE} ${OMZ_DOCS_DIR}/ci/prepare-documentation.py ${CMAKE_BINARY_DIR}/open_model_zoo)
-        list(APPEND commands COMMAND ${Python3_EXECUTABLE} ${DOXY_MD_FILTER}
+            COMMAND ${Python3_EXECUTABLE} ${OpenVINO_SOURCE_DIR}/thirdparty/open_model_zoo/ci/prepare-documentation.py ${CMAKE_BINARY_DIR}/open_model_zoo)
+        list(APPEND commands COMMAND ${Python3_EXECUTABLE} ${FILE_HELPER_SCRIPT}
+            --filetype=md
             --input_dir=${CMAKE_BINARY_DIR}/open_model_zoo
-            --output_dir=${DOCS_BUILD_DIR}/open_model_zoo)
-    endif()
-
-    # workbench doc files
-    if(EXISTS "${WORKBENCH_DOCS_DIR}")
-        get_filename_component(WORKBENCH_DOCS_DIR "${WORKBENCH_DOCS_DIR}" ABSOLUTE)
-
-        list(APPEND commands COMMAND ${Python3_EXECUTABLE} ${DOXY_MD_FILTER}
-            --input_dir=${WORKBENCH_DOCS_DIR}
-            --output_dir=${DOCS_BUILD_DIR}/workbench)
+            --output_dir=${SPHINX_SOURCE_DIR}
+            --exclude_dir=${SPHINX_SOURCE_DIR})
+        list(APPEND commands COMMAND ${CMAKE_COMMAND} -E cmake_echo_color --green "FINISHED preprocessing OMZ")
     endif()
 
-    # ote doc files
-    if(EXISTS "${OTE_DOCS_DIR}")
-        get_filename_component(WORKBENCH_DOCS_DIR "${OTE_DOCS_DIR}" ABSOLUTE)
-
-        list(APPEND commands COMMAND ${Python3_EXECUTABLE} ${DOXY_MD_FILTER}
-            --input_dir=${OTE_DOCS_DIR}
-            --output_dir=${DOCS_BUILD_DIR}/ote)
-    endif()
-
-    # ovms doc files
-    if(EXISTS "${OVMS_DOCS_DIR}")
-        get_filename_component(OVMS_DOCS_DIR "${OVMS_DOCS_DIR}" ABSOLUTE)
-
-        list(APPEND commands COMMAND ${Python3_EXECUTABLE} ${DOXY_MD_FILTER}
-            --input_dir=${OVMS_DOCS_DIR}
-            --output_dir=${DOCS_BUILD_DIR}/ovms)
-    endif()
-
-    add_custom_target(preprocess_docs
-                     COMMENT "Preprocess documentation"
-                     VERBATIM)
-
     # Preprocess docs
-    add_custom_command(TARGET preprocess_docs
-                       POST_BUILD
-                       ${commands}
-                       WORKING_DIRECTORY ${DOCS_BUILD_DIR}
-                       COMMENT "Preprocess documentation"
-                       VERBATIM)
-
-    add_custom_target(doxygen_xml
-                      DEPENDS preprocess_docs
-                      COMMAND ${Python3_EXECUTABLE} ${REMOVE_XML_SCRIPT} ${XML_OUTPUT}
-                      COMMAND ${DOXYGEN_EXECUTABLE} ${DOXYFILE_BUILD}
+    add_custom_target(preprocess_docs
+                      COMMAND ${CMAKE_COMMAND} -E remove_directory ${SPHINX_SOURCE_DIR}
+                      ${commands}
                       WORKING_DIRECTORY ${DOCS_BUILD_DIR}
-                      COMMENT "Generate doxygen XML output"
                       VERBATIM)
 
-    # Post-process docs
-    add_custom_command(TARGET doxygen_xml
+    add_custom_command(TARGET preprocess_docs
                        POST_BUILD
-                       COMMAND ${Python3_EXECUTABLE} ${PREPARE_XML_SCRIPT} ${XML_OUTPUT}
-                       COMMAND ${CMAKE_COMMAND} -E copy_directory ${DOXYREST_IN} ${DOXYREST_OUT}
-                       COMMAND ${DOXYREST_EXECUTABLE} -c ${DOXYREST_CONFIG_OUT}
-                       COMMAND ${Python3_EXECUTABLE} ${COPY_IMAGES_SCRIPT} ${XML_OUTPUT} ${RST_OUTPUT}
-                       COMMAND ${Python3_EXECUTABLE} ${DOXYGEN_MAPPING_SCRIPT} ${XML_OUTPUT} ${DOCS_BUILD_DIR} ${OpenVINO_SOURCE_DIR}/../
-                       COMMAND ${CMAKE_COMMAND} -E copy ${SPHINX_INDEX_IN} ${SPHINX_INDEX_OUT}
-                       COMMAND ${CMAKE_COMMAND} -E copy_directory ${SPHINX_TEMPLATES_IN} ${SPHINX_TEMPLATES_OUT}
-                       COMMAND ${CMAKE_COMMAND} -E copy_directory ${DOXYREST_IN} ${DOXYREST_OUT}
-                       COMMAND ${CMAKE_COMMAND} -E copy_directory ${DOXYREST_SPHINX_IN} ${DOXYREST_SPHINX_OUT}
-                       COMMAND ${CMAKE_COMMAND} -E copy_directory ${SPHINX_STATIC_IN} ${SPHINX_STATIC_OUT}
-                       COMMENT "Prepare xml"
+                       COMMAND ${Python3_EXECUTABLE} ${COPY_IMAGES_SCRIPT} ${DOXYGEN_XML_OUTPUT} ${SPHINX_SOURCE_DIR}
+                       COMMAND ${Python3_EXECUTABLE} ${DOXYGEN_MAPPING_SCRIPT} ${DOXYGEN_XML_OUTPUT} ${DOCS_BUILD_DIR} ${OpenVINO_SOURCE_DIR}/../
+                       COMMAND ${CMAKE_COMMAND} -E copy_directory ${SPHINX_SETUP_DIR} ${SPHINX_SOURCE_DIR}
+                       ${post_commands}
                        VERBATIM)
 
+    # Build docs
     add_custom_target(sphinx_docs
-                      DEPENDS doxygen_xml
-                      COMMAND sphinx-build -q -j auto -w ${DOCS_BUILD_DIR}/sphinx.log -b html ${RST_OUTPUT} ${SPHINX_OUTPUT}
-                      WORKING_DIRECTORY ${RST_OUTPUT}
+                      DEPENDS preprocess_docs
+                      COMMAND ${CMAKE_COMMAND} -E cmake_echo_color --green "STARTED sphinx documentation build"
+                      COMMAND sphinx-build -j auto -w ${DOCS_BUILD_DIR}/sphinx.log -b html ${SPHINX_SOURCE_DIR} ${SPHINX_OUTPUT}
+                      COMMAND ${CMAKE_COMMAND} -E cmake_echo_color --green "FINISHED sphinx documentation build"
+                      WORKING_DIRECTORY ${SPHINX_SOURCE_DIR}
                       VERBATIM)
 
-    set_target_properties(doxygen_xml sphinx_docs
-                          PROPERTIES FOLDER docs)
+    set_target_properties(preprocess_docs sphinx_docs PROPERTIES FOLDER docs)
 
-
-    find_program(browser NAMES xdg-open)
-    if(browser)
-        add_custom_target(ie_docs_open
-                          COMMAND ${browser} "${SPHINX_OUTPUT}/index.html"
-                          DEPENDS sphinx_docs
-                          COMMENT "Open OpenVINO documentation"
-                          VERBATIM)
-        set_target_properties(ie_docs_open PROPERTIES FOLDER docs)
-    endif()
 endfunction()
 
 if(ENABLE_DOCS)

docs/Doxyfile.config

@@ -829,8 +829,7 @@ WARN_LOGFILE           = "@DOCS_BUILD_DIR@/doxygen.log"
 # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
 # Note: If this tag is empty the current directory is searched.
 
-INPUT                  = "@MARKDOWN_INPUT@" \
-                         "@OpenVINO_SOURCE_DIR@/src/frontends/common/include/" \
+INPUT                  = "@OpenVINO_SOURCE_DIR@/src/frontends/common/include/" \
                          "@OpenVINO_SOURCE_DIR@/src/common/conditional_compilation/include/" \
                          "@OpenVINO_SOURCE_DIR@/src/common/itt/include/" \
                          "@OpenVINO_SOURCE_DIR@/src/common/low_precision_transformations/include/" \