From 2076af0336dedb818d2e8ac3c0af905e84d7849b Mon Sep 17 00:00:00 2001 From: Kunal Tyagi Date: Sun, 25 Oct 2020 05:46:12 +0900 Subject: [PATCH 01/19] Remove deprecated use of set-env --- .github/workflows/pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 4af662b..756e9e4 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -43,7 +43,7 @@ jobs: # Add dist-package to path to enable apt installed python3-clang import - name: Add dist-packages to PYTHONPATH - run: echo "::set-env name=PYTHONPATH::${PYTHON_PATH}:/usr/lib/python3/dist-packages" + run: echo "PYTHONPATH=${PYTHON_PATH}:/usr/lib/python3/dist-packages" >> $GITHUB_ENV - name: Display PYTHONPATH run: python -c "import sys; print('\n'.join(sys.path))" From 24dc99f384c2f28e91ea67a58253597e1801374c Mon Sep 17 00:00:00 2001 From: Kunal Tyagi Date: Sun, 25 Oct 2020 20:52:41 +0900 Subject: [PATCH 02/19] Remove unneeded file --- bindings/python/tests/compile_commands.json | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 bindings/python/tests/compile_commands.json diff --git a/bindings/python/tests/compile_commands.json b/bindings/python/tests/compile_commands.json deleted file mode 100644 index 4eefaf4..0000000 --- a/bindings/python/tests/compile_commands.json +++ /dev/null @@ -1,7 +0,0 @@ -[ - { - "directory": "/home/divyanshu/Projects/active/pcl/bindings/python/tests", - "command": "/usr/bin/clang++ -std=c++14 /home/divyanshu/Projects/active/pcl/common/include/pcl/impl/point_types.hpp", - "file": "/home/divyanshu/Projects/active/pcl/common/include/pcl/impl/point_types.hpp" - } -] From 98495f0b588b1fca391f7b453c514e75c1bf8012 Mon Sep 17 00:00:00 2001 From: Kunal Tyagi Date: Sun, 25 Oct 2020 21:46:07 +0900 Subject: [PATCH 03/19] Changes to make it more intuitive to use Verbose output while writing files --- bindings/python/scripts/generate.py | 9 +++++++-- bindings/python/scripts/parse.py | 8 ++++++-- bindings/python/scripts/utils.py | 13 ++++++++++--- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/bindings/python/scripts/generate.py b/bindings/python/scripts/generate.py index 8ec3b85..1380860 100644 --- a/bindings/python/scripts/generate.py +++ b/bindings/python/scripts/generate.py @@ -1,6 +1,8 @@ +from typing import Any, List, Dict +import os + from context import scripts import scripts.utils as utils -from typing import Any, List, Dict class bind: @@ -427,12 +429,15 @@ def main(): for source in args.files: source = utils.get_realpath(path=source) lines_to_write = generate(module_name="pcl", source=source) + output_dir = utils.join_path(args.pybind11_output_path, "pybind11-gen") output_filepath = utils.get_output_path( source=source, - output_dir=utils.join_path(args.pybind11_output_path, "pybind11-gen"), + output_dir=output_dir, split_from="json", extension=".cpp", ) + out_rel_path = os.path.relpath(output_filepath, args.pybind11_output_path) + print(f"Producing ./{out_rel_path}") utils.write_to_file(filename=output_filepath, linelist=lines_to_write) diff --git a/bindings/python/scripts/parse.py b/bindings/python/scripts/parse.py index cf005ba..1e1806c 100644 --- a/bindings/python/scripts/parse.py +++ b/bindings/python/scripts/parse.py @@ -1,3 +1,4 @@ +import os import sys import clang.cindex as clang @@ -267,12 +268,15 @@ def main(): parsed_info = parse_file(source, args.compilation_database_path) # Output path for dumping the parsed info into a json file + output_dir=utils.join_path(args.json_output_path, "json") output_filepath = utils.get_output_path( source=source, - output_dir=utils.join_path(args.json_output_path, "json"), - split_from="pcl", + output_dir=output_dir, + split_from=args.project_root, extension=".json", ) + out_rel_path = os.path.relpath(output_filepath, args.json_output_path) + print(f"Producing ./{out_rel_path}") # Dump the parsed info at output path utils.dump_json(filepath=output_filepath, info=parsed_info) diff --git a/bindings/python/scripts/utils.py b/bindings/python/scripts/utils.py index f141077..e2358ff 100644 --- a/bindings/python/scripts/utils.py +++ b/bindings/python/scripts/utils.py @@ -35,7 +35,9 @@ def get_output_path(source, output_dir, split_from, extension): """ # split_path: contains the path after splitting. For split_path = pcl, contains the path as seen in the pcl directory - _, split_path = source.split(f"{split_from}{os.sep}", 1) + split_path = source.split(f"{split_from}{os.sep}", 1) + # handle the case where there's nothing to split + split_path = split_path[1] if len(split_path) == 2 else split_path[0] # relative_dir: contains the relative output path for the json file # source_filename: contains the source's file name @@ -93,9 +95,14 @@ def parse_arguments(script): ) parser.add_argument( "--json_output_path", - default=get_parent_directory(file=__file__), + default=os.getcwd(), help="Output path for generated json", ) + parser.add_argument( + "--project-root", + default=os.path.dirname(os.getcwd()), + help="Path to split to make output paths shorter", + ) parser.add_argument("files", nargs="+", help="The source files to parse") if script == "generate": @@ -103,7 +110,7 @@ def parse_arguments(script): parser.add_argument("files", nargs="+", help="JSON input") parser.add_argument( "--pybind11_output_path", - default=get_parent_directory(file=__file__), + default=os.getcwd(), help="Output path for generated cpp", ) From 387e9bf4d2926decc85a1474a9c73dc242fd39e6 Mon Sep 17 00:00:00 2001 From: Kunal Tyagi Date: Sun, 25 Oct 2020 21:47:18 +0900 Subject: [PATCH 04/19] Add a sample (simple?) CMake project --- bindings/python/tests/test_project/CMakeLists.txt | 10 ++++++++++ .../test_project/include/clang_bind_test/function.hpp | 5 +++++ bindings/python/tests/test_project/src/simple.cpp | 5 +++++ 3 files changed, 20 insertions(+) create mode 100644 bindings/python/tests/test_project/CMakeLists.txt create mode 100644 bindings/python/tests/test_project/include/clang_bind_test/function.hpp create mode 100644 bindings/python/tests/test_project/src/simple.cpp diff --git a/bindings/python/tests/test_project/CMakeLists.txt b/bindings/python/tests/test_project/CMakeLists.txt new file mode 100644 index 0000000..d943684 --- /dev/null +++ b/bindings/python/tests/test_project/CMakeLists.txt @@ -0,0 +1,10 @@ +cmake_minimum_required(VERSION 3.10) +project(clang_bind_test VERSION 0.0.1 LANGUAGES CXX) + +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +add_library(simple src/simple.cpp) +target_include_directories(simple PUBLIC + $ + $ +) diff --git a/bindings/python/tests/test_project/include/clang_bind_test/function.hpp b/bindings/python/tests/test_project/include/clang_bind_test/function.hpp new file mode 100644 index 0000000..67a4cb8 --- /dev/null +++ b/bindings/python/tests/test_project/include/clang_bind_test/function.hpp @@ -0,0 +1,5 @@ +#pragma once + +namespace simple { +int add(int a, int b); +} // namespace simple diff --git a/bindings/python/tests/test_project/src/simple.cpp b/bindings/python/tests/test_project/src/simple.cpp new file mode 100644 index 0000000..00cda0e --- /dev/null +++ b/bindings/python/tests/test_project/src/simple.cpp @@ -0,0 +1,5 @@ +#include + +namespace simple { +int add(int a, int b) { return a + b; } +} // namespace simple From b1bd3a65329fa679946d0ffaebeb45e9bfe7375e Mon Sep 17 00:00:00 2001 From: Kunal Tyagi Date: Sun, 25 Oct 2020 21:51:33 +0900 Subject: [PATCH 05/19] All hail Black --- bindings/python/scripts/parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/python/scripts/parse.py b/bindings/python/scripts/parse.py index 1e1806c..4205de6 100644 --- a/bindings/python/scripts/parse.py +++ b/bindings/python/scripts/parse.py @@ -268,7 +268,7 @@ def main(): parsed_info = parse_file(source, args.compilation_database_path) # Output path for dumping the parsed info into a json file - output_dir=utils.join_path(args.json_output_path, "json") + output_dir = utils.join_path(args.json_output_path, "json") output_filepath = utils.get_output_path( source=source, output_dir=output_dir, From 821a662066ed057f0e3d65043cd5756bd3f65f00 Mon Sep 17 00:00:00 2001 From: Kunal Tyagi Date: Mon, 2 Nov 2020 00:33:02 +0900 Subject: [PATCH 06/19] Add minimal files required for a nice interface and conan integration --- bindings/python/scripts/CMakeLists.txt.in | 52 ++ bindings/python/scripts/conanfile.txt | 5 + bindings/python/scripts/interface.py | 168 +++++ .../python/scripts/third_party/conan.cmake | 610 ++++++++++++++++++ 4 files changed, 835 insertions(+) create mode 100644 bindings/python/scripts/CMakeLists.txt.in create mode 100644 bindings/python/scripts/conanfile.txt create mode 100644 bindings/python/scripts/interface.py create mode 100644 bindings/python/scripts/third_party/conan.cmake diff --git a/bindings/python/scripts/CMakeLists.txt.in b/bindings/python/scripts/CMakeLists.txt.in new file mode 100644 index 0000000..c3cf1c0 --- /dev/null +++ b/bindings/python/scripts/CMakeLists.txt.in @@ -0,0 +1,52 @@ +cmake_minimum_required(VERSION 3.10) +project (${project.name}_pybind11) + +% if not cmake.no_conan: +include (conan.cmake) + +conan_cmake_run (CONANFILE conanfile.txt + BASIC_SETUP + CMAKE_TARGETS + BUILD missing + % if conan.keep_rpaths: + KEEP_RPATHS + % endif + % if conan.no_output_dirs: + NO_OUTPUT_DIRS + % endif + % if conan.arch: + ARCH ${conan.arch} + % endif + % if conan.build_type: + BUILD_TYPE ${conan.build_type} + % endif + % if conan.profile: + PROFILE ${conan.profile} + % endif + % if conan.profile_auto: + PROFILE_AUTO ${conan.profile_auto} + % endif +) +% endif + +# `find_package` can be replaced with `add_subdirectory` by a power user +# https://pybind11.readthedocs.io/en/stable/compiling.html#find-package-vs-add-subdirectory +find_package(pybind11 REQUIRED) + +set (CMAKE_CXX_STANDARD ${cmake.cpp_standard}) + +pybind11_add_module (${project.name} + % if pybind11.lib_type: + # for MODULE or SHARED + ${pybind11.lib_type} + % endif + % if pybind11.thin_lto: + THIN_LTO + % endif + % if pybind11.optimise_for_size: + OPT_SIZE + % endif + % for file in files: + file + % endfor +) diff --git a/bindings/python/scripts/conanfile.txt b/bindings/python/scripts/conanfile.txt new file mode 100644 index 0000000..34b60f4 --- /dev/null +++ b/bindings/python/scripts/conanfile.txt @@ -0,0 +1,5 @@ +[requires] +pybind11[>2.2.2] + +[generators] +cmake diff --git a/bindings/python/scripts/interface.py b/bindings/python/scripts/interface.py new file mode 100644 index 0000000..5af2f19 --- /dev/null +++ b/bindings/python/scripts/interface.py @@ -0,0 +1,168 @@ +#! /usr/bin/env python + +import argparse +from functools import lru_cache +import os + +from mako.template import Template + + +def get_args(parser): + args, _ = parser.parse_known_args() + return args + + +def list_by_type(directory): + from os.path import isfile, isdir, islink, join + + all_files = os.listdir(directory) + categorized = { + "file": [f for f in all_files if isfile(join(directory, f))], + "directory": [d for d in all_files if isdir(join(directory, d))], + "link": [l for l in all_files if islink(join(directory, l))], + } + return categorized + + +@lru_cache(maxsize=2) +def guess_build_dir(guess=os.getcwd()): + ls = list_by_type(guess) + if "compile_commands.json" in ls["file"]: + return guess + second_guess = os.path.join(guess, "build") + if second_guess in ls["directory"]: + return second_guess + return None + + +def guess_project_dir(): + guess = os.getcwd() + return guess + + +class CategorizedArgs: + def __init__(self): + self._internal_categories = ["cmake", "conan", "project", "pybind11"] + self._parse_group = {} + self._parser = None + self._generate() + self.args = self._get_args() + self.categorized_args = { + category: self._get_args(category) for category in self._internal_categories + } + + def _cmake(self, parser=argparse.ArgumentParser()): + parser.add_argument( + "--cpp-standard", + required=False, + default="14", + choices=["03", "11", "14", "17", "20"], + help="C++ Standard used by your project", + ) + parser.add_argument( + "--no-conan", + default=False, + action="store_true", + help="Use CMake's own package resolution instead of relying on conan", + ) + return parser + + def _conan(self, parser=argparse.ArgumentParser()): + parser.add_argument("--keep-rpaths", required=False, action="store_true") + parser.add_argument("--no-output-dirs", required=False, action="store_true") + parser.add_argument("--arch", required=False) + parser.add_argument("--build-type", required=False) + parser.add_argument("--profile", required=False) + parser.add_argument("--profile-auto", required=False) + return parser + + def _pybind11(self, parser=argparse.ArgumentParser()): + parser.add_argument("--lib-type", required=False) + parser.add_argument("--thin-lto", required=False, action="store_true") + parser.add_argument("--optimise-for-size", required=False, action="store_true") + return parser + + def _project(self, parser=argparse.ArgumentParser()): + parser.add_argument("--name", required=True) + return parser + + def _generate(self): + parser = argparse.ArgumentParser( + description="Generate bindings from C++", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "--compile-commands", + default=guess_build_dir(), + required=guess_build_dir() is None, + help="Path to compile_commands.json file or the directory containing it", + ) + parser.add_argument( + "--project-root", + default=guess_project_dir(), + help="Path to the project root to reduce name length of generated files", + ) + parser.add_argument( + "--out-dir", + default=guess_build_dir(), + required=guess_build_dir() is None, + help="Output directory for intermediate json files", + ) + parser.add_argument( + "--select-files", + nargs="*", + metavar=("CHOSEN_FILE", "OTHER_CHOSEN_FILES"), + default=None, + help="Select only certain files from the compile database", + ) + parser.add_argument( + "--ignore-files", + nargs="*", + metavar=("IGNORED_FILE", "OTHER_IGNORED_FILES"), + default=None, + help="Ignore certain files from the compile database", + ) + parser.add_argument( + "--language", + default="py", + choices=["py"], + help="Target language for the bindings", + ) + parser.add_argument( + "--use-latest-conan-cmake", + default=False, + action="store_true", + help="Use the latest release of conan-cmake instead of the bundled release (0.15.0)", + ) + + for category in self._internal_categories: + gp = parser.add_argument_group(title=f"arguments for {category}") + getattr(self, f"_{category}")(gp) + # also create categorized parser + self._parse_group[category] = getattr(self, f"_{category}")() + self._parser = parser + return self._parser + + def _get_args(self, category=None): + if category is None: + args = get_args(self._parser) + root, last_item = args.compile_commands.rsplit(os.path.sep, 1) + if (last_item == "compile_commands.json") and os.path.isfile( + args.compile_commands + ): + args.compile_commands = root + return args + elif category in self._internal_categories: + return get_args(self._parse_group[category]) + + +def main(): + cmake = Template(filename="CMakeLists.txt.in") + cat = CategorizedArgs() + all_args = cat.categorized_args + data = cmake.render(files=[], **(all_args)) + print(data) + + +if __name__ == "__main__": + main() diff --git a/bindings/python/scripts/third_party/conan.cmake b/bindings/python/scripts/third_party/conan.cmake new file mode 100644 index 0000000..b27dc33 --- /dev/null +++ b/bindings/python/scripts/third_party/conan.cmake @@ -0,0 +1,610 @@ +# The MIT License (MIT) + +# Copyright (c) 2018 JFrog + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + + +# This file comes from: https://github.com/conan-io/cmake-conan. Please refer +# to this repository for issues and documentation. + +# Its purpose is to wrap and launch Conan C/C++ Package Manager when cmake is called. +# It will take CMake current settings (os, compiler, compiler version, architecture) +# and translate them to conan settings for installing and retrieving dependencies. + +# It is intended to facilitate developers building projects that have conan dependencies, +# but it is only necessary on the end-user side. It is not necessary to create conan +# packages, in fact it shouldn't be use for that. Check the project documentation. + +# version: 0.15.0 + +include(CMakeParseArguments) + +function(_get_msvc_ide_version result) + set(${result} "" PARENT_SCOPE) + if(NOT MSVC_VERSION VERSION_LESS 1400 AND MSVC_VERSION VERSION_LESS 1500) + set(${result} 8 PARENT_SCOPE) + elseif(NOT MSVC_VERSION VERSION_LESS 1500 AND MSVC_VERSION VERSION_LESS 1600) + set(${result} 9 PARENT_SCOPE) + elseif(NOT MSVC_VERSION VERSION_LESS 1600 AND MSVC_VERSION VERSION_LESS 1700) + set(${result} 10 PARENT_SCOPE) + elseif(NOT MSVC_VERSION VERSION_LESS 1700 AND MSVC_VERSION VERSION_LESS 1800) + set(${result} 11 PARENT_SCOPE) + elseif(NOT MSVC_VERSION VERSION_LESS 1800 AND MSVC_VERSION VERSION_LESS 1900) + set(${result} 12 PARENT_SCOPE) + elseif(NOT MSVC_VERSION VERSION_LESS 1900 AND MSVC_VERSION VERSION_LESS 1910) + set(${result} 14 PARENT_SCOPE) + elseif(NOT MSVC_VERSION VERSION_LESS 1910 AND MSVC_VERSION VERSION_LESS 1920) + set(${result} 15 PARENT_SCOPE) + elseif(NOT MSVC_VERSION VERSION_LESS 1920 AND MSVC_VERSION VERSION_LESS 1930) + set(${result} 16 PARENT_SCOPE) + else() + message(FATAL_ERROR "Conan: Unknown MSVC compiler version [${MSVC_VERSION}]") + endif() +endfunction() + +function(conan_cmake_settings result) + #message(STATUS "COMPILER " ${CMAKE_CXX_COMPILER}) + #message(STATUS "COMPILER " ${CMAKE_CXX_COMPILER_ID}) + #message(STATUS "VERSION " ${CMAKE_CXX_COMPILER_VERSION}) + #message(STATUS "FLAGS " ${CMAKE_LANG_FLAGS}) + #message(STATUS "LIB ARCH " ${CMAKE_CXX_LIBRARY_ARCHITECTURE}) + #message(STATUS "BUILD TYPE " ${CMAKE_BUILD_TYPE}) + #message(STATUS "GENERATOR " ${CMAKE_GENERATOR}) + #message(STATUS "GENERATOR WIN64 " ${CMAKE_CL_64}) + + message(STATUS "Conan: Automatic detection of conan settings from cmake") + + parse_arguments(${ARGV}) + + if(ARGUMENTS_BUILD_TYPE) + set(_CONAN_SETTING_BUILD_TYPE ${ARGUMENTS_BUILD_TYPE}) + elseif(CMAKE_BUILD_TYPE) + set(_CONAN_SETTING_BUILD_TYPE ${CMAKE_BUILD_TYPE}) + else() + message(FATAL_ERROR "Please specify in command line CMAKE_BUILD_TYPE (-DCMAKE_BUILD_TYPE=Release)") + endif() + + string(TOUPPER ${_CONAN_SETTING_BUILD_TYPE} _CONAN_SETTING_BUILD_TYPE_UPPER) + if (_CONAN_SETTING_BUILD_TYPE_UPPER STREQUAL "DEBUG") + set(_CONAN_SETTING_BUILD_TYPE "Debug") + elseif(_CONAN_SETTING_BUILD_TYPE_UPPER STREQUAL "RELEASE") + set(_CONAN_SETTING_BUILD_TYPE "Release") + elseif(_CONAN_SETTING_BUILD_TYPE_UPPER STREQUAL "RELWITHDEBINFO") + set(_CONAN_SETTING_BUILD_TYPE "RelWithDebInfo") + elseif(_CONAN_SETTING_BUILD_TYPE_UPPER STREQUAL "MINSIZEREL") + set(_CONAN_SETTING_BUILD_TYPE "MinSizeRel") + endif() + + if(ARGUMENTS_ARCH) + set(_CONAN_SETTING_ARCH ${ARGUMENTS_ARCH}) + endif() + #handle -s os setting + if(CMAKE_SYSTEM_NAME) + #use default conan os setting if CMAKE_SYSTEM_NAME is not defined + set(CONAN_SYSTEM_NAME ${CMAKE_SYSTEM_NAME}) + if(${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") + set(CONAN_SYSTEM_NAME Macos) + endif() + set(CONAN_SUPPORTED_PLATFORMS Windows Linux Macos Android iOS FreeBSD WindowsStore) + list (FIND CONAN_SUPPORTED_PLATFORMS "${CONAN_SYSTEM_NAME}" _index) + if (${_index} GREATER -1) + #check if the cmake system is a conan supported one + set(_CONAN_SETTING_OS ${CONAN_SYSTEM_NAME}) + else() + message(FATAL_ERROR "cmake system ${CONAN_SYSTEM_NAME} is not supported by conan. Use one of ${CONAN_SUPPORTED_PLATFORMS}") + endif() + endif() + + get_property(_languages GLOBAL PROPERTY ENABLED_LANGUAGES) + if (";${_languages};" MATCHES ";CXX;") + set(LANGUAGE CXX) + set(USING_CXX 1) + elseif (";${_languages};" MATCHES ";C;") + set(LANGUAGE C) + set(USING_CXX 0) + else () + message(FATAL_ERROR "Conan: Neither C or C++ was detected as a language for the project. Unabled to detect compiler version.") + endif() + + if (${CMAKE_${LANGUAGE}_COMPILER_ID} STREQUAL GNU) + # using GCC + # TODO: Handle other params + string(REPLACE "." ";" VERSION_LIST ${CMAKE_${LANGUAGE}_COMPILER_VERSION}) + list(GET VERSION_LIST 0 MAJOR) + list(GET VERSION_LIST 1 MINOR) + set(COMPILER_VERSION ${MAJOR}.${MINOR}) + if(${MAJOR} GREATER 4) + set(COMPILER_VERSION ${MAJOR}) + endif() + set(_CONAN_SETTING_COMPILER gcc) + set(_CONAN_SETTING_COMPILER_VERSION ${COMPILER_VERSION}) + if (USING_CXX) + conan_cmake_detect_unix_libcxx(_LIBCXX) + set(_CONAN_SETTING_COMPILER_LIBCXX ${_LIBCXX}) + endif () + elseif (${CMAKE_${LANGUAGE}_COMPILER_ID} STREQUAL AppleClang) + # using AppleClang + string(REPLACE "." ";" VERSION_LIST ${CMAKE_${LANGUAGE}_COMPILER_VERSION}) + list(GET VERSION_LIST 0 MAJOR) + list(GET VERSION_LIST 1 MINOR) + set(_CONAN_SETTING_COMPILER apple-clang) + set(_CONAN_SETTING_COMPILER_VERSION ${MAJOR}.${MINOR}) + if (USING_CXX) + conan_cmake_detect_unix_libcxx(_LIBCXX) + set(_CONAN_SETTING_COMPILER_LIBCXX ${_LIBCXX}) + endif () + elseif (${CMAKE_${LANGUAGE}_COMPILER_ID} STREQUAL Clang) + string(REPLACE "." ";" VERSION_LIST ${CMAKE_${LANGUAGE}_COMPILER_VERSION}) + list(GET VERSION_LIST 0 MAJOR) + list(GET VERSION_LIST 1 MINOR) + set(_CONAN_SETTING_COMPILER clang) + set(_CONAN_SETTING_COMPILER_VERSION ${MAJOR}.${MINOR}) + if(APPLE) + cmake_policy(GET CMP0025 APPLE_CLANG_POLICY) + if(NOT APPLE_CLANG_POLICY STREQUAL NEW) + message(STATUS "Conan: APPLE and Clang detected. Assuming apple-clang compiler. Set CMP0025 to avoid it") + set(_CONAN_SETTING_COMPILER apple-clang) + endif() + endif() + if(${_CONAN_SETTING_COMPILER} STREQUAL clang AND ${MAJOR} GREATER 7) + set(_CONAN_SETTING_COMPILER_VERSION ${MAJOR}) + endif() + if (USING_CXX) + conan_cmake_detect_unix_libcxx(_LIBCXX) + set(_CONAN_SETTING_COMPILER_LIBCXX ${_LIBCXX}) + endif () + elseif(${CMAKE_${LANGUAGE}_COMPILER_ID} STREQUAL MSVC) + set(_VISUAL "Visual Studio") + _get_msvc_ide_version(_VISUAL_VERSION) + if("${_VISUAL_VERSION}" STREQUAL "") + message(FATAL_ERROR "Conan: Visual Studio not recognized") + else() + set(_CONAN_SETTING_COMPILER ${_VISUAL}) + set(_CONAN_SETTING_COMPILER_VERSION ${_VISUAL_VERSION}) + endif() + + if(NOT _CONAN_SETTING_ARCH) + if (MSVC_${LANGUAGE}_ARCHITECTURE_ID MATCHES "64") + set(_CONAN_SETTING_ARCH x86_64) + elseif (MSVC_${LANGUAGE}_ARCHITECTURE_ID MATCHES "^ARM") + message(STATUS "Conan: Using default ARM architecture from MSVC") + set(_CONAN_SETTING_ARCH armv6) + elseif (MSVC_${LANGUAGE}_ARCHITECTURE_ID MATCHES "86") + set(_CONAN_SETTING_ARCH x86) + else () + message(FATAL_ERROR "Conan: Unknown MSVC architecture [${MSVC_${LANGUAGE}_ARCHITECTURE_ID}]") + endif() + endif() + + conan_cmake_detect_vs_runtime(_vs_runtime) + message(STATUS "Conan: Detected VS runtime: ${_vs_runtime}") + set(_CONAN_SETTING_COMPILER_RUNTIME ${_vs_runtime}) + + if (CMAKE_GENERATOR_TOOLSET) + set(_CONAN_SETTING_COMPILER_TOOLSET ${CMAKE_VS_PLATFORM_TOOLSET}) + elseif(CMAKE_VS_PLATFORM_TOOLSET AND (CMAKE_GENERATOR STREQUAL "Ninja")) + set(_CONAN_SETTING_COMPILER_TOOLSET ${CMAKE_VS_PLATFORM_TOOLSET}) + endif() + else() + message(FATAL_ERROR "Conan: compiler setup not recognized") + endif() + + # If profile is defined it is used + if(CMAKE_BUILD_TYPE STREQUAL "Debug" AND ARGUMENTS_DEBUG_PROFILE) + set(_APPLIED_PROFILES ${ARGUMENTS_DEBUG_PROFILE}) + elseif(CMAKE_BUILD_TYPE STREQUAL "Release" AND ARGUMENTS_RELEASE_PROFILE) + set(_APPLIED_PROFILES ${ARGUMENTS_RELEASE_PROFILE}) + elseif(CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo" AND ARGUMENTS_RELWITHDEBINFO_PROFILE) + set(_APPLIED_PROFILES ${ARGUMENTS_RELWITHDEBINFO_PROFILE}) + elseif(CMAKE_BUILD_TYPE STREQUAL "MinSizeRel" AND ARGUMENTS_MINSIZEREL_PROFILE) + set(_APPLIED_PROFILES ${ARGUMENTS_MINSIZEREL_PROFILE}) + elseif(ARGUMENTS_PROFILE) + set(_APPLIED_PROFILES ${ARGUMENTS_PROFILE}) + endif() + + foreach(ARG ${_APPLIED_PROFILES}) + set(_SETTINGS ${_SETTINGS} -pr=${ARG}) + endforeach() + + if(NOT _SETTINGS OR ARGUMENTS_PROFILE_AUTO STREQUAL "ALL") + set(ARGUMENTS_PROFILE_AUTO arch build_type compiler compiler.version + compiler.runtime compiler.libcxx compiler.toolset) + endif() + + # Automatic from CMake + foreach(ARG ${ARGUMENTS_PROFILE_AUTO}) + string(TOUPPER ${ARG} _arg_name) + string(REPLACE "." "_" _arg_name ${_arg_name}) + if(_CONAN_SETTING_${_arg_name}) + set(_SETTINGS ${_SETTINGS} -s ${ARG}=${_CONAN_SETTING_${_arg_name}}) + endif() + endforeach() + + foreach(ARG ${ARGUMENTS_SETTINGS}) + set(_SETTINGS ${_SETTINGS} -s ${ARG}) + endforeach() + + message(STATUS "Conan: Settings= ${_SETTINGS}") + + set(${result} ${_SETTINGS} PARENT_SCOPE) +endfunction() + + +function(conan_cmake_detect_unix_libcxx result) + # Take into account any -stdlib in compile options + get_directory_property(compile_options DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMPILE_OPTIONS) + + # Take into account any _GLIBCXX_USE_CXX11_ABI in compile definitions + get_directory_property(defines DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMPILE_DEFINITIONS) + foreach(define ${defines}) + if(define MATCHES "_GLIBCXX_USE_CXX11_ABI") + if(define MATCHES "^-D") + set(compile_options ${compile_options} "${define}") + else() + set(compile_options ${compile_options} "-D${define}") + endif() + endif() + endforeach() + + execute_process( + COMMAND ${CMAKE_COMMAND} -E echo "#include " + COMMAND ${CMAKE_CXX_COMPILER} -x c++ ${compile_options} -E -dM - + OUTPUT_VARIABLE string_defines + ) + + if(string_defines MATCHES "#define __GLIBCXX__") + # Allow -D_GLIBCXX_USE_CXX11_ABI=ON/OFF as argument to cmake + if(DEFINED _GLIBCXX_USE_CXX11_ABI) + if(_GLIBCXX_USE_CXX11_ABI) + set(${result} libstdc++11 PARENT_SCOPE) + return() + else() + set(${result} libstdc++ PARENT_SCOPE) + return() + endif() + endif() + + if(string_defines MATCHES "#define _GLIBCXX_USE_CXX11_ABI 1\n") + set(${result} libstdc++11 PARENT_SCOPE) + else() + # Either the compiler is missing the define because it is old, and so + # it can't use the new abi, or the compiler was configured to use the + # old abi by the user or distro (e.g. devtoolset on RHEL/CentOS) + set(${result} libstdc++ PARENT_SCOPE) + endif() + else() + set(${result} libc++ PARENT_SCOPE) + endif() +endfunction() + +function(conan_cmake_detect_vs_runtime result) + string(TOUPPER ${CMAKE_BUILD_TYPE} build_type) + set(variables CMAKE_CXX_FLAGS_${build_type} CMAKE_C_FLAGS_${build_type} CMAKE_CXX_FLAGS CMAKE_C_FLAGS) + foreach(variable ${variables}) + if(NOT "${${variable}}" STREQUAL "") + string(REPLACE " " ";" flags ${${variable}}) + foreach (flag ${flags}) + if(${flag} STREQUAL "/MD" OR ${flag} STREQUAL "/MDd" OR ${flag} STREQUAL "/MT" OR ${flag} STREQUAL "/MTd") + string(SUBSTRING ${flag} 1 -1 runtime) + set(${result} ${runtime} PARENT_SCOPE) + return() + endif() + endforeach() + endif() + endforeach() + if(${build_type} STREQUAL "DEBUG") + set(${result} "MDd" PARENT_SCOPE) + else() + set(${result} "MD" PARENT_SCOPE) + endif() +endfunction() + + +macro(parse_arguments) + set(options BASIC_SETUP CMAKE_TARGETS UPDATE KEEP_RPATHS NO_LOAD NO_OUTPUT_DIRS OUTPUT_QUIET NO_IMPORTS SKIP_STD) + set(oneValueArgs CONANFILE ARCH BUILD_TYPE INSTALL_FOLDER CONAN_COMMAND) + set(multiValueArgs DEBUG_PROFILE RELEASE_PROFILE RELWITHDEBINFO_PROFILE MINSIZEREL_PROFILE + PROFILE REQUIRES OPTIONS IMPORTS SETTINGS BUILD ENV GENERATORS PROFILE_AUTO + INSTALL_ARGS CONFIGURATION_TYPES) + cmake_parse_arguments(ARGUMENTS "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) +endmacro() + +function(conan_cmake_install) + # Calls "conan install" + # Argument BUILD is equivalant to --build={missing, PkgName,...} or + # --build when argument is 'BUILD all' (which builds all packages from source) + # Argument CONAN_COMMAND, to specify the conan path, e.g. in case of running from source + # cmake does not identify conan as command, even if it is +x and it is in the path + parse_arguments(${ARGV}) + + if(CONAN_CMAKE_MULTI) + set(ARGUMENTS_GENERATORS ${ARGUMENTS_GENERATORS} cmake_multi) + else() + set(ARGUMENTS_GENERATORS ${ARGUMENTS_GENERATORS} cmake) + endif() + + set(CONAN_BUILD_POLICY "") + foreach(ARG ${ARGUMENTS_BUILD}) + if(${ARG} STREQUAL "all") + set(CONAN_BUILD_POLICY ${CONAN_BUILD_POLICY} --build) + break() + else() + set(CONAN_BUILD_POLICY ${CONAN_BUILD_POLICY} --build=${ARG}) + endif() + endforeach() + if(ARGUMENTS_CONAN_COMMAND) + set(CONAN_CMD ${ARGUMENTS_CONAN_COMMAND}) + else() + conan_check(REQUIRED) + endif() + set(CONAN_OPTIONS "") + if(ARGUMENTS_CONANFILE) + set(CONANFILE ${CMAKE_CURRENT_SOURCE_DIR}/${ARGUMENTS_CONANFILE}) + # A conan file has been specified - apply specified options as well if provided + foreach(ARG ${ARGUMENTS_OPTIONS}) + set(CONAN_OPTIONS ${CONAN_OPTIONS} -o=${ARG}) + endforeach() + else() + set(CONANFILE ".") + endif() + if(ARGUMENTS_UPDATE) + set(CONAN_INSTALL_UPDATE --update) + endif() + if(ARGUMENTS_NO_IMPORTS) + set(CONAN_INSTALL_NO_IMPORTS --no-imports) + endif() + set(CONAN_INSTALL_FOLDER "") + if(ARGUMENTS_INSTALL_FOLDER) + set(CONAN_INSTALL_FOLDER -if=${ARGUMENTS_INSTALL_FOLDER}) + endif() + foreach(ARG ${ARGUMENTS_GENERATORS}) + set(CONAN_GENERATORS ${CONAN_GENERATORS} -g=${ARG}) + endforeach() + foreach(ARG ${ARGUMENTS_ENV}) + set(CONAN_ENV_VARS ${CONAN_ENV_VARS} -e=${ARG}) + endforeach() + set(conan_args install ${CONANFILE} ${settings} ${CONAN_ENV_VARS} ${CONAN_GENERATORS} ${CONAN_BUILD_POLICY} ${CONAN_INSTALL_UPDATE} ${CONAN_INSTALL_NO_IMPORTS} ${CONAN_OPTIONS} ${CONAN_INSTALL_FOLDER} ${ARGUMENTS_INSTALL_ARGS}) + + string (REPLACE ";" " " _conan_args "${conan_args}") + message(STATUS "Conan executing: ${CONAN_CMD} ${_conan_args}") + + if(ARGUMENTS_OUTPUT_QUIET) + execute_process(COMMAND ${CONAN_CMD} ${conan_args} + RESULT_VARIABLE return_code + OUTPUT_VARIABLE conan_output + ERROR_VARIABLE conan_output + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + else() + execute_process(COMMAND ${CONAN_CMD} ${conan_args} + RESULT_VARIABLE return_code + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + endif() + + if(NOT "${return_code}" STREQUAL "0") + message(FATAL_ERROR "Conan install failed='${return_code}'") + endif() + +endfunction() + + +function(conan_cmake_setup_conanfile) + parse_arguments(${ARGV}) + if(ARGUMENTS_CONANFILE) + get_filename_component(_CONANFILE_NAME ${ARGUMENTS_CONANFILE} NAME) + # configure_file will make sure cmake re-runs when conanfile is updated + configure_file(${ARGUMENTS_CONANFILE} ${CMAKE_CURRENT_BINARY_DIR}/${_CONANFILE_NAME}.junk COPYONLY) + file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/${_CONANFILE_NAME}.junk) + else() + conan_cmake_generate_conanfile(${ARGV}) + endif() +endfunction() + +function(conan_cmake_generate_conanfile) + # Generate, writing in disk a conanfile.txt with the requires, options, and imports + # specified as arguments + # This will be considered as temporary file, generated in CMAKE_CURRENT_BINARY_DIR) + parse_arguments(${ARGV}) + set(_FN "${CMAKE_CURRENT_BINARY_DIR}/conanfile.txt") + + file(WRITE ${_FN} "[generators]\ncmake\n\n[requires]\n") + foreach(ARG ${ARGUMENTS_REQUIRES}) + file(APPEND ${_FN} ${ARG} "\n") + endforeach() + + file(APPEND ${_FN} ${ARG} "\n[options]\n") + foreach(ARG ${ARGUMENTS_OPTIONS}) + file(APPEND ${_FN} ${ARG} "\n") + endforeach() + + file(APPEND ${_FN} ${ARG} "\n[imports]\n") + foreach(ARG ${ARGUMENTS_IMPORTS}) + file(APPEND ${_FN} ${ARG} "\n") + endforeach() +endfunction() + + +macro(conan_load_buildinfo) + if(CONAN_CMAKE_MULTI) + set(_CONANBUILDINFO conanbuildinfo_multi.cmake) + else() + set(_CONANBUILDINFO conanbuildinfo.cmake) + endif() + if(ARGUMENTS_INSTALL_FOLDER) + set(_CONANBUILDINFOFOLDER ${ARGUMENTS_INSTALL_FOLDER}) + else() + set(_CONANBUILDINFOFOLDER ${CMAKE_CURRENT_BINARY_DIR}) + endif() + # Checks for the existence of conanbuildinfo.cmake, and loads it + # important that it is macro, so variables defined at parent scope + if(EXISTS "${_CONANBUILDINFOFOLDER}/${_CONANBUILDINFO}") + message(STATUS "Conan: Loading ${_CONANBUILDINFO}") + include(${_CONANBUILDINFOFOLDER}/${_CONANBUILDINFO}) + else() + message(FATAL_ERROR "${_CONANBUILDINFO} doesn't exist in ${CMAKE_CURRENT_BINARY_DIR}") + endif() +endmacro() + + +macro(conan_cmake_run) + parse_arguments(${ARGV}) + + if(ARGUMENTS_CONFIGURATION_TYPES AND NOT CMAKE_CONFIGURATION_TYPES) + message(WARNING "CONFIGURATION_TYPES should only be specified for multi-configuration generators") + elseif(ARGUMENTS_CONFIGURATION_TYPES AND ARGUMENTS_BUILD_TYPE) + message(WARNING "CONFIGURATION_TYPES and BUILD_TYPE arguments should not be defined at the same time.") + endif() + + if(CMAKE_CONFIGURATION_TYPES AND NOT CMAKE_BUILD_TYPE AND NOT CONAN_EXPORTED + AND NOT ARGUMENTS_BUILD_TYPE) + set(CONAN_CMAKE_MULTI ON) + if (NOT ARGUMENTS_CONFIGURATION_TYPES) + set(ARGUMENTS_CONFIGURATION_TYPES "Release;Debug") + endif() + message(STATUS "Conan: Using cmake-multi generator") + else() + set(CONAN_CMAKE_MULTI OFF) + endif() + + if(NOT CONAN_EXPORTED) + conan_cmake_setup_conanfile(${ARGV}) + if(CONAN_CMAKE_MULTI) + foreach(CMAKE_BUILD_TYPE ${ARGUMENTS_CONFIGURATION_TYPES}) + set(ENV{CONAN_IMPORT_PATH} ${CMAKE_BUILD_TYPE}) + conan_cmake_settings(settings ${ARGV}) + conan_cmake_install(SETTINGS ${settings} ${ARGV}) + endforeach() + set(CMAKE_BUILD_TYPE) + else() + conan_cmake_settings(settings ${ARGV}) + conan_cmake_install(SETTINGS ${settings} ${ARGV}) + endif() + endif() + + if (NOT ARGUMENTS_NO_LOAD) + conan_load_buildinfo() + endif() + + if(ARGUMENTS_BASIC_SETUP) + foreach(_option CMAKE_TARGETS KEEP_RPATHS NO_OUTPUT_DIRS SKIP_STD) + if(ARGUMENTS_${_option}) + if(${_option} STREQUAL "CMAKE_TARGETS") + list(APPEND _setup_options "TARGETS") + else() + list(APPEND _setup_options ${_option}) + endif() + endif() + endforeach() + conan_basic_setup(${_setup_options}) + endif() +endmacro() + +macro(conan_check) + # Checks conan availability in PATH + # Arguments REQUIRED and VERSION are optional + # Example usage: + # conan_check(VERSION 1.0.0 REQUIRED) + message(STATUS "Conan: checking conan executable") + set(options REQUIRED) + set(oneValueArgs VERSION) + cmake_parse_arguments(CONAN "${options}" "${oneValueArgs}" "" ${ARGN}) + + find_program(CONAN_CMD conan) + if(NOT CONAN_CMD AND CONAN_REQUIRED) + message(FATAL_ERROR "Conan executable not found!") + endif() + message(STATUS "Conan: Found program ${CONAN_CMD}") + execute_process(COMMAND ${CONAN_CMD} --version + OUTPUT_VARIABLE CONAN_VERSION_OUTPUT + ERROR_VARIABLE CONAN_VERSION_OUTPUT) + message(STATUS "Conan: Version found ${CONAN_VERSION_OUTPUT}") + + if(DEFINED CONAN_VERSION) + string(REGEX MATCH ".*Conan version ([0-9]+\.[0-9]+\.[0-9]+)" FOO + "${CONAN_VERSION_OUTPUT}") + if(${CMAKE_MATCH_1} VERSION_LESS ${CONAN_VERSION}) + message(FATAL_ERROR "Conan outdated. Installed: ${CMAKE_MATCH_1}, \ + required: ${CONAN_VERSION}. Consider updating via 'pip \ + install conan==${CONAN_VERSION}'.") + endif() + endif() +endmacro() + +function(conan_add_remote) + # Adds a remote + # Arguments URL and NAME are required, INDEX and COMMAND are optional + # Example usage: + # conan_add_remote(NAME bincrafters INDEX 1 + # URL https://api.bintray.com/conan/bincrafters/public-conan) + set(oneValueArgs URL NAME INDEX COMMAND) + cmake_parse_arguments(CONAN "" "${oneValueArgs}" "" ${ARGN}) + + if(DEFINED CONAN_INDEX) + set(CONAN_INDEX_ARG "-i ${CONAN_INDEX}") + endif() + if(CONAN_COMMAND) + set(CONAN_CMD ${CONAN_COMMAND}) + else() + conan_check(REQUIRED) + endif() + message(STATUS "Conan: Adding ${CONAN_NAME} remote repository (${CONAN_URL})") + execute_process(COMMAND ${CONAN_CMD} remote add ${CONAN_NAME} ${CONAN_URL} + ${CONAN_INDEX_ARG} -f) +endfunction() + +macro(conan_config_install) + # install a full configuration from a local or remote zip file + # Argument ITEM is required, arguments TYPE, SOURCE, TARGET and VERIFY_SSL are optional + # Example usage: + # conan_config_install(ITEM https://github.com/conan-io/cmake-conan.git + # TYPE git SOURCE source-folder TARGET target-folder VERIFY_SSL false) + set(oneValueArgs ITEM TYPE SOURCE TARGET VERIFY_SSL) + set(multiValueArgs ARGS) + cmake_parse_arguments(CONAN "" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + set(CONAN_CONFIG_INSTALL_ARGS "") + + find_program(CONAN_CMD conan) + if(NOT CONAN_CMD AND CONAN_REQUIRED) + message(FATAL_ERROR "Conan executable not found!") + endif() + + if(DEFINED CONAN_VERIFY_SSL) + set(CONAN_CONFIG_INSTALL_ARGS "${CONAN_CONFIG_INSTALL_ARGS} --verify-ssl ${CONAN_VERIFY_SSL}") + endif() + + if(DEFINED CONAN_TYPE) + set(CONAN_CONFIG_INSTALL_ARGS "${CONAN_CONFIG_INSTALL_ARGS} --type ${CONAN_TYPE}") + endif() + + if(DEFINED CONAN_ARGS) + set(CONAN_CONFIG_INSTALL_ARGS "${CONAN_CONFIG_INSTALL_ARGS} --args \"${CONAN_ARGS}\"") + endif() + + if(DEFINED CONAN_SOURCE) + set(CONAN_CONFIG_INSTALL_ARGS "${CONAN_CONFIG_INSTALL_ARGS} --source-folder ${CONAN_SOURCE}") + endif() + + if(DEFINED CONAN_TARGET) + set(CONAN_CONFIG_INSTALL_ARGS "${CONAN_CONFIG_INSTALL_ARGS} --target-folder ${CONAN_TARGET}") + endif() + + message(STATUS "Conan: Installing config from ${CONAN_ITEM}") + execute_process(COMMAND ${CONAN_CMD} config install ${CONAN_CONFIG_INSTALL_ARGS} ${CONAN_ITEM}) +endmacro() From 555b8f5729df11c483277092a3da53e87c757fc7 Mon Sep 17 00:00:00 2001 From: Felipe Ximenes Date: Thu, 5 Nov 2020 04:45:18 -0300 Subject: [PATCH 07/19] Initial README.md (#16) --- README.md | 33 +++++++++++++++++++++++++++++++++ requirements.txt | 7 +++++++ 2 files changed, 40 insertions(+) create mode 100644 requirements.txt diff --git a/README.md b/README.md index e69de29..dfa78c3 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,33 @@ +# Description + +The clang-bind is a project to generate python bindings for C++ code using clang python bindings and pybind11. + +# Dependencies + +**C++** + +*libclang* + +``` +wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - +echo 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-11 main' | sudo tee -a /etc/apt/sources.list +echo 'deb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic-11 main' | sudo tee -a /etc/apt/sources.list +sudo apt-get update +sudo apt-get install -y libclang-11-dev python3-clang-11 +``` + +**Python** + +`pip install -r requirements.txt` + +# Demonstration + +1. Go to `clang-bind/bindings/python/tests/test_project/` folder +2. Create a build folder +3. Run `cmake ..` +4. Run `make -j$(nproc)` +5. Run `python ../../scripts/parse.py --com ./ ../src/simple.cpp` +6. Run `python ../../scripts/generate.py --com json/src/simple.json` + +The binding code will be available in `pybind11-gen/src` folder. + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..4f590eb --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +Mako==1.1.3 +requests==2.24.0 +conan==1.31.0 +pybind11==2.6.0 +black==20.8b1 +pytest==6.1.2 +cmake==3.18.2 \ No newline at end of file From 4343a2f4fdfdfc52b773a86e942b66373498aa58 Mon Sep 17 00:00:00 2001 From: Divyanshu Madan Date: Sun, 28 Mar 2021 22:06:10 +0530 Subject: [PATCH 08/19] [interface.py] CMakeLists.txt.in filepath minor fix --- bindings/python/scripts/interface.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bindings/python/scripts/interface.py b/bindings/python/scripts/interface.py index 5af2f19..5159bfd 100644 --- a/bindings/python/scripts/interface.py +++ b/bindings/python/scripts/interface.py @@ -157,7 +157,8 @@ def _get_args(self, category=None): def main(): - cmake = Template(filename="CMakeLists.txt.in") + file_dir = os.path.dirname(__file__) + cmake = Template(filename=os.path.join(file_dir, "CMakeLists.txt.in")) cat = CategorizedArgs() all_args = cat.categorized_args data = cmake.render(files=[], **(all_args)) From 3e1d46f4fdbcac47a64eb30f291db4eeb5c41881 Mon Sep 17 00:00:00 2001 From: Divyanshu Madan Date: Fri, 23 Oct 2020 11:22:55 +0530 Subject: [PATCH 09/19] [clang_utils.py] Introduce clang_utils.py to get checks from cindex.py --- bindings/python/scripts/clang_utils.py | 120 +++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 bindings/python/scripts/clang_utils.py diff --git a/bindings/python/scripts/clang_utils.py b/bindings/python/scripts/clang_utils.py new file mode 100644 index 0000000..f9e00c8 --- /dev/null +++ b/bindings/python/scripts/clang_utils.py @@ -0,0 +1,120 @@ +import inspect +import clang.cindex as clang + + +def getmembers_static(object, predicate=None): + """ + Return all members of an object as (name, value) pairs sorted by name via `getattr_static`. + Optionally, only return members that satisfy a given predicate. + + + - A static version of `get_members` function at: + https://github.com/python/cpython/blob/3.9/Lib/inspect.py#L326-L368 + https://github.com/python/cpython/blob/14ba761078b5ae83519e34d66ab883743912c45b/Lib/inspect.py#L444-L486 + - `getmembers` function (from the inspect module) triggers execution instead of doing static analysis. + - This leads to errors, particularly on properties of classes in cindex.py, which causes segmentation errors or raises an Exception if a particular condition is not satisfied. + - To curb this, we fetch the members statically. We define a custom function based on the one in the inspect module. + """ + + results = [] + names = dir(object) + # :dd any DynamicClassAttributes to the list of names if object is a class; + # this may result in duplicate entries if, for example, a virtual + # attribute with the same name as a DynamicClassAttribute exists + try: + base_members = filter( + lambda k, v: isinstance(v, types.DynamicClassAttribute), + object.__bases__.__dict__.items(), + ) + names.extend(base_members) + except AttributeError: + pass + for key in names: + value = inspect.getattr_static(object, key) + if not predicate or predicate(value): + results.append((key, value)) + results.sort(key=lambda pair: pair[0]) + return results + + +class ClangUtils: + """ + Clang's cindex class utilities. + + Supports the following objects: + CursorKind: + https://github.com/llvm/llvm-project/blob/release/12.x/clang/bindings/python/clang/cindex.py#L657 + https://github.com/llvm/llvm-project/blob/1acd9a1a29ac30044ecefb6613485d5d168f66ca/clang/bindings/python/clang/cindex.py#L657 + - A CursorKind describes the kind of entity that a cursor points to. + Cursor: + https://github.com/llvm/llvm-project/blob/release/12.x/clang/bindings/python/clang/cindex.py#L1415 + https://github.com/llvm/llvm-project/blob/1acd9a1a29ac30044ecefb6613485d5d168f66ca/clang/bindings/python/clang/cindex.py#L1415 + - The Cursor class represents a reference to an element within the AST. It acts as a kind of iterator. + Type: + https://github.com/llvm/llvm-project/blob/release/12.x/clang/bindings/python/clang/cindex.py#L2180 + https://github.com/llvm/llvm-project/blob/1acd9a1a29ac30044ecefb6613485d5d168f66ca/clang/bindings/python/clang/cindex.py#L2180 + - The Type class represents the type of an element in the abstract syntax tree. + """ + + def __init__(self, object): + if not ( + isinstance(object, clang.CursorKind) + or isinstance(object, clang.Cursor) + or isinstance(object, clang.Type) + ): + raise NotImplementedError(f"Not implemented for {object}") + + self.check_functions_dict = {} + self.get_functions_dict = {} + self.properties_dict = {} + + # A list to ignore the functions/properties that causes segmentation errors. + ignore_list = [ + "mangled_name", + "get_address_space", + "get_typedef_name", + "tls_kind", + ] + + # populate dicts + valid_entries = filter( + lambda entry: entry[0] not in ignore_list, getmembers_static(object) + ) + for name, func in valid_entries: + if inspect.isfunction(func): # if function + try: # cindex.py's functions raise exceptions internally + if name.startswith("is_"): + self.check_functions_dict[name] = func(object) + if name.startswith("get_"): + self.get_functions_dict[name] = func(object) + except: + continue + elif isinstance(func, property): # else, property + try: # cindex.py's property functions raise exceptions internally + self.properties_dict[name] = getattr(object, name) + except: + continue + + def get_check_functions_dict(self): + """ + Returns: `check_functions_dict`: + - functions that begin with "is_" i.e., checking functions + - {function_name, function_result} + """ + return self.check_functions_dict + + def get_get_functions_dict(self): + """ + Returns: `get_functions_dict`: + - functions that begin with "get_" i.e., getter functions + - {function_name, function_result} + """ + return self.get_functions_dict + + def get_properties_dict(self): + """ + Returns: properties_dict + - Properties + - {property_name, property} + """ + return self.properties_dict From 8f07af2d9849b8e340fcb6140710d78085442047 Mon Sep 17 00:00:00 2001 From: Divyanshu Madan Date: Sun, 28 Feb 2021 03:57:15 +0530 Subject: [PATCH 10/19] changes coming from clang_utils.py --- bindings/python/scripts/generate.py | 78 +++-- bindings/python/scripts/parse.py | 420 +++++++++---------------- bindings/python/tests/test_generate.py | 2 +- bindings/python/tests/test_parse.py | 203 ++++++------ 4 files changed, 301 insertions(+), 402 deletions(-) diff --git a/bindings/python/scripts/generate.py b/bindings/python/scripts/generate.py index 1380860..9426718 100644 --- a/bindings/python/scripts/generate.py +++ b/bindings/python/scripts/generate.py @@ -148,10 +148,14 @@ def get_fields_from_anonymous(item: dict) -> list: fields = [] for sub_item in item["members"]: # base condition - if sub_item["kind"] == "FIELD_DECL": + if sub_item["cursor_kind"]["name"] == "FIELD_DECL": fields.append(sub_item) # recurse - elif sub_item["kind"] in ("ANONYMOUS_UNION_DECL", "ANONYMOUS_STRUCT_DECL"): + # @TODO Fix this, `ANONYMOUS_kind` was removed, now test via `is_anonymous` + elif sub_item["cursor_kind"]["name"] in ( + "ANONYMOUS_UNION_DECL", + "ANONYMOUS_STRUCT_DECL", + ): fields += bind.get_fields_from_anonymous(item=sub_item) return fields @@ -173,8 +177,8 @@ def handle_node(self, item: dict) -> None: """ self.item = item - self.kind = self.item["kind"] - self.name = self.item["name"] + self.kind = self.item["cursor_kind"]["name"] + self.name = self.item["cursor"]["spelling"] self.members = self.item["members"] self.depth = self.item["depth"] @@ -218,17 +222,21 @@ def handle_struct_decl(self) -> None: template_class_name = None template_class_name_python = None for sub_item in self.members: - if sub_item["kind"] == "TYPE_REF": + if sub_item["cursor_kind"]["name"] == "TYPE_REF": # TODO: Will this case only apply to templates? # @TODO: Make more robust - type_ref = sub_item["name"].replace("struct ", "").replace("pcl::", "") + type_ref = ( + sub_item["cursor"]["spelling"] + .replace("struct ", "") + .replace("pcl::", "") + ) template_class_name = f"{self.name}<{type_ref}>" template_class_name_python = f"{self.name}_{type_ref}" base_class_list = [ - sub_item["name"] + sub_item["cursor"]["spelling"] for sub_item in self.members - if sub_item["kind"] == "CXX_BASE_SPECIFIER" + if sub_item["cursor_kind"]["name"] == "CXX_BASE_SPECIFIER" ] base_class_list_string = [ @@ -253,35 +261,35 @@ def handle_struct_decl(self) -> None: for sub_item in self.members: fields = self.get_fields_from_anonymous(sub_item) for field in fields: - if field["element_type"] == "ConstantArray": + if field["type"]["kind"] == "ConstantArray": # TODO: FIX: readwrite, not readonly self._linelist.append( - f'.def_property_readonly("{field["name"]}", []({self.name}& obj) {{return obj.{field["name"]}; }})' # float[ ' + f'obj.{sub_item["name"]}' + '.size()];} )' + f'.def_property_readonly("{field["cursor"]["spelling"]}", []({self.name}& obj) {{return obj.{field["cursor"]["spelling"]}; }})' # float[ ' + f'obj.{sub_item["cursor"]["spelling"]}' + '.size()];} )' ) else: self._linelist.append( - f'.def_readwrite("{field["name"]}", &{self.name}::{field["name"]})' + f'.def_readwrite("{field["cursor"]["spelling"]}", &{self.name}::{field["cursor"]["spelling"]})' ) for sub_item in self.members: # handle field declarations - if sub_item["kind"] == "FIELD_DECL": - if sub_item["element_type"] == "ConstantArray": + if sub_item["cursor_kind"]["name"] == "FIELD_DECL": + if sub_item["type"]["kind"] == "ConstantArray": self._linelist.append( - f'.def_property_readonly("{sub_item["name"]}", []({self.name}& obj) {{return obj.{sub_item["name"]}; }})' # float[ ' + f'obj.{sub_item["name"]}' + '.size()];} )' + f'.def_property_readonly("{sub_item["cursor"]["spelling"]}", []({self.name}& obj) {{return obj.{sub_item["cursor"]["spelling"]}; }})' # float[ ' + f'obj.{sub_item["cursor"]["spelling"]}' + '.size()];} )' ) else: self._linelist.append( - f'.def_readwrite("{sub_item["name"]}", &{self.name}::{sub_item["name"]})' + f'.def_readwrite("{sub_item["cursor"]["spelling"]}", &{self.name}::{sub_item["cursor"]["spelling"]})' ) # handle class methods - elif sub_item["kind"] == "CXX_METHOD": + elif sub_item["cursor_kind"]["name"] == "CXX_METHOD": # TODO: Add template args, currently blank - if sub_item["name"] not in ("PCL_DEPRECATED"): + if sub_item["cursor"]["spelling"] not in ("PCL_DEPRECATED"): self._linelist.append( - f'.def("{sub_item["name"]}", py::overload_cast<>(&{self.name}::{sub_item["name"]}))' + f'.def("{sub_item["cursor"]["spelling"]}", py::overload_cast<>(&{self.name}::{sub_item["cursor"]["spelling"]}))' ) def handle_function(self) -> None: @@ -293,8 +301,8 @@ def handle_function(self) -> None: parameter_type_list = [] details = self._state_stack[-1] for sub_item in self.members: - if sub_item["kind"] == "PARM_DECL": - parameter_type_list.append(f'"{sub_item["name"]}"_a') + if sub_item["cursor_kind"]["name"] == "PARM_DECL": + parameter_type_list.append(f'"{sub_item["cursor"]["spelling"]}"_a') parameter_type_list = ",".join(parameter_type_list) if parameter_type_list: @@ -317,7 +325,7 @@ def handle_constructor(self) -> None: # generate parameter type list for sub_item in self.members: - if sub_item["kind"] == "PARM_DECL": + if sub_item["cursor_kind"]["name"] == "PARM_DECL": parameter_type_list.append(self.get_parm_types(sub_item)) parameter_type_list = ",".join(parameter_type_list) @@ -326,25 +334,29 @@ def handle_constructor(self) -> None: self._linelist.append(f".def(py::init<{parameter_type_list}>())") def get_parm_types(self, item: Dict[str, Any]) -> List[str]: - if item["element_type"] == "LValueReference": + if item["type"]["kind"] == "LValueReference": for sub_item in item["members"]: - if sub_item["kind"] == "TYPE_REF": + if sub_item["cursor_kind"]["name"] == "TYPE_REF": # @TODO: Make more robust type_ref = ( - sub_item["name"].replace("struct ", "").replace("pcl::", "") + sub_item["cursor"]["spelling"] + .replace("struct ", "") + .replace("pcl::", "") ) parameter_type_list = f"{type_ref} &" - elif item["element_type"] == "Elaborated": + elif item["type"]["kind"] == "Elaborated": namespace_ref = "" for sub_item in item["members"]: - if sub_item["kind"] == "NAMESPACE_REF": - namespace_ref += f'{sub_item["name"]}::' - if sub_item["kind"] == "TYPE_REF": - parameter_type_list = f'{namespace_ref}{sub_item["name"]}' - elif item["element_type"] in ("Float", "Double", "Int"): - parameter_type_list = f'{item["element_type"].lower()}' + if sub_item["cursor_kind"]["name"] == "NAMESPACE_REF": + namespace_ref += f'{sub_item["cursor"]["spelling"]}::' + if sub_item["cursor_kind"]["name"] == "TYPE_REF": + parameter_type_list = ( + f'{namespace_ref}{sub_item["cursor"]["spelling"]}' + ) + elif item["type"]["kind"] in ("Float", "Double", "Int"): + parameter_type_list = f'{item["type"]["kind"].lower()}' else: - parameter_type_list = f'{item["element_type"]}' + parameter_type_list = f'{item["type"]["kind"]}' return parameter_type_list def handle_inclusion_directive(self) -> None: @@ -417,7 +429,7 @@ def combine_lines() -> list or Exception: if parsed_info: bind_object = bind(root=parsed_info, module_name=module_name) # Extract filename from parsed_info (TRANSLATION_UNIT's name contains the filepath) - filename = "pcl" + parsed_info["name"].rsplit("pcl")[-1] + filename = "pcl" + parsed_info["cursor"]["spelling"].rsplit("pcl")[-1] return combine_lines() else: raise Exception("Empty dict: parsed_info") diff --git a/bindings/python/scripts/parse.py b/bindings/python/scripts/parse.py index 4205de6..246dd94 100644 --- a/bindings/python/scripts/parse.py +++ b/bindings/python/scripts/parse.py @@ -1,286 +1,162 @@ -import os -import sys import clang.cindex as clang from context import scripts import scripts.utils as utils +from scripts.clang_utils import ClangUtils -def valid_children(node): +class Parse: """ - A generator function yielding valid children nodes - - Parameters: - - node (dict): - - The node in the AST - - Keys: - - cursor: The cursor pointing to a node - - filename: - - The file's name to check if the node belongs to it - - Needed to ensure that only symbols belonging to the file gets parsed, not the included files' symbols - - depth: The depth of the node (root=0) - - Yields: - - child_node (dict): Same structure as the argument - """ - - cursor = node["cursor"] - filename = node["filename"] - depth = node["depth"] - - for child in cursor.get_children(): - child_node = {"cursor": child, "filename": filename, "depth": depth + 1} - # Check if the child belongs to the file - if child.location.file and child.location.file.name == filename: - yield (child_node) - - -def print_ast(node): - """ - Prints the AST by recursively traversing it - - Parameters: - - node (dict): - - The node in the AST - - Keys: - - cursor: The cursor pointing to a node - - filename: - - The file's name to check if the node belongs to it - - Needed to ensure that only symbols belonging to the file gets parsed, not the included files' symbols - - depth: The depth of the node (root=0) - - Returns: - - None - """ - - cursor = node["cursor"] - depth = node["depth"] - - print( - "-" * depth, - cursor.location.file, - f"L{cursor.location.line} C{cursor.location.column}", - cursor.kind.name, - cursor.spelling, - ) - - # Get cursor's children and recursively print - for child_node in valid_children(node): - print_ast(child_node) - - -def generate_parsed_info(node): - """ - Generates parsed information by recursively traversing the AST - - Parameters: - - node (dict): - - The node in the AST - - Keys: - - cursor: The cursor pointing to a node - - filename: - - The file's name to check if the node belongs to it - - Needed to ensure that only symbols belonging to the file gets parsed, not the included files' symbols - - depth: The depth of the node (root=0) - - Returns: - - parsed_info (dict): - - Contains key-value pairs of various traits of a node - - The key 'members' contains the node's children's `parsed_info` - """ - - parsed_info = dict() - - cursor = node["cursor"] - depth = node["depth"] - - parsed_info["depth"] = depth - parsed_info["line"] = cursor.location.line - parsed_info["column"] = cursor.location.column - parsed_info["kind"] = cursor.kind.name - parsed_info["tokens"] = [x.spelling for x in cursor.get_tokens()] - - if cursor.is_anonymous(): - parsed_info["kind"] = "ANONYMOUS_" + parsed_info["kind"] - parsed_info["name"] = cursor.spelling - if cursor.type.kind.spelling != "Invalid": - parsed_info["element_type"] = cursor.type.kind.spelling - if cursor.access_specifier.name != "INVALID": - parsed_info["access_specifier"] = cursor.access_specifier.name - if cursor.result_type.spelling != "": - parsed_info["result_type"] = cursor.result_type.spelling - if cursor.brief_comment: - parsed_info["brief_comment"] = cursor.brief_comment - if cursor.raw_comment: - parsed_info["raw_comment"] = cursor.raw_comment - - # add result of various kinds of checks available in cindex.py - - cursorkind_checks = { - "kind_is_declaration": cursor.kind.is_declaration, - "kind_is_reference": cursor.kind.is_reference, - "kind_is_expression": cursor.kind.is_expression, - "kind_is_statement": cursor.kind.is_statement, - "kind_is_attribute": cursor.kind.is_attribute, - "kind_is_invalid": cursor.kind.is_invalid, - "kind_is_translation_unit": cursor.kind.is_translation_unit, - "kind_is_preprocessing": cursor.kind.is_preprocessing, - "kind_is_unexposed": cursor.kind.is_unexposed, - } - - # check for deleted ctor analogous to `is_default_constructor` unavailable - cursor_checks = { - "is_definition": cursor.is_definition, - "is_const_method": cursor.is_const_method, - "is_converting_constructor": cursor.is_converting_constructor, - "is_copy_constructor": cursor.is_copy_constructor, - "is_default_constructor": cursor.is_default_constructor, - "is_move_constructor": cursor.is_move_constructor, - "is_default_method": cursor.is_default_method, - "is_mutable_field": cursor.is_mutable_field, - "is_pure_virtual_method": cursor.is_pure_virtual_method, - "is_static_method": cursor.is_static_method, - "is_virtual_method": cursor.is_virtual_method, - "is_abstract_record": cursor.is_abstract_record, - "is_scoped_enum": cursor.is_scoped_enum, - "is_anonymous": cursor.is_anonymous, - "is_bitfield": cursor.is_bitfield, - } - - type_checks = { - "type_is_const_qualified": cursor.type.is_const_qualified, - "type_is_volatile_qualified": cursor.type.is_volatile_qualified, - "type_is_restrict_qualified": cursor.type.is_restrict_qualified, - "type_is_pod": cursor.type.is_pod, - } - - for checks in (cursorkind_checks, cursor_checks, type_checks): - for check, check_call in checks.items(): - parsed_info[check] = check_call() - - # special case handling for `cursor.type.is_function_variadic()` - if cursor.type.kind.spelling == "FunctionProto": - parsed_info["type_is_function_variadic"] = cursor.type.is_function_variadic() - - parsed_info["members"] = [] - - # Get cursor's children and recursively add their info to a dictionary, as members of the parent - for child_node in valid_children(node): - child_parsed_info = generate_parsed_info(child_node) - parsed_info["members"].append(child_parsed_info) - - return parsed_info - - -def get_compilation_commands(compilation_database_path, filename): - """ - Returns the compilation commands extracted from the compilation database - - Parameters: - - compilation_database_path: The path to `compile_commands.json` - - filename: The file's name to get its compilation commands - - Returns: - - compilation commands (list): The arguments passed to the compiler - """ - - # Build a compilation database found in the given directory - compilation_database = clang.CompilationDatabase.fromDirectory( - buildDir=compilation_database_path - ) - - # Get compiler arguments from the compilation database for the given file - compilation_commands = compilation_database.getCompileCommands(filename=filename) - - """ - - compilation_commands: - - An iterable object providing all the compilation commands available to build filename. - - type: - - compilation_commands[0]: - - Since we have only one command per filename in the compile_commands.json, extract 0th element - - type: - - compilation_commands[0].arguments: - - Get compiler arguments from the CompileCommand object - - type: - - list(compilation_commands[0].arguments)[1:-1]: - - Convert the generator object to list, and extract compiler arguments - - 0th element is the compiler name - - nth element is the filename + Class containing functions to generate an AST of a file and parse it to retrieve relevant information. """ - return list(compilation_commands[0].arguments)[1:-1] + def __init__(self, file, compiler_arguments): + index = clang.Index.create() + """ + - Why parse using the option `PARSE_DETAILED_PROCESSING_RECORD`? + - Indicates that the parser should construct a detailed preprocessing record, + including all macro definitions and instantiations + - Required to retrieve `CursorKind.INCLUSION_DIRECTIVE` + """ -def parse_file(source, compilation_database_path=None): - """ - Returns the parsed_info for a file - - Parameters: - - source: Source to parse - - compilation_database_path: The path to `compile_commands.json` - - Returns: - - parsed_info (dict) - """ - - # Create a new index to start parsing - index = clang.Index.create() - - # Get compiler arguments - compilation_commands = get_compilation_commands( - compilation_database_path=compilation_database_path, - filename=source, - ) - - """ - - Parse the given source code file by running clang and generating the AST before loading - - option `PARSE_DETAILED_PROCESSING_RECORD`: - - Indicates that the parser should construct a detailed preprocessing record, - including all macro definitions and instantiations. - - Required to get the `INCLUSION_DIRECTIVE`s. - """ - source_ast = index.parse( - path=source, - args=compilation_commands, - options=clang.TranslationUnit.PARSE_DETAILED_PROCESSING_RECORD, - ) - - # Dictionary to hold a node's information - root_node = { - "cursor": source_ast.cursor, - "filename": source_ast.spelling, - "depth": 0, - } - - # For testing purposes - # print_ast(root_node) - - return generate_parsed_info(root_node) - - -def main(): - # Get command line arguments - args = utils.parse_arguments(script="parse") - for source in args.files: - source = utils.get_realpath(path=source) - - # Parse the source file - parsed_info = parse_file(source, args.compilation_database_path) - - # Output path for dumping the parsed info into a json file - output_dir = utils.join_path(args.json_output_path, "json") - output_filepath = utils.get_output_path( - source=source, - output_dir=output_dir, - split_from=args.project_root, - extension=".json", + source_ast = index.parse( + path=file, + args=compiler_arguments, + options=clang.TranslationUnit.PARSE_DETAILED_PROCESSING_RECORD, ) - out_rel_path = os.path.relpath(output_filepath, args.json_output_path) - print(f"Producing ./{out_rel_path}") - - # Dump the parsed info at output path - utils.dump_json(filepath=output_filepath, info=parsed_info) + self.root_node = { + "cursor": source_ast.cursor, + "filename": source_ast.spelling, + "depth": 0, + } + + @staticmethod + def is_valid_child(parent_node, child_node): + child = child_node.get("cursor") + parent_filename = parent_node.get("filename") + + if child.location.file and child.location.file.name == parent_filename: + return True + return False + + @staticmethod + def get_parsed_node(node): + cursor = node.get("cursor") + + # Objects to get various kinds of checks available in cindex.py via clang_utils.py + cursor_kind_utils = ClangUtils(cursor.kind) + cursor_utils = ClangUtils(cursor) + cursor_type_utils = ClangUtils(cursor.type) + + parsed_node = { + "depth": node.get("depth"), + "line": cursor.location.line, + "column": cursor.location.column, + "tokens": [x.spelling for x in cursor.get_tokens()], + "cursor_kind": { + **cursor_kind_utils.get_check_functions_dict(), # Functions that begin with "is_" i.e., checking functions + **cursor_kind_utils.get_get_functions_dict(), # Functions that begin with "get_" i.e., getter functions + **cursor_kind_utils.get_properties_dict(), # Properties + }, + "cursor": { + **cursor_utils.get_check_functions_dict(), + **cursor_utils.get_get_functions_dict(), + **cursor_utils.get_properties_dict(), + }, + "type": { + **cursor_type_utils.get_check_functions_dict(), + **cursor_type_utils.get_get_functions_dict(), + **cursor_type_utils.get_properties_dict(), + }, + "members": [], + } + + # HACKY FIXES + # get spelling from object + parsed_node["cursor"]["result_type"] = parsed_node["cursor"][ + "result_type" + ].spelling + # replace `AccessSpecifier.value` with just `value` + parsed_node["cursor"]["access_specifier"] = parsed_node["cursor"][ + "access_specifier" + ].name + # replace `TypeKind.value` with just `value` + parsed_node["type"]["kind"] = parsed_node["type"]["kind"].name + + return parsed_node + + @classmethod + def parse_node_recursive(cls, node): + """ + Generates parsed information by recursively traversing the AST + + Parameters: + - node (dict): + - The node in the AST + - Keys: + - cursor: The cursor pointing to a node + - filename: + - The file's name to check if the node belongs to it + - Needed to ensure that only symbols belonging to the file gets parsed, not the included files' symbols + - depth: The depth of the node (root=0) + + Returns: + - parsed_info (dict): + - Contains key-value pairs of various traits of a node + - The key 'members' contains the node's children's `parsed_info` + """ + + cursor = node.get("cursor") + filename = node.get("filename") + depth = node.get("depth") + + parsed_info = cls.get_parsed_node(node) + + # Get cursor's children and recursively add their info to a dictionary, as members of the parent + for child in cursor.get_children(): + child_node = {"cursor": child, "filename": filename, "depth": depth + 1} + if cls.is_valid_child(node, child_node): + child_parsed_info = cls.parse_node_recursive(child_node) + parsed_info["members"].append(child_parsed_info) + + return parsed_info + + def get_parsed_info(self): + """ + Returns the parsed information for a file by recursively traversing the AST + + Returns: + - parsed_info (dict): + - Contains key-value pairs of various traits of a node + - The key 'members' contains the node's children's `parsed_info` + """ + return self.parse_node_recursive(self.root_node) + + # @TODO: Move the function out of this file in a separate PR + @staticmethod + def get_compilation_arguments(compilation_database_path, filename): + """ + Yields the compilation commands extracted from the compilation database + + Parameters: + - compilation_database_path: The path to `compile_commands.json` + - filename: The file's name to get its compilation commands + + Yields: + - compilation commands (list): The arguments passed to the compiler + """ + + # Build a compilation database found in the given directory + compilation_database = clang.CompilationDatabase.fromDirectory( + buildDir=compilation_database_path + ) + # Get compilation commands from the compilation database for the given file + compilation_commands = compilation_database.getCompileCommands( + filename=filename + ) -if __name__ == "__main__": - main() + for compilation_command in compilation_commands: + # Extract compiler arguments, excluding compiler and filename + yield list(compilation_command.arguments)[1:-1] diff --git a/bindings/python/tests/test_generate.py b/bindings/python/tests/test_generate.py index dd671f1..d64938c 100644 --- a/bindings/python/tests/test_generate.py +++ b/bindings/python/tests/test_generate.py @@ -49,7 +49,7 @@ def generate_bindings(cpp_code_block, module_name, tmp_path): tmp_path=tmp_path, file_contents=cpp_code_block ) - file_include = "pcl" + parsed_info["name"].rsplit("pcl")[-1] + file_include = "pcl" + parsed_info["cursor"]["spelling"].rsplit("pcl")[-1] # Get the binded code binded_code = generate.generate(module_name=module_name, parsed_info=parsed_info) diff --git a/bindings/python/tests/test_parse.py b/bindings/python/tests/test_parse.py index 7b1a791..6ea1846 100644 --- a/bindings/python/tests/test_parse.py +++ b/bindings/python/tests/test_parse.py @@ -1,5 +1,6 @@ from context import scripts -import scripts.parse as parse +import clang.cindex as clang +from scripts.parse import Parse def create_compilation_database(tmp_path, filepath): @@ -24,12 +25,16 @@ def get_parsed_info(tmp_path, file_contents): with open(source_path, "w") as f: f.write(str(file_contents)) - parsed_info = parse.parse_file( - source=str(source_path), - compilation_database_path=create_compilation_database( - tmp_path=tmp_path, filepath=source_path - ), - ) + compiler_arguments = next( + Parse.get_compilation_arguments( + compilation_database_path=create_compilation_database( + tmp_path=tmp_path, filepath=source_path + ), + filename=source_path, + ) + ) + + parsed_info = Parse(source_path, compiler_arguments).get_parsed_info() return parsed_info @@ -46,27 +51,30 @@ def test_anonymous_decls(tmp_path): union_decl = parsed_info["members"][0] - assert union_decl["kind"] == "ANONYMOUS_UNION_DECL" - assert union_decl["name"] == "" + assert union_decl["cursor_kind"]["name"] == "UNION_DECL" + assert union_decl["cursor"]["is_anonymous"] == True + assert union_decl["cursor"]["spelling"] == "" struct_decl = union_decl["members"][0] - assert struct_decl["kind"] == "ANONYMOUS_STRUCT_DECL" - assert struct_decl["name"] == "" + assert struct_decl["cursor_kind"]["name"] == "STRUCT_DECL" + assert union_decl["cursor"]["is_anonymous"] == True + assert union_decl["cursor"]["spelling"] == "" enum_decl = struct_decl["members"][0] - assert enum_decl["kind"] == "ANONYMOUS_ENUM_DECL" - assert enum_decl["name"] == "" + assert enum_decl["cursor_kind"]["name"] == "ENUM_DECL" + assert union_decl["cursor"]["is_anonymous"] == True + assert union_decl["cursor"]["spelling"] == "" def test_translation_unit(tmp_path): file_contents = "" parsed_info = get_parsed_info(tmp_path=tmp_path, file_contents=file_contents) - assert parsed_info["kind"] == "TRANSLATION_UNIT" + assert parsed_info["cursor_kind"]["name"] == "TRANSLATION_UNIT" assert parsed_info["depth"] == 0 - assert parsed_info["name"] == str(tmp_path / "file.cpp") + assert parsed_info["cursor"]["spelling"] == str(tmp_path / "file.cpp") def test_namespace(tmp_path): @@ -75,8 +83,8 @@ def test_namespace(tmp_path): namespace = parsed_info["members"][0] - assert namespace["kind"] == "NAMESPACE" - assert namespace["name"] == "a_namespace" + assert namespace["cursor_kind"]["name"] == "NAMESPACE" + assert namespace["cursor"]["spelling"] == "a_namespace" def test_namespace_ref(tmp_path): @@ -88,14 +96,14 @@ def test_namespace_ref(tmp_path): inclusion_directive = parsed_info["members"][0] - assert inclusion_directive["kind"] == "INCLUSION_DIRECTIVE" - assert inclusion_directive["name"] == "ostream" + assert inclusion_directive["cursor_kind"]["name"] == "INCLUSION_DIRECTIVE" + assert inclusion_directive["cursor"]["spelling"] == "ostream" var_decl = parsed_info["members"][1] namespace_ref = var_decl["members"][0] - assert namespace_ref["kind"] == "NAMESPACE_REF" - assert namespace_ref["name"] == "std" + assert namespace_ref["cursor_kind"]["name"] == "NAMESPACE_REF" + assert namespace_ref["cursor"]["spelling"] == "std" def test_var_decl(tmp_path): @@ -104,9 +112,9 @@ def test_var_decl(tmp_path): var_decl = parsed_info["members"][0] - assert var_decl["kind"] == "VAR_DECL" - assert var_decl["element_type"] == "Int" - assert var_decl["name"] == "anInt" + assert var_decl["cursor_kind"]["name"] == "VAR_DECL" + assert var_decl["type"]["kind"] == "INT" + assert var_decl["cursor"]["spelling"] == "anInt" def test_field_decl(tmp_path): @@ -120,9 +128,9 @@ def test_field_decl(tmp_path): struct_decl = parsed_info["members"][0] field_decl = struct_decl["members"][0] - assert field_decl["kind"] == "FIELD_DECL" - assert field_decl["element_type"] == "Int" - assert field_decl["name"] == "aClassMember" + assert field_decl["cursor_kind"]["name"] == "FIELD_DECL" + assert field_decl["type"]["kind"] == "INT" + assert field_decl["cursor"]["spelling"] == "aClassMember" def test_parsed_info_structure(tmp_path): @@ -142,9 +150,9 @@ def test_function_decl_without_parameters(tmp_path): func_decl = parsed_info["members"][0] - assert func_decl["kind"] == "FUNCTION_DECL" - assert func_decl["name"] == "aFunction" - assert func_decl["result_type"] == "int" + assert func_decl["cursor_kind"]["name"] == "FUNCTION_DECL" + assert func_decl["cursor"]["spelling"] == "aFunction" + assert func_decl["cursor"]["result_type"] == "int" def test_function_decl_with_parameters(tmp_path): @@ -155,18 +163,18 @@ def test_function_decl_with_parameters(tmp_path): func_decl = parsed_info["members"][0] - assert func_decl["kind"] == "FUNCTION_DECL" - assert func_decl["name"] == "aFunction" - assert func_decl["result_type"] == "int" + assert func_decl["cursor_kind"]["name"] == "FUNCTION_DECL" + assert func_decl["cursor"]["spelling"] == "aFunction" + assert func_decl["cursor"]["result_type"] == "int" first_param = func_decl["members"][0] second_param = func_decl["members"][1] - assert first_param["name"] == "firstParam" - assert first_param["element_type"] == "Int" + assert first_param["cursor"]["spelling"] == "firstParam" + assert first_param["type"]["kind"] == "INT" - assert second_param["name"] == "secondParam" - assert second_param["element_type"] == "Double" + assert second_param["cursor"]["spelling"] == "secondParam" + assert second_param["type"]["kind"] == "DOUBLE" def test_simple_call_expr(tmp_path): @@ -181,10 +189,10 @@ def test_simple_call_expr(tmp_path): var_decl = parsed_info["members"][1] call_expr = var_decl["members"][0] - assert call_expr["kind"] == "CALL_EXPR" - assert call_expr["name"] == "aFunction" + assert call_expr["cursor_kind"]["name"] == "CALL_EXPR" + assert call_expr["cursor"]["spelling"] == "aFunction" - assert var_decl["name"] == "anInt" + assert var_decl["cursor"]["spelling"] == "anInt" def test_struct_decl(tmp_path): @@ -193,8 +201,8 @@ def test_struct_decl(tmp_path): struct_decl = parsed_info["members"][0] - assert struct_decl["kind"] == "STRUCT_DECL" - assert struct_decl["name"] == "AStruct" + assert struct_decl["cursor_kind"]["name"] == "STRUCT_DECL" + assert struct_decl["cursor"]["spelling"] == "AStruct" def test_public_inheritance(tmp_path): @@ -207,9 +215,9 @@ def test_public_inheritance(tmp_path): child_struct_decl = parsed_info["members"][1] cxx_base_specifier = child_struct_decl["members"][0] - assert cxx_base_specifier["kind"] == "CXX_BASE_SPECIFIER" - assert cxx_base_specifier["access_specifier"] == "PUBLIC" - assert cxx_base_specifier["name"] == "struct BaseStruct" + assert cxx_base_specifier["cursor_kind"]["name"] == "CXX_BASE_SPECIFIER" + assert cxx_base_specifier["cursor"]["access_specifier"] == "PUBLIC" + assert cxx_base_specifier["cursor"]["spelling"] == "struct BaseStruct" def test_member_function(tmp_path): @@ -223,9 +231,9 @@ def test_member_function(tmp_path): struct_decl = parsed_info["members"][0] cxx_method = struct_decl["members"][0] - assert cxx_method["kind"] == "CXX_METHOD" - assert cxx_method["result_type"] == "void" - assert cxx_method["name"] == "aMethod" + assert cxx_method["cursor_kind"]["name"] == "CXX_METHOD" + assert cxx_method["cursor"]["result_type"] == "void" + assert cxx_method["cursor"]["spelling"] == "aMethod" def test_type_ref(tmp_path): @@ -242,12 +250,12 @@ class AClass { cxx_method = class_decl["members"][0] parm_decl = cxx_method["members"][0] - assert parm_decl["name"] == "aParameter" + assert parm_decl["cursor"]["spelling"] == "aParameter" type_ref = parm_decl["members"][0] - assert type_ref["kind"] == "TYPE_REF" - assert type_ref["name"] == "struct SomeUsefulType" + assert type_ref["cursor_kind"]["name"] == "TYPE_REF" + assert type_ref["cursor"]["spelling"] == "struct SomeUsefulType" def test_simple_constructor(tmp_path): @@ -261,9 +269,9 @@ def test_simple_constructor(tmp_path): struct_decl = parsed_info["members"][0] constructor = struct_decl["members"][0] - assert constructor["kind"] == "CONSTRUCTOR" - assert constructor["access_specifier"] == "PUBLIC" - assert constructor["name"] == "AStruct" + assert constructor["cursor_kind"]["name"] == "CONSTRUCTOR" + assert constructor["cursor"]["access_specifier"] == "PUBLIC" + assert constructor["cursor"]["spelling"] == "AStruct" def test_unexposed_expr(tmp_path): @@ -279,12 +287,12 @@ class SimpleClassWithConstructor { constructor = struct_decl["members"][1] member_ref = constructor["members"][1] - assert member_ref["name"] == "aClassMember" + assert member_ref["cursor"]["spelling"] == "aClassMember" unexposed_expr = constructor["members"][2] - assert unexposed_expr["kind"] == "UNEXPOSED_EXPR" - assert unexposed_expr["name"] == "aConstructorParameter" + assert unexposed_expr["cursor_kind"]["name"] == "UNEXPOSED_EXPR" + assert unexposed_expr["cursor"]["spelling"] == "aConstructorParameter" # @TODO: Not sure how to reproduce. Maybe later. @@ -309,10 +317,10 @@ def test_decl_ref_expr(tmp_path): decl_ref_expr_1 = unexposed_expr_1["members"][0] decl_ref_expr_2 = unexposed_expr_2["members"][0] - assert decl_ref_expr_1["kind"] == "DECL_REF_EXPR" - assert decl_ref_expr_2["kind"] == "DECL_REF_EXPR" - assert decl_ref_expr_1["name"] == "secondFunctionParameter" - assert decl_ref_expr_2["name"] == "firstFunctionParameter" + assert decl_ref_expr_1["cursor_kind"]["name"] == "DECL_REF_EXPR" + assert decl_ref_expr_2["cursor_kind"]["name"] == "DECL_REF_EXPR" + assert decl_ref_expr_1["cursor"]["spelling"] == "secondFunctionParameter" + assert decl_ref_expr_2["cursor"]["spelling"] == "firstFunctionParameter" def test_member_ref(tmp_path): @@ -330,12 +338,12 @@ def test_member_ref(tmp_path): member_ref_1 = constructor["members"][2] member_ref_2 = constructor["members"][4] - assert member_ref_1["kind"] == "MEMBER_REF" - assert member_ref_2["kind"] == "MEMBER_REF" - assert member_ref_1["element_type"] == "Int" - assert member_ref_2["element_type"] == "Int" - assert member_ref_1["name"] == "firstMember" - assert member_ref_2["name"] == "secondMember" + assert member_ref_1["cursor_kind"]["name"] == "MEMBER_REF" + assert member_ref_2["cursor_kind"]["name"] == "MEMBER_REF" + assert member_ref_1["type"]["kind"] == "INT" + assert member_ref_2["type"]["kind"] == "INT" + assert member_ref_1["cursor"]["spelling"] == "firstMember" + assert member_ref_2["cursor"]["spelling"] == "secondMember" def test_class_template(tmp_path): @@ -347,14 +355,14 @@ def test_class_template(tmp_path): class_template = parsed_info["members"][0] - assert class_template["kind"] == "CLASS_TEMPLATE" - assert class_template["name"] == "AStruct" + assert class_template["cursor_kind"]["name"] == "CLASS_TEMPLATE" + assert class_template["cursor"]["spelling"] == "AStruct" template_type_parameter = class_template["members"][0] - assert template_type_parameter["kind"] == "TEMPLATE_TYPE_PARAMETER" - assert template_type_parameter["name"] == "T" - assert template_type_parameter["access_specifier"] == "PUBLIC" + assert template_type_parameter["cursor_kind"]["name"] == "TEMPLATE_TYPE_PARAMETER" + assert template_type_parameter["cursor"]["spelling"] == "T" + assert template_type_parameter["cursor"]["access_specifier"] == "PUBLIC" def test_template_non_type_parameter(tmp_path): @@ -366,14 +374,17 @@ def test_template_non_type_parameter(tmp_path): class_template = parsed_info["members"][0] - assert class_template["kind"] == "CLASS_TEMPLATE" - assert class_template["name"] == "AStruct" + assert class_template["cursor_kind"]["name"] == "CLASS_TEMPLATE" + assert class_template["cursor"]["spelling"] == "AStruct" template_non_type_parameter = class_template["members"][0] - assert template_non_type_parameter["kind"] == "TEMPLATE_NON_TYPE_PARAMETER" - assert template_non_type_parameter["element_type"] == "Int" - assert template_non_type_parameter["name"] == "N" + assert ( + template_non_type_parameter["cursor_kind"]["name"] + == "TEMPLATE_NON_TYPE_PARAMETER" + ) + assert template_non_type_parameter["type"]["kind"] == "INT" + assert template_non_type_parameter["cursor"]["spelling"] == "N" def test_function_template(tmp_path): @@ -385,15 +396,15 @@ def test_function_template(tmp_path): function_template = parsed_info["members"][0] - assert function_template["kind"] == "FUNCTION_TEMPLATE" - assert function_template["result_type"] == "void" - assert function_template["name"] == "aFunction" + assert function_template["cursor_kind"]["name"] == "FUNCTION_TEMPLATE" + assert function_template["cursor"]["result_type"] == "void" + assert function_template["cursor"]["spelling"] == "aFunction" template_type_parameter = function_template["members"][0] - assert template_type_parameter["kind"] == "TEMPLATE_TYPE_PARAMETER" - assert template_type_parameter["name"] == "T" - assert template_type_parameter["access_specifier"] == "PUBLIC" + assert template_type_parameter["cursor_kind"]["name"] == "TEMPLATE_TYPE_PARAMETER" + assert template_type_parameter["cursor"]["spelling"] == "T" + assert template_type_parameter["cursor"]["access_specifier"] == "PUBLIC" def test_template_type_parameter(tmp_path): @@ -409,16 +420,16 @@ def test_template_type_parameter(tmp_path): class_template = parsed_info["members"][0] template_type_parameter = class_template["members"][0] - assert template_type_parameter["kind"] == "TEMPLATE_TYPE_PARAMETER" - assert template_type_parameter["element_type"] == "Unexposed" - assert template_type_parameter["name"] == "T" + assert template_type_parameter["cursor_kind"]["name"] == "TEMPLATE_TYPE_PARAMETER" + assert template_type_parameter["type"]["kind"] == "UNEXPOSED" + assert template_type_parameter["cursor"]["spelling"] == "T" function_template = parsed_info["members"][1] template_type_parameter = function_template["members"][0] - assert template_type_parameter["kind"] == "TEMPLATE_TYPE_PARAMETER" - assert template_type_parameter["element_type"] == "Unexposed" - assert template_type_parameter["name"] == "P" + assert template_type_parameter["cursor_kind"]["name"] == "TEMPLATE_TYPE_PARAMETER" + assert template_type_parameter["type"]["kind"] == "UNEXPOSED" + assert template_type_parameter["cursor"]["spelling"] == "P" def test_default_delete_constructor(tmp_path): @@ -436,14 +447,14 @@ class aClass { default_constructor = class_decl["members"][0] - assert default_constructor["kind"] == "CONSTRUCTOR" - assert default_constructor["name"] == "aClass" - assert default_constructor["result_type"] == "void" - assert default_constructor["is_default_constructor"] + assert default_constructor["cursor_kind"]["name"] == "CONSTRUCTOR" + assert default_constructor["cursor"]["spelling"] == "aClass" + assert default_constructor["cursor"]["result_type"] == "void" + assert default_constructor["cursor"]["is_default_constructor"] delete_constructor = class_decl["members"][1] - assert delete_constructor["kind"] == "CONSTRUCTOR" - assert delete_constructor["name"] == "aClass" - assert delete_constructor["result_type"] == "void" + assert delete_constructor["cursor_kind"]["name"] == "CONSTRUCTOR" + assert delete_constructor["cursor"]["spelling"] == "aClass" + assert delete_constructor["cursor"]["result_type"] == "void" # no check available for deleted ctor analogous to `is_default_constructor` From f0edede37170ae85e0c48744c1f986bba19472c5 Mon Sep 17 00:00:00 2001 From: Divyanshu Madan Date: Sun, 28 Feb 2021 04:15:26 +0530 Subject: [PATCH 11/19] [pytest.yml] bump clang to v12 and ubuntu to 20.04 --- .github/workflows/pytest.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 756e9e4..73056d2 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -6,7 +6,7 @@ on: [push, pull_request] jobs: Pytest: # The type of runner that the job will run on - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: matrix: python-version: [3.6, 3.7, 3.8, 3.9] @@ -34,12 +34,12 @@ jobs: - name: Add llvm keys run: | wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - - echo 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-11 main' | sudo tee -a /etc/apt/sources.list - echo 'deb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic-11 main' | sudo tee -a /etc/apt/sources.list + echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-12 main' | sudo tee -a /etc/apt/sources.list + echo 'deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-12 main' | sudo tee -a /etc/apt/sources.list - name: Install libclang and its python bindings run: | sudo apt-get update - sudo apt-get install -y libclang-11-dev python3-clang-11 + sudo apt-get install -y libclang-12-dev python3-clang-12 # Add dist-package to path to enable apt installed python3-clang import - name: Add dist-packages to PYTHONPATH From 838ac8709ffae84742ba37a39a0ab3bf9727da72 Mon Sep 17 00:00:00 2001 From: Divyanshu Madan Date: Fri, 4 Jun 2021 03:50:53 +0530 Subject: [PATCH 12/19] [compilation_database.py] add compilation_database.py --- .../python/scripts/compilation_database.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 bindings/python/scripts/compilation_database.py diff --git a/bindings/python/scripts/compilation_database.py b/bindings/python/scripts/compilation_database.py new file mode 100644 index 0000000..7883ac6 --- /dev/null +++ b/bindings/python/scripts/compilation_database.py @@ -0,0 +1,40 @@ +import clang.cindex as clang + + +class CompilationDatabase: + """ + Build a compilation database from a given directory + """ + + def __init__(self, compilation_database_path): + self.compilation_database = clang.CompilationDatabase.fromDirectory( + buildDir=compilation_database_path + ) + + def get_compilation_arguments(self, filename=None): + """ + Returns the compilation commands extracted from the compilation database + + Parameters: + - compilation_database_path: The path to `compile_commands.json` + - filename (optional): To get compilaton commands of a file + + Returns: + - compilation_arguments (dict): {filename: compiler arguments} + """ + + if filename: + # Get compilation commands from the compilation database for the given file + compilation_commands = self.compilation_database.getCompileCommands( + filename=filename + ) + else: + # Get all compilation commands from the compilation database + compilation_commands = self.compilation_database.getAllCompileCommands() + + # {file: compiler arguments} + compilation_arguments = { + command.filename: list(command.arguments)[1:-1] + for command in compilation_commands + } + return compilation_arguments From 6f4c13e31773e01a75e35ad431e88085d739b924 Mon Sep 17 00:00:00 2001 From: Divyanshu Madan Date: Fri, 4 Jun 2021 03:50:28 +0530 Subject: [PATCH 13/19] update compilation db logic --- bindings/python/scripts/parse.py | 31 ++--------------------------- bindings/python/tests/test_parse.py | 22 ++++++++++++-------- 2 files changed, 16 insertions(+), 37 deletions(-) diff --git a/bindings/python/scripts/parse.py b/bindings/python/scripts/parse.py index 246dd94..fd378b3 100644 --- a/bindings/python/scripts/parse.py +++ b/bindings/python/scripts/parse.py @@ -33,7 +33,7 @@ def __init__(self, file, compiler_arguments): } @staticmethod - def is_valid_child(parent_node, child_node): + def _is_valid_child(parent_node, child_node): child = child_node.get("cursor") parent_filename = parent_node.get("filename") @@ -117,7 +117,7 @@ def parse_node_recursive(cls, node): # Get cursor's children and recursively add their info to a dictionary, as members of the parent for child in cursor.get_children(): child_node = {"cursor": child, "filename": filename, "depth": depth + 1} - if cls.is_valid_child(node, child_node): + if cls._is_valid_child(node, child_node): child_parsed_info = cls.parse_node_recursive(child_node) parsed_info["members"].append(child_parsed_info) @@ -133,30 +133,3 @@ def get_parsed_info(self): - The key 'members' contains the node's children's `parsed_info` """ return self.parse_node_recursive(self.root_node) - - # @TODO: Move the function out of this file in a separate PR - @staticmethod - def get_compilation_arguments(compilation_database_path, filename): - """ - Yields the compilation commands extracted from the compilation database - - Parameters: - - compilation_database_path: The path to `compile_commands.json` - - filename: The file's name to get its compilation commands - - Yields: - - compilation commands (list): The arguments passed to the compiler - """ - - # Build a compilation database found in the given directory - compilation_database = clang.CompilationDatabase.fromDirectory( - buildDir=compilation_database_path - ) - # Get compilation commands from the compilation database for the given file - compilation_commands = compilation_database.getCompileCommands( - filename=filename - ) - - for compilation_command in compilation_commands: - # Extract compiler arguments, excluding compiler and filename - yield list(compilation_command.arguments)[1:-1] diff --git a/bindings/python/tests/test_parse.py b/bindings/python/tests/test_parse.py index 6ea1846..05bf7c3 100644 --- a/bindings/python/tests/test_parse.py +++ b/bindings/python/tests/test_parse.py @@ -1,9 +1,10 @@ from context import scripts import clang.cindex as clang from scripts.parse import Parse +from scripts.compilation_database import CompilationDatabase -def create_compilation_database(tmp_path, filepath): +def get_compilation_database_path(tmp_path, filepath): input = tmp_path / "compile_commands.json" x = [ { @@ -25,15 +26,20 @@ def get_parsed_info(tmp_path, file_contents): with open(source_path, "w") as f: f.write(str(file_contents)) - compiler_arguments = next( - Parse.get_compilation_arguments( - compilation_database_path=create_compilation_database( - tmp_path=tmp_path, filepath=source_path - ), - filename=source_path, - ) + compilation_database_path = get_compilation_database_path( + tmp_path=tmp_path, filepath=source_path ) + compilation_database = CompilationDatabase( + compilation_database_path=compilation_database_path + ) + + compilation_arguments = compilation_database.get_compilation_arguments( + filename=source_path + ) + + compiler_arguments = compilation_arguments.get(source_path) + parsed_info = Parse(source_path, compiler_arguments).get_parsed_info() return parsed_info From de695adb549ad197478962a3c86198b7527fac04 Mon Sep 17 00:00:00 2001 From: Kunal Tyagi Date: Sun, 25 Oct 2020 03:44:28 +0900 Subject: [PATCH 14/19] Add class explicitly to the tests --- bindings/python/tests/test_parse.py | 35 ++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/bindings/python/tests/test_parse.py b/bindings/python/tests/test_parse.py index 05bf7c3..9414d5a 100644 --- a/bindings/python/tests/test_parse.py +++ b/bindings/python/tests/test_parse.py @@ -356,10 +356,24 @@ def test_class_template(tmp_path): file_contents = """ template struct AStruct {}; + + template + class AClass {}; """ parsed_info = get_parsed_info(tmp_path=tmp_path, file_contents=file_contents) - class_template = parsed_info["members"][0] + struct_template = parsed_info["members"][0] + + assert struct_template["kind"] == "CLASS_TEMPLATE" + assert struct_template["name"] == "AStruct" + + template_type_parameter = struct_template["members"][0] + + assert template_type_parameter["kind"] == "TEMPLATE_TYPE_PARAMETER" + assert template_type_parameter["name"] == "T" + assert template_type_parameter["access_specifier"] == "PUBLIC" + + class_template = parsed_info["members"][1] assert class_template["cursor_kind"]["name"] == "CLASS_TEMPLATE" assert class_template["cursor"]["spelling"] == "AStruct" @@ -371,6 +385,7 @@ def test_class_template(tmp_path): assert template_type_parameter["cursor"]["access_specifier"] == "PUBLIC" + def test_template_non_type_parameter(tmp_path): file_contents = """ template @@ -413,24 +428,34 @@ def test_function_template(tmp_path): assert template_type_parameter["cursor"]["access_specifier"] == "PUBLIC" -def test_template_type_parameter(tmp_path): +def test_template_type_single_parameter(tmp_path): file_contents = """ template struct AStruct {}; + template + class AClass {}; + template void aFunction() {} """ parsed_info = get_parsed_info(tmp_path=tmp_path, file_contents=file_contents) - class_template = parsed_info["members"][0] - template_type_parameter = class_template["members"][0] + struct_template = parsed_info["members"][0] + template_type_parameter = struct_template["members"][0] assert template_type_parameter["cursor_kind"]["name"] == "TEMPLATE_TYPE_PARAMETER" assert template_type_parameter["type"]["kind"] == "UNEXPOSED" assert template_type_parameter["cursor"]["spelling"] == "T" - function_template = parsed_info["members"][1] + class_template = parsed_info["members"][1] + template_type_parameter = class_template["members"][0] + + assert template_type_parameter["kind"] == "TEMPLATE_TYPE_PARAMETER" + assert template_type_parameter["element_type"] == "Unexposed" + assert template_type_parameter["name"] == "U" + + function_template = parsed_info["members"][2] template_type_parameter = function_template["members"][0] assert template_type_parameter["cursor_kind"]["name"] == "TEMPLATE_TYPE_PARAMETER" From 721bbe0584be5966d74aedc4a8a82c23712c377e Mon Sep 17 00:00:00 2001 From: Kunal Tyagi Date: Sun, 25 Oct 2020 03:44:43 +0900 Subject: [PATCH 15/19] Add tests for instantiation parsing --- bindings/python/tests/test_parse.py | 31 +++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/bindings/python/tests/test_parse.py b/bindings/python/tests/test_parse.py index 9414d5a..254cbb3 100644 --- a/bindings/python/tests/test_parse.py +++ b/bindings/python/tests/test_parse.py @@ -463,6 +463,37 @@ class AClass {}; assert template_type_parameter["cursor"]["spelling"] == "P" +def test_template_instantiate_single_parameter(tmp_path): + file_contents = """ + template + struct AStruct; + template + class AClass; + + template + void aFunction(bool); + """ + parsed_info = get_parsed_info(tmp_path=tmp_path, file_contents=file_contents) + print(parsed_info["members"]) + # As per generate.py:219, there should be some member with "kind" `TYPE_REF` + print( + "Member details:", + {sub_item["name"]: sub_item["members"] for sub_item in parsed_info["members"]}, + ) + # Doesn't detect 3 items in next line + assert len(parsed_info["members"]) == 3 + + struct_inst = parsed_info["members"][0] + class_inst = parsed_info["members"][1] + func_inst = parsed_info["members"][2] + + assert struct_inst["kind"] == "STRUCT_DECL" + assert struct_inst["kind_is_declaration"] == True + + assert class_inst["kind"] == "CLASS_DEC" + assert class_inst["kind_is_declaration"] == True + + def test_default_delete_constructor(tmp_path): file_contents = """ class aClass { From 647b0e5d64d16169636b74f955a946e835f4d958 Mon Sep 17 00:00:00 2001 From: Kunal Tyagi Date: Sun, 25 Oct 2020 05:35:53 +0900 Subject: [PATCH 16/19] Satisfy black --- bindings/python/tests/test_parse.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bindings/python/tests/test_parse.py b/bindings/python/tests/test_parse.py index 254cbb3..11abc3e 100644 --- a/bindings/python/tests/test_parse.py +++ b/bindings/python/tests/test_parse.py @@ -385,7 +385,6 @@ class AClass {}; assert template_type_parameter["cursor"]["access_specifier"] == "PUBLIC" - def test_template_non_type_parameter(tmp_path): file_contents = """ template From b990c30ec111cb1439b4a35a7824ec2e79b0e355 Mon Sep 17 00:00:00 2001 From: Kunal Tyagi Date: Sun, 25 Oct 2020 16:02:03 +0900 Subject: [PATCH 17/19] Update just in case the ill-formed nature of previous example was the cause --- bindings/python/tests/test_parse.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/bindings/python/tests/test_parse.py b/bindings/python/tests/test_parse.py index 11abc3e..6dc5214 100644 --- a/bindings/python/tests/test_parse.py +++ b/bindings/python/tests/test_parse.py @@ -464,34 +464,45 @@ class AClass {}; def test_template_instantiate_single_parameter(tmp_path): file_contents = """ + template + struct AStruct { T value; }; + template + class AClass { const T privateValue; } + template + T aFunction(T&& aValue) { return aValue; } + template struct AStruct; template class AClass; - template - void aFunction(bool); + bool aFunction(bool&&); """ parsed_info = get_parsed_info(tmp_path=tmp_path, file_contents=file_contents) print(parsed_info["members"]) # As per generate.py:219, there should be some member with "kind" `TYPE_REF` print( "Member details:", - {sub_item["name"]: sub_item["members"] for sub_item in parsed_info["members"]}, + [{sub_item["name"]: sub_item["kind"]} for sub_item in parsed_info["members"]], ) # Doesn't detect 3 items in next line - assert len(parsed_info["members"]) == 3 + assert len(parsed_info["members"]) == 6 struct_inst = parsed_info["members"][0] - class_inst = parsed_info["members"][1] - func_inst = parsed_info["members"][2] assert struct_inst["kind"] == "STRUCT_DECL" assert struct_inst["kind_is_declaration"] == True - assert class_inst["kind"] == "CLASS_DEC" + class_inst = parsed_info["members"][1] + + assert class_inst["kind"] == "CLASS_DECL" assert class_inst["kind_is_declaration"] == True + func_inst = parsed_info["members"][2] + + assert func_inst["kind"] == "FUNCTION_DECL" + assert func_inst["kind_is_declaration"] == True + def test_default_delete_constructor(tmp_path): file_contents = """ From dcac8f9a0c0bb6f9e741f87764b6553761f793e7 Mon Sep 17 00:00:00 2001 From: Divyanshu Madan Date: Thu, 17 Jun 2021 00:37:17 +0530 Subject: [PATCH 18/19] [test_parse.py] changes coming from clang_utils.py --- bindings/python/tests/test_parse.py | 52 +++++++++++++---------------- 1 file changed, 23 insertions(+), 29 deletions(-) diff --git a/bindings/python/tests/test_parse.py b/bindings/python/tests/test_parse.py index 6dc5214..dafc4ef 100644 --- a/bindings/python/tests/test_parse.py +++ b/bindings/python/tests/test_parse.py @@ -364,24 +364,24 @@ class AClass {}; struct_template = parsed_info["members"][0] - assert struct_template["kind"] == "CLASS_TEMPLATE" - assert struct_template["name"] == "AStruct" + assert struct_template["cursor_kind"]["name"] == "CLASS_TEMPLATE" + assert struct_template["cursor"]["spelling"] == "AStruct" template_type_parameter = struct_template["members"][0] - assert template_type_parameter["kind"] == "TEMPLATE_TYPE_PARAMETER" - assert template_type_parameter["name"] == "T" - assert template_type_parameter["access_specifier"] == "PUBLIC" + assert template_type_parameter["cursor_kind"]["name"] == "TEMPLATE_TYPE_PARAMETER" + assert template_type_parameter["cursor"]["spelling"] == "T" + assert template_type_parameter["cursor"]["access_specifier"] == "PUBLIC" class_template = parsed_info["members"][1] assert class_template["cursor_kind"]["name"] == "CLASS_TEMPLATE" - assert class_template["cursor"]["spelling"] == "AStruct" + assert class_template["cursor"]["spelling"] == "AClass" template_type_parameter = class_template["members"][0] assert template_type_parameter["cursor_kind"]["name"] == "TEMPLATE_TYPE_PARAMETER" - assert template_type_parameter["cursor"]["spelling"] == "T" + assert template_type_parameter["cursor"]["spelling"] == "U" assert template_type_parameter["cursor"]["access_specifier"] == "PUBLIC" @@ -450,9 +450,9 @@ class AClass {}; class_template = parsed_info["members"][1] template_type_parameter = class_template["members"][0] - assert template_type_parameter["kind"] == "TEMPLATE_TYPE_PARAMETER" - assert template_type_parameter["element_type"] == "Unexposed" - assert template_type_parameter["name"] == "U" + assert template_type_parameter["cursor_kind"]["name"] == "TEMPLATE_TYPE_PARAMETER" + assert template_type_parameter["type"]["kind"] == "UNEXPOSED" + assert template_type_parameter["cursor"]["spelling"] == "U" function_template = parsed_info["members"][2] template_type_parameter = function_template["members"][0] @@ -467,8 +467,8 @@ def test_template_instantiate_single_parameter(tmp_path): template struct AStruct { T value; }; template - class AClass { const T privateValue; } - template + class AClass { const T privateValue; }; + template T aFunction(T&& aValue) { return aValue; } template @@ -479,29 +479,23 @@ class AClass; bool aFunction(bool&&); """ parsed_info = get_parsed_info(tmp_path=tmp_path, file_contents=file_contents) - print(parsed_info["members"]) - # As per generate.py:219, there should be some member with "kind" `TYPE_REF` - print( - "Member details:", - [{sub_item["name"]: sub_item["kind"]} for sub_item in parsed_info["members"]], - ) - # Doesn't detect 3 items in next line - assert len(parsed_info["members"]) == 6 - struct_inst = parsed_info["members"][0] + parsed_info = parsed_info["members"][0] - assert struct_inst["kind"] == "STRUCT_DECL" - assert struct_inst["kind_is_declaration"] == True + struct_inst = parsed_info["members"][3] - class_inst = parsed_info["members"][1] + assert struct_inst["cursor_kind"]["name"] == "STRUCT_DECL" + assert struct_inst["cursor_kind"]["is_declaration"] == True - assert class_inst["kind"] == "CLASS_DECL" - assert class_inst["kind_is_declaration"] == True + class_inst = parsed_info["members"][4] - func_inst = parsed_info["members"][2] + assert class_inst["cursor_kind"]["name"] == "CLASS_DECL" + assert class_inst["cursor_kind"]["is_declaration"] == True - assert func_inst["kind"] == "FUNCTION_DECL" - assert func_inst["kind_is_declaration"] == True + func_inst = parsed_info["members"][5] + + assert func_inst["cursor_kind"]["name"] == "FUNCTION_DECL" + assert func_inst["cursor_kind"]["is_declaration"] == True def test_default_delete_constructor(tmp_path): From 36350bff3b13ad9282caca6a38f398547c743f67 Mon Sep 17 00:00:00 2001 From: Divyanshu Madan Date: Thu, 17 Jun 2021 00:37:45 +0530 Subject: [PATCH 19/19] [clang_utils.py] add objc_type_encoding to ignore list --- bindings/python/scripts/clang_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bindings/python/scripts/clang_utils.py b/bindings/python/scripts/clang_utils.py index f9e00c8..6ad11bc 100644 --- a/bindings/python/scripts/clang_utils.py +++ b/bindings/python/scripts/clang_utils.py @@ -74,6 +74,7 @@ def __init__(self, object): "get_address_space", "get_typedef_name", "tls_kind", + "objc_type_encoding" ] # populate dicts