From fb04e7fbe8a8d43c78c9c10f89d39fac095c2519 Mon Sep 17 00:00:00 2001 From: Divyanshu Madan Date: Sat, 14 Aug 2021 20:31:38 +0530 Subject: [PATCH 1/9] [cmake_frontend.py] add get_library_targets; add some documentation --- clang_bind/cmake_frontend.py | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/clang_bind/cmake_frontend.py b/clang_bind/cmake_frontend.py index b3d5bf6..ca6bbe3 100644 --- a/clang_bind/cmake_frontend.py +++ b/clang_bind/cmake_frontend.py @@ -4,7 +4,11 @@ class CompilationDatabase: - """Class to get information from a CMake compilation database.""" + """Class to get information from a CMake compilation database. + + :param build_dir: Build directory path, where compile_commands.json is present. + :type build_dir: str + """ def __init__(self, build_dir): self.compilation_database = clang.CompilationDatabase.fromDirectory( @@ -36,7 +40,11 @@ def get_compilation_arguments(self, filename=None): class Target: - """Class to get information about targets found from the CMake file API.""" + """Class to get information about targets found from the CMake file API. + + :param target_file: Target file path. + :type target_file: str + """ def __init__(self, target_file): with open(target_file) as f: @@ -196,7 +204,11 @@ def get_type(self): class CMakeFileAPI: - """CMake File API front end.""" + """CMake File API front end. + + :param build_dir: Build directory path, where .cmake directory is present. + :type build_dir: str + """ def __init__(self, build_dir): self.reply_dir = Path(build_dir, ".cmake", "api", "v1", "reply") @@ -220,6 +232,17 @@ def _set_targets_from_codemodel(self): target_obj = Target(Path(self.reply_dir, target_file)) self.targets[target_obj.get_name()] = target_obj + def get_library_targets(self): + """Get all library targets' names. + + :return: Library targets. + :rtype: list + """ + library_targets_objs = filter( + lambda target: target.get_type() == "SHARED_LIBRARY", self.targets.values() + ) + return list(map(lambda target: target.get_name(), library_targets_objs)) + def get_dependencies(self, target=None): """Get dependencies of the target(s). From 31fd0b64da735bfab6a41214b1f5d364145ad706 Mon Sep 17 00:00:00 2001 From: Divyanshu Madan Date: Sun, 15 Aug 2021 14:55:44 +0530 Subject: [PATCH 2/9] [cmake_frontend.py] change parameter optionality; ret type of functions - makes easier for the user, doesn't have to access dict values in the case of single use. --- clang_bind/cmake_frontend.py | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/clang_bind/cmake_frontend.py b/clang_bind/cmake_frontend.py index ca6bbe3..9524e86 100644 --- a/clang_bind/cmake_frontend.py +++ b/clang_bind/cmake_frontend.py @@ -243,29 +243,24 @@ def get_library_targets(self): ) return list(map(lambda target: target.get_name(), library_targets_objs)) - def get_dependencies(self, target=None): - """Get dependencies of the target(s). + def get_dependencies(self, target): + """Get dependencies of the target. - :param target: Target to get the dependencies, defaults to None - :type target: str, optional - :return: Dependencies of the target(s). - :rtype: dict + :param target: Target to get the dependencies of. + :type target: str + :return: Dependencies of the target. + :rtype: list """ - targets = [self.targets.get(target)] if target else self.targets.values() - return { - target.get_name(): list( - map(lambda x: x.split("::")[0], target.get_dependencies()) - ) - for target in targets - } + return list( + map(lambda x: x.split("::")[0], self.targets.get(target).get_dependencies()) + ) - def get_sources(self, target=None): + def get_sources(self, target): """Get sources of the target(s). - :param target: Target to get the dependencies, defaults to None - :type target: str, optional - :return: Sources of the target(s). - :rtype: dict + :param target: Target to get the sources of. + :type target: str + :return: Sources of the target. + :rtype: list """ - targets = [self.targets.get(target)] if target else self.targets.values() - return {target.get_name(): target.get_sources() for target in targets} + return self.targets.get(target).get_sources() From 6a5933de2e6b6dead696ba8539af20301a7a4ece Mon Sep 17 00:00:00 2001 From: Divyanshu Madan Date: Sun, 15 Aug 2021 19:32:40 +0530 Subject: [PATCH 3/9] [bind.py] bind.py --- clang_bind/bind.py | 90 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 clang_bind/bind.py diff --git a/clang_bind/bind.py b/clang_bind/bind.py new file mode 100644 index 0000000..c902ebb --- /dev/null +++ b/clang_bind/bind.py @@ -0,0 +1,90 @@ +from pathlib import Path + +from clang_bind.cmake_frontend import CMakeFileAPI, CompilationDatabase +from clang_bind.parse import Parse + + +class Bind: + """Class to bind cpp modules. + + :param source_dir: Source dir of cpp library. + :type source_dir: str + :param build_dir: CMake build dir of cpp library, containing .cmake dir or compile_commands.json + :type build_dir: str + :param output_dir: Output dir + :type output_dir: str + :param output_module_name: Module name in python + :type output_module_name: str + :param cpp_modules: List of cpp modules to bind, defaults to []: bind all. + :type cpp_modules: list, optional + :param allow_inclusions_from_other_modules: Allow inclusions from other modules, which are not specified in cpp_modules, defaults to True + :type allow_inclusions_from_other_modules: bool, optional + """ + + def __init__( + self, + source_dir, + build_dir, + output_dir, + output_module_name, + cpp_modules=[], + allow_inclusions_from_other_modules=True, + ): + all_cpp_modules = CMakeFileAPI(build_dir).get_library_targets() + if not cpp_modules: + cpp_modules = all_cpp_modules # bind all cpp modules + + all_inclusion_sources = [] + for module in all_cpp_modules: # for all cpp modules, populate the variable + sources = CMakeFileAPI(build_dir).get_sources(module) # module's sources + cpp_sources = list( + filter(lambda source: source.endswith(".cpp"), sources) + ) # sources ending with .cpp + all_inclusion_sources += list( + set(sources) - set(cpp_sources) + ) # other sources like .h and .hpp files + + self.binding_db = {} # binding database + for module in cpp_modules: + sources = CMakeFileAPI(build_dir).get_sources(module) # module's sources + cpp_sources = list( + filter(lambda source: source.endswith(".cpp"), sources) + ) # sources ending with .cpp + inclusion_sources = ( + all_inclusion_sources + if allow_inclusions_from_other_modules + else list(set(sources) - set(cpp_sources)) + ) # other sources like .h and .hpp files + + self.binding_db[module] = { + "inclusion_sources": inclusion_sources, # inclusions for the module + "files": [ # list of files' information + { + "source_path": Path(source_dir, cpp_source), + "output_path": Path(output_dir, cpp_source), + "compiler_arguments": CompilationDatabase( + build_dir + ).get_compilation_arguments(cpp_source), + } + for cpp_source in cpp_sources + ], + } + + def _parse(self): + """For all input files, get the parsed tree and update the db.""" + for module in self.binding_db.values(): + for file in module.get("files"): + # for each file, update the db with the parsed tree + file.update( + { + "parsed_tree": Parse( + file.get("source_path"), + module.get("inclusion_sources"), + file.get("compiler_arguments"), + ).get_tree() + } + ) + + def bind(self): + """Function to bind the input files.""" + self._parse() From 64bea429863552299d0c9f2aecb4ae214325b21a Mon Sep 17 00:00:00 2001 From: Divyanshu Madan Date: Sun, 15 Aug 2021 23:58:14 +0530 Subject: [PATCH 4/9] [cmake_frontend.py] change parameter optionality; return type of function `get_compilation_arguments` --- clang_bind/cmake_frontend.py | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/clang_bind/cmake_frontend.py b/clang_bind/cmake_frontend.py index 9524e86..cdf770a 100644 --- a/clang_bind/cmake_frontend.py +++ b/clang_bind/cmake_frontend.py @@ -15,28 +15,19 @@ def __init__(self, build_dir): buildDir=build_dir ) - def get_compilation_arguments(self, filename=None): - """Returns the compilation commands extracted from the compilation database + def get_compilation_arguments(self, filename): + """Returns the compilation commands extracted from the compilation database. - :param filename: Get compilation arguments of the file, defaults to None: get for all files - :type filename: str, optional - :return: ilenames and their compiler arguments: {filename: compiler arguments} - :rtype: dict + :param filename: Get compilation arguments of the file. + :type filename: str + :return: Compiler arguments. + :rtype: list """ - if filename: - # Get compilation commands from the compilation database for the given file - compilation_commands = self.compilation_database.getCompileCommands( - filename=filename - ) - else: - # Get all compilation commands from the compilation database - compilation_commands = self.compilation_database.getAllCompileCommands() - - return { - command.filename: list(command.arguments)[1:-1] - for command in compilation_commands - } + compilation_arguments = [] + for command in self.compilation_database.getCompileCommands(filename=filename): + compilation_arguments += list(command.arguments)[1:-1] + return compilation_arguments class Target: From 279fa3c53f6273b6cfe6741d8ffcd2e0d27c5121 Mon Sep 17 00:00:00 2001 From: Divyanshu Madan Date: Wed, 18 Aug 2021 02:52:12 +0530 Subject: [PATCH 5/9] [parse.py] add support for inclusion sources - `is_cursor_in_file` -> `is_cursor_in_files`: support multiple files - update `_is_valid_child` to also check for inclusion sources too --- clang_bind/parse.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/clang_bind/parse.py b/clang_bind/parse.py index f745e0b..6664b9d 100644 --- a/clang_bind/parse.py +++ b/clang_bind/parse.py @@ -35,7 +35,8 @@ class Parse: :type compiler_arguments: list, optional """ - def __init__(self, file, compiler_arguments=[]): + def __init__(self, file, inclusion_sources=[], compiler_arguments=[]): + self.inclusion_sources = inclusion_sources self._parsed_info_map = {} index = clang.Index.create() """ @@ -57,27 +58,31 @@ def __init__(self, file, compiler_arguments=[]): self._construct_tree(self.root_node) @staticmethod - def is_cursor_in_file(cursor, filename): - """Checks if the cursor belongs in the file. + def is_cursor_in_files(cursor, files): + """Checks if the cursor belongs in the files. :param cursor: An object of :class:`clang.cindex.Cursor` :type cursor: class:`clang.cindex.Cursor` - :param filename: Filename to search the cursor - :type filename: str - :return: `True` if cursor in file, else `False` + :param files: Filepaths to search the cursor + :type files: list + :return: `True` if cursor in files, else `False` :rtype: bool """ - return cursor.location.file and cursor.location.file.name == filename + return cursor.location.file and cursor.location.file.name in files def _is_valid_child(self, child_cursor): - """Checks if the child is valid (child should be in the same file as the parent). + """Checks if the child is valid: + - Either child should be in the same file as the parent or, + - the child should be in the list of valid inclusion sources :param child_cursor: The child cursor to check, an object of :class:`clang.cindex.Cursor` :type child_cursor: class:`clang.cindex.Cursor` :return: `True` if child cursor in file, else `False` :rtype: bool """ - return self.is_cursor_in_file(child_cursor, self.filename) + return self.is_cursor_in_files( + child_cursor, [self.filename, *self.inclusion_sources] + ) def _construct_tree(self, node): """Recursively generates tree by traversing the AST of the node. From 29c9b0bf1ed359c75f88e3a4445fb0b66dcdb5e6 Mon Sep 17 00:00:00 2001 From: divmad Date: Wed, 18 Aug 2021 02:55:56 +0530 Subject: [PATCH 6/9] fixup! [bind.py] bind.py --- clang_bind/bind.py | 42 ++++++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/clang_bind/bind.py b/clang_bind/bind.py index c902ebb..3be046a 100644 --- a/clang_bind/bind.py +++ b/clang_bind/bind.py @@ -57,11 +57,12 @@ def __init__( ) # other sources like .h and .hpp files self.binding_db[module] = { + "source_dir": source_dir, # source dir containing cpp modules + "output_dir": output_dir, # output dir "inclusion_sources": inclusion_sources, # inclusions for the module "files": [ # list of files' information { - "source_path": Path(source_dir, cpp_source), - "output_path": Path(output_dir, cpp_source), + "source": cpp_source, "compiler_arguments": CompilationDatabase( build_dir ).get_compilation_arguments(cpp_source), @@ -73,17 +74,34 @@ def __init__( def _parse(self): """For all input files, get the parsed tree and update the db.""" for module in self.binding_db.values(): + source_dir = module.get("source_dir") + inclusion_sources = [ + str(Path(source_dir, inclusion_source)) + for inclusion_source in module.get("inclusion_sources") + ] # string full paths of inclusion sources + for file in module.get("files"): - # for each file, update the db with the parsed tree - file.update( - { - "parsed_tree": Parse( - file.get("source_path"), - module.get("inclusion_sources"), - file.get("compiler_arguments"), - ).get_tree() - } - ) + parsed_tree = Parse( + Path(source_dir, file.get("source")), + inclusion_sources, + file.get("compiler_arguments"), + ).get_tree() + + file.update({"parsed_tree": parsed_tree}) # update db + + # Debugging: + # + # - To print the trees: + # parsed_tree.show() + # + # - To save the JSONs: + # json_output_path = Path( + # module.get("output_dir"), + # "parsed", + # file.get("source").replace(".cpp", ".json"), + # ) + # json_output_path.parent.mkdir(parents=True, exist_ok=True) + # parsed_tree.save2file(json_output_path) def bind(self): """Function to bind the input files.""" From 2502c9ce91a102fe070fb9f2bcf2940f07549e63 Mon Sep 17 00:00:00 2001 From: divmad Date: Wed, 18 Aug 2021 03:48:40 +0530 Subject: [PATCH 7/9] fixup! [bind.py] bind.py --- clang_bind/bind.py | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/clang_bind/bind.py b/clang_bind/bind.py index 3be046a..248dcb9 100644 --- a/clang_bind/bind.py +++ b/clang_bind/bind.py @@ -5,7 +5,7 @@ class Bind: - """Class to bind cpp modules. + """Class to bind C++ targets. :param source_dir: Source dir of cpp library. :type source_dir: str @@ -15,10 +15,10 @@ class Bind: :type output_dir: str :param output_module_name: Module name in python :type output_module_name: str - :param cpp_modules: List of cpp modules to bind, defaults to []: bind all. - :type cpp_modules: list, optional - :param allow_inclusions_from_other_modules: Allow inclusions from other modules, which are not specified in cpp_modules, defaults to True - :type allow_inclusions_from_other_modules: bool, optional + :param cpp_targets: List of C++ targets to bind, defaults to []: bind all. + :type cpp_targets: list, optional + :param allow_inclusions_from_other_targets: Allow inclusions from other targets, which are not specified in cpp_targets, defaults to True + :type allow_inclusions_from_other_targets: bool, optional """ def __init__( @@ -27,16 +27,16 @@ def __init__( build_dir, output_dir, output_module_name, - cpp_modules=[], - allow_inclusions_from_other_modules=True, + cpp_targets=[], + allow_inclusions_from_other_targets=True, ): - all_cpp_modules = CMakeFileAPI(build_dir).get_library_targets() - if not cpp_modules: - cpp_modules = all_cpp_modules # bind all cpp modules + all_cpp_targets = CMakeFileAPI(build_dir).get_library_targets() + if not cpp_targets: + cpp_targets = all_cpp_targets # bind all C++ targets all_inclusion_sources = [] - for module in all_cpp_modules: # for all cpp modules, populate the variable - sources = CMakeFileAPI(build_dir).get_sources(module) # module's sources + for target in all_cpp_targets: # for all C++ targets, populate the variable + sources = CMakeFileAPI(build_dir).get_sources(target) # target's sources cpp_sources = list( filter(lambda source: source.endswith(".cpp"), sources) ) # sources ending with .cpp @@ -45,21 +45,21 @@ def __init__( ) # other sources like .h and .hpp files self.binding_db = {} # binding database - for module in cpp_modules: - sources = CMakeFileAPI(build_dir).get_sources(module) # module's sources + for target in cpp_targets: + sources = CMakeFileAPI(build_dir).get_sources(target) # target's sources cpp_sources = list( filter(lambda source: source.endswith(".cpp"), sources) ) # sources ending with .cpp inclusion_sources = ( all_inclusion_sources - if allow_inclusions_from_other_modules + if allow_inclusions_from_other_targets else list(set(sources) - set(cpp_sources)) ) # other sources like .h and .hpp files - self.binding_db[module] = { - "source_dir": source_dir, # source dir containing cpp modules + self.binding_db[target] = { + "source_dir": source_dir, # source dir containing C++ targets "output_dir": output_dir, # output dir - "inclusion_sources": inclusion_sources, # inclusions for the module + "inclusion_sources": inclusion_sources, # inclusions for the target "files": [ # list of files' information { "source": cpp_source, @@ -73,14 +73,14 @@ def __init__( def _parse(self): """For all input files, get the parsed tree and update the db.""" - for module in self.binding_db.values(): - source_dir = module.get("source_dir") + for target in self.binding_db.values(): + source_dir = target.get("source_dir") inclusion_sources = [ str(Path(source_dir, inclusion_source)) - for inclusion_source in module.get("inclusion_sources") + for inclusion_source in target.get("inclusion_sources") ] # string full paths of inclusion sources - for file in module.get("files"): + for file in target.get("files"): parsed_tree = Parse( Path(source_dir, file.get("source")), inclusion_sources, @@ -96,7 +96,7 @@ def _parse(self): # # - To save the JSONs: # json_output_path = Path( - # module.get("output_dir"), + # target.get("output_dir"), # "parsed", # file.get("source").replace(".cpp", ".json"), # ) From 7112bbc51e0326eb668a9a61afa8a286fd48fad3 Mon Sep 17 00:00:00 2001 From: Divyanshu Madan Date: Fri, 20 Aug 2021 22:56:47 +0530 Subject: [PATCH 8/9] fixup! [bind.py] bind.py --- clang_bind/bind.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/clang_bind/bind.py b/clang_bind/bind.py index 248dcb9..c1300bb 100644 --- a/clang_bind/bind.py +++ b/clang_bind/bind.py @@ -95,13 +95,15 @@ def _parse(self): # parsed_tree.show() # # - To save the JSONs: + # import json # json_output_path = Path( # target.get("output_dir"), # "parsed", # file.get("source").replace(".cpp", ".json"), # ) # json_output_path.parent.mkdir(parents=True, exist_ok=True) - # parsed_tree.save2file(json_output_path) + # with open(json_output_path, 'w') as f: + # json.dump(parsed_tree.to_dict(), f, indent=4) def bind(self): """Function to bind the input files.""" From 7bdcbcdf3d7bbc9ad706a5184d634f4524d29428 Mon Sep 17 00:00:00 2001 From: divmad Date: Tue, 31 Aug 2021 01:20:46 +0530 Subject: [PATCH 9/9] [init_bindings.sh] add file --- init_bindings.sh | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100755 init_bindings.sh diff --git a/init_bindings.sh b/init_bindings.sh new file mode 100755 index 0000000..8651fbb --- /dev/null +++ b/init_bindings.sh @@ -0,0 +1,23 @@ +if [ -z "$1" ]; then # if `project_dir` not specified, exit + echo "Sample usage: ./init_bindings.sh project_dir build_dir" + echo "No project directory specified, exiting.." + exit +else + PROJECT_DIR=$1 +fi + +if [ -z "$2" ]; then # if `build_dir` not specified, create in `project_dir` + BUILD_DIR=$PROJECT_DIR/build + echo "Sample usage: ./init_bindings.sh project_dir build_dir" + echo "No build directory supplied, creating in project directory.." + mkdir -p $BUILD_DIR + echo "Created directory: $BUILD_DIR" +else + BUILD_DIR=$2 +fi + +cd $BUILD_DIR +mkdir -p .cmake/api/v1/query && touch .cmake/api/v1/query/codemodel-v2 # create the API directory and query file +cmake $PROJECT_DIR # CMake should have automatically created the directory ``.cmake/api/v1/reply` containing the replies to our query. + +export PROJECT_BUILD_DIR=$BUILD_DIR