diff --git a/README.md b/README.md index 0480960829c..6f1714fa310 100644 --- a/README.md +++ b/README.md @@ -83,5 +83,15 @@ output-artifact: python-files pipeline: python/multiapiscript: scope: multiapiscript + output-artifact: python-files + + python/multiapiscript/emitter: + input: multiapiscript + scope: scope-multiapiscript/emitter + +scope-multiapiscript/emitter: + input-artifact: python-files + output-uri-expr: $key +output-artifact: python-files ``` diff --git a/autorest/multiapi/__init__.py b/autorest/multiapi/__init__.py index 420303368c1..a788f9fd1fd 100644 --- a/autorest/multiapi/__init__.py +++ b/autorest/multiapi/__init__.py @@ -7,12 +7,12 @@ import logging import json import shutil -import os from collections import defaultdict from pathlib import Path -from typing import Dict, List, Tuple, Union, Optional, cast +from typing import Dict, List, Tuple, Optional, cast from .multiapi_serializer import MultiAPISerializer +from ..jsonrpc import AutorestAPI from .. import Plugin @@ -22,33 +22,17 @@ class MultiApiScriptPlugin(Plugin): def process(self) -> bool: input_package_name: str = self._autorestapi.get_value("package-name") - python_sdks_folder: str = self._autorestapi.get_value("python-sdks-folder") + output_folder: str = self._autorestapi.get_value("output-folder") default_api: str = self._autorestapi.get_value("default-api") generator = MultiAPI( input_package_name, - python_sdks_folder, + output_folder, + self._autorestapi, default_api ) return generator.process() -def _patch_import(file_path: Union[str, Path]) -> None: - """If multi-client package, we need to patch import to be - from ..version - and not - from .version - That should probably means those files should become a template, but since right now - it's literally one dot, let's do it the raw way. - """ - # That's a dirty hack, maybe it's worth making configuration a template? - with open(file_path, "rb") as read_fd: - conf_bytes = read_fd.read() - conf_bytes = conf_bytes.replace( - b" .version", b" ..version" - ) # Just a dot right? Worth its own template for that? :) - with open(file_path, "wb") as write_fd: - write_fd.write(conf_bytes) - def _parse_input(input_parameter: str): """From a syntax like package_name#submodule, build a package name and complete module name. @@ -58,16 +42,7 @@ def _parse_input(input_parameter: str): module_name = package_name.replace("-", ".") if len(split_package_name) >= 2: module_name = ".".join([module_name, split_package_name[1]]) - return package_name, module_name - -def _get_paths_to_versions(path_to_package: Path) -> List[Path]: - - paths_to_versions = [] - for child in [x for x in path_to_package.iterdir() if x.is_dir()]: - child_dir = (path_to_package / child).resolve() - if Path(child_dir / '_metadata.json') in child_dir.iterdir(): - paths_to_versions.append(child_dir) - return paths_to_versions + return module_name def _get_floating_latest(api_versions_list: List[str], preview_mode: bool): """Get the floating latest, from a random list of API versions. @@ -89,69 +64,6 @@ def _get_floating_latest(api_versions_list: List[str], preview_mode: bool): # If not preview mode, and there is preview, take the latest known stable return sorted(trimmed_preview)[-1] -def _build_operation_meta(paths_to_versions: List[Path]): - """Introspect the client: - - version_dict => { - 'application_gateways': [ - ('v2018_05_01', 'ApplicationGatewaysOperations') - ] - } - mod_to_api_version => {'v2018_05_01': '2018-05-01'} - """ - mod_to_api_version: Dict[str, str] = defaultdict(str) - versioned_operations_dict: Dict[str, List[Tuple[str, str]]] = defaultdict(list) - for version_path in paths_to_versions: - with open(version_path / "_metadata.json") as f: - metadata_json = json.load(f) - operation_groups = metadata_json['operation_groups'] - version = metadata_json['chosen_version'] - total_api_version_list = metadata_json['total_api_version_list'] - if not version: - if total_api_version_list: - sys.exit( - f"Unable to match {total_api_version_list} to label {version_path.stem}" - ) - else: - sys.exit( - f"Unable to extract api version of {version_path.stem}" - ) - mod_to_api_version[version_path.name] = version - for operation_group, operation_group_class_name in operation_groups.items(): - versioned_operations_dict[operation_group].append((version_path.name, operation_group_class_name)) - return versioned_operations_dict, mod_to_api_version - -def _build_operation_mixin_meta(paths_to_versions: List[Path]) -> Dict[str, Dict[str, List[str]]]: - """Introspect the client: - - version_dict => { - 'check_dns_name_availability': { - 'doc': 'docstring', - 'signature': '(self, p1, p2, **operation_config), - 'call': 'p1, p2', - 'available_apis': [ - 'v2018_05_01' - ] - } - } - """ - mixin_operations: Dict[str, Dict[str, List[str]]] = {} - for version_path in paths_to_versions: - with open(version_path / "_metadata.json") as f: - metadata_json = json.load(f) - if not metadata_json.get('operation_mixins'): - continue - for func_name, func in metadata_json['operation_mixins'].items(): - if func_name.startswith("_"): - continue - mixin_operations.setdefault(func_name, {}).setdefault( - "available_apis", [] - ).append(version_path.name) - mixin_operations[func_name]['doc'] = func['doc'] - mixin_operations[func_name]['signature'] = func['signature'] - mixin_operations[func_name]['call'] = func['call'] - return mixin_operations - def _build_last_rt_list( versioned_operations_dict: Dict[str, Tuple[str, str]], mixin_operations: Dict[str, Dict[str, List[str]]], @@ -218,34 +130,87 @@ def there_is_a_rt_that_contains_api_version(rt_dict, api_version): return last_rt_list class MultiAPI: - def __init__(self, input_package_name: str, python_sdks_folder: str, default_api: Optional[str] = None): + def __init__( + self, + input_package_name: str, + output_folder: str, + autorestapi: AutorestAPI, + default_api: Optional[str] = None + ): self.input_package_name = input_package_name - self.python_sdks_folder = Path(python_sdks_folder).resolve() + self.output_folder = Path(output_folder).resolve() + self._autorestapi = autorestapi self.default_api = default_api - def _resolve_package_directory(self, package_name: str) -> str: - """Returns the appropriate relative diff between the python sdks root and the actual package_directory + def _get_paths_to_versions(self) -> List[Path]: + + paths_to_versions = [] + for child in [x for x in self.output_folder.iterdir() if x.is_dir()]: + child_dir = (self.output_folder / child).resolve() + if Path(child_dir / '_metadata.json') in child_dir.iterdir(): + paths_to_versions.append(Path(child.stem)) + return paths_to_versions + + def _build_operation_mixin_meta(self, paths_to_versions: List[Path]) -> Dict[str, Dict[str, List[str]]]: + """Introspect the client: + + version_dict => { + 'check_dns_name_availability': { + 'doc': 'docstring', + 'signature': '(self, p1, p2, **operation_config), + 'call': 'p1, p2', + 'available_apis': [ + 'v2018_05_01' + ] + } + } """ - packages = [ - p.parent - for p in ( - list(self.python_sdks_folder.glob(f"*/{package_name}/setup.py")) + - list(self.python_sdks_folder.glob(f"sdk/*/{package_name}/setup.py")) - ) - ] - - if len(packages) > 1: - print( - "There should only be a single package matched in either repository structure." + - f" The following were found: {packages}" - ) - sys.exit(1) - if not packages: - print( - f"Was unable to find {self.input_package_name} anything in {self.python_sdks_folder}" - ) - sys.exit(1) - return str(packages[0].relative_to(self.python_sdks_folder)) + mixin_operations: Dict[str, Dict[str, List[str]]] = {} + for version_path in paths_to_versions: + metadata_json = json.loads(self._autorestapi.read_file(version_path / "_metadata.json")) + if not metadata_json.get('operation_mixins'): + continue + for func_name, func in metadata_json['operation_mixins'].items(): + if func_name.startswith("_"): + continue + mixin_operations.setdefault(func_name, {}).setdefault( + "available_apis", [] + ).append(version_path.name) + mixin_operations[func_name]['doc'] = func['doc'] + mixin_operations[func_name]['signature'] = func['signature'] + mixin_operations[func_name]['call'] = func['call'] + return mixin_operations + + def _build_operation_meta(self, paths_to_versions: List[Path]): + """Introspect the client: + + version_dict => { + 'application_gateways': [ + ('v2018_05_01', 'ApplicationGatewaysOperations') + ] + } + mod_to_api_version => {'v2018_05_01': '2018-05-01'} + """ + mod_to_api_version: Dict[str, str] = defaultdict(str) + versioned_operations_dict: Dict[str, List[Tuple[str, str]]] = defaultdict(list) + for version_path in paths_to_versions: + metadata_json = json.loads(self._autorestapi.read_file(version_path / "_metadata.json")) + operation_groups = metadata_json['operation_groups'] + version = metadata_json['chosen_version'] + total_api_version_list = metadata_json['total_api_version_list'] + if not version: + if total_api_version_list: + sys.exit( + f"Unable to match {total_api_version_list} to label {version_path.stem}" + ) + else: + sys.exit( + f"Unable to extract api version of {version_path.stem}" + ) + mod_to_api_version[version_path.name] = version + for operation_group, operation_group_class_name in operation_groups.items(): + versioned_operations_dict[operation_group].append((version_path.name, operation_group_class_name)) + return versioned_operations_dict, mod_to_api_version def process(self) -> bool: _LOGGER.info("Generating multiapi client") @@ -253,16 +218,9 @@ def process(self) -> bool: # If not, if it exists a stable API version for a global or RT, will always be used preview_mode = cast(bool, self.default_api and "preview" in self.default_api) - # The only known multi-client package right now is azure-mgmt-resource - is_multi_client_package = "#" in self.input_package_name - - package_name, module_name = _parse_input(self.input_package_name) - path_to_package = ( - self.python_sdks_folder / self._resolve_package_directory(package_name) / - Path(module_name.replace(".", os.sep)) - ).resolve() - paths_to_versions = _get_paths_to_versions(path_to_package) - versioned_operations_dict, mod_to_api_version = _build_operation_meta( + module_name = _parse_input(self.input_package_name) + paths_to_versions = self._get_paths_to_versions() + versioned_operations_dict, mod_to_api_version = self._build_operation_meta( paths_to_versions ) @@ -289,25 +247,21 @@ def process(self) -> bool: _LOGGER.info("Default API version will be: %s", last_api_version) # In case we are transitioning from a single api generation, clean old folders - shutil.rmtree(str(path_to_package / "operations"), ignore_errors=True) - shutil.rmtree(str(path_to_package / "models"), ignore_errors=True) + shutil.rmtree(str(self.output_folder / "operations"), ignore_errors=True) + shutil.rmtree(str(self.output_folder / "models"), ignore_errors=True) shutil.copy( - str(path_to_package / last_api_version / "_configuration.py"), - str(path_to_package / "_configuration.py"), + str(self.output_folder / last_api_version / "_configuration.py"), + str(self.output_folder / "_configuration.py"), ) shutil.copy( - str(path_to_package / last_api_version / "__init__.py"), - str(path_to_package / "__init__.py"), + str(self.output_folder / last_api_version / "__init__.py"), + str(self.output_folder / "__init__.py"), ) - if is_multi_client_package: - _LOGGER.warning("Patching multi-api client basic files") - _patch_import(path_to_package / "_configuration.py") - _patch_import(path_to_package / "__init__.py") # Detect if this client is using an operation mixin (Network) # Operation mixins are available since Autorest.Python 4.x - mixin_operations = _build_operation_mixin_meta(paths_to_versions) + mixin_operations = self._build_operation_mixin_meta(paths_to_versions) # get client name from default api version path_to_default_version = Path() @@ -315,8 +269,7 @@ def process(self) -> bool: if last_api_version.replace("-", "_") == path_to_version.stem: path_to_default_version = path_to_version break - with open(path_to_default_version / "_metadata.json") as f: - metadata_json = json.load(f) + metadata_json = json.loads(self._autorestapi.read_file(path_to_default_version / "_metadata.json")) # versioned_operations_dict => { # 'application_gateways': [ @@ -357,13 +310,22 @@ def process(self) -> bool: ), "config": metadata_json["config"] } - multiapi_serializer = MultiAPISerializer( - conf=conf, path_to_package=path_to_package, service_client_name=metadata_json["client"]["filename"] + multiapi_serializer = MultiAPISerializer(conf=conf) + + self._autorestapi.write_file( + Path(metadata_json["client"]["filename"]), + multiapi_serializer.serialize_multiapi_client() + ) + + self._autorestapi.write_file( + Path("_configuration.py"), + multiapi_serializer.serialize_multiapi_config() ) - multiapi_serializer.serialize_multiapi_client() - multiapi_serializer.serialize_multiapi_config() if mixin_operations: - multiapi_serializer.serialize_multiapi_operation_mixins() + self._autorestapi.write_file( + Path("_operations_mixin.py"), + multiapi_serializer.serialize_multiapi_operation_mixins() + ) _LOGGER.info("Done!") return True diff --git a/autorest/multiapi/__main__.py b/autorest/multiapi/__main__.py deleted file mode 100644 index 624c55ddb39..00000000000 --- a/autorest/multiapi/__main__.py +++ /dev/null @@ -1,47 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -import argparse -import logging -import sys -from . import MultiAPI - - -parser = argparse.ArgumentParser( - description="Multi-API client generation for Azure SDK for Python" -) -parser.add_argument( - "--debug", dest="debug", action="store_true", help="Verbosity in DEBUG mode" -) -parser.add_argument( - "--default-api-version", - dest="default_api", - default=None, - help="Force default API version, do not detect it. [default: %(default)s]", -) -parser.add_argument("package_name", help="The package name.") -parser.add_argument( - "--python-sdks-folder", - dest="python_sdks_folder", - help="The root of your python sdk repo." -) - -args = parser.parse_args() - -main_logger = logging.getLogger() -logging.basicConfig() -main_logger.setLevel(logging.DEBUG if args.debug else logging.INFO) - -input_package_name: str = args.package_name -python_sdks_folder: str = args.python_sdks_folder -default_api: str = args.default_api - -generator = MultiAPI( - input_package_name, - python_sdks_folder, - default_api -) -if not generator.process(): - sys.exit(1) diff --git a/autorest/multiapi/multiapi_serializer.py b/autorest/multiapi/multiapi_serializer.py index dba2b1c7e30..99463883423 100644 --- a/autorest/multiapi/multiapi_serializer.py +++ b/autorest/multiapi/multiapi_serializer.py @@ -8,10 +8,8 @@ class MultiAPISerializer: - def __init__(self, conf, path_to_package, service_client_name): + def __init__(self, conf): self.conf = conf - self.path_to_package = path_to_package - self.service_client_name = service_client_name self.env = Environment( loader=PackageLoader("autorest.multiapi", "templates"), keep_trailing_newline=True, @@ -21,23 +19,14 @@ def __init__(self, conf, path_to_package, service_client_name): lstrip_blocks=True, ) - def serialize_multiapi_client(self): + def serialize_multiapi_client(self) -> str: template = self.env.get_template("multiapi_service_client.py.jinja2") - result = template.render(**self.conf) + return template.render(**self.conf) - with (self.path_to_package / self.service_client_name).open("w") as fd: - fd.write(result) - - def serialize_multiapi_config(self): + def serialize_multiapi_config(self) -> str: template = self.env.get_template("multiapi_config.py.jinja2") - result = template.render(client_name=self.conf["client_name"], **self.conf["config"]) - - with (self.path_to_package / "_configuration.py").open("w") as fd: - fd.write(result) + return template.render(client_name=self.conf["client_name"], **self.conf["config"]) - def serialize_multiapi_operation_mixins(self): + def serialize_multiapi_operation_mixins(self) -> str: template = self.env.get_template("multiapi_operations_mixin.py.jinja2") - result = template.render(**self.conf) - - with (self.path_to_package / "_operations_mixin.py").open("w") as fd: - fd.write(result) + return template.render(**self.conf) diff --git a/autorest/multiapi/templates/multiapi_config.py.jinja2 b/autorest/multiapi/templates/multiapi_config.py.jinja2 index 6fcae6b2bc3..ef09851eb6d 100644 --- a/autorest/multiapi/templates/multiapi_config.py.jinja2 +++ b/autorest/multiapi/templates/multiapi_config.py.jinja2 @@ -47,9 +47,11 @@ class {{ client_name }}Configuration(Configuration): {{ method_signature()|indent }} # type: (...) -> None -{% for gp in global_parameters["method"] | selectattr('required') %} - if {{ gp }} is None: - raise ValueError("Parameter '{{ gp }}' must not be None.") +{% for serialized_name, gp_dict in global_parameters["method"].items() %} + {% if gp_dict["required"] %} + if {{ serialized_name }} is None: + raise ValueError("Parameter '{{ serialized_name }}' must not be None.") + {% endif %} {% endfor %} super({{ client_name }}Configuration, self).__init__(**kwargs)