# -*- coding: utf-8 -*- # # Copyright 2016 Google LLC. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Utility wrappers around apitools generator.""" from __future__ import absolute_import from __future__ import annotations from __future__ import division from __future__ import unicode_literals import collections import dataclasses import logging import os from apitools.gen import gen_client from googlecloudsdk.api_lib.regen import api_def from googlecloudsdk.api_lib.regen import resource_generator from googlecloudsdk.core.util import files from mako import runtime from mako import template import six _INIT_FILE_CONTENT = """\ # -*- coding: utf-8 -*- # # Copyright 2016 Google LLC. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ class NoDefaultApiError(Exception): """Multiple apis versions are specified but no default is set.""" class WrongDiscoveryDocError(Exception): """Unexpected discovery doc.""" def GenerateApitoolsApi( discovery_doc, output_dir, root_package, api_name, api_version, api_config): """Invokes apitools generator for given api.""" args = [gen_client.__file__] unelidable_request_methods = api_config.get('unelidable_request_methods') if unelidable_request_methods: args.append('--unelidable_request_methods={0}'.format( ','.join(api_config['unelidable_request_methods']))) args.extend([ '--init-file=empty', '--nogenerate_cli', '--infile={0}'.format(discovery_doc), '--outdir={0}'.format(os.path.join(output_dir, api_name, api_version)), '--overwrite', '--apitools_version=CloudSDK', '--user_agent=google-cloud-sdk', '--version-identifier={0}'.format(api_version), '--root_package', '{0}.{1}.{2}'.format(root_package, api_name, api_version), 'client', ]) logging.debug('Apitools gen %s', args) gen_client.main(args) package_dir, dir_name = os.path.split(output_dir) for subdir in [dir_name, api_name, api_version]: package_dir = os.path.join(package_dir, subdir) init_file = os.path.join(package_dir, '__init__.py') if not os.path.isfile(init_file): logging.warning('%s does not have __init__.py file, generating ...', package_dir) files.WriteFileContents(init_file, _INIT_FILE_CONTENT) def _CamelCase(snake_case): return ''.join(x.capitalize() for x in snake_case.split('_')) def _MakeApitoolsClientDef(root_package, api_name, api_version): """Makes an ApitoolsClientDef.""" class_path = '.'.join([root_package, api_name, api_version]) # TODO(b/142448542) Roll back the hack if api_name == 'admin' and api_version == 'v1': client_classpath = 'admin_v1_client.AdminDirectoryV1' else: client_classpath = '.'.join([ '_'.join([api_name, api_version, 'client']), _CamelCase(api_name) + _CamelCase(api_version)]) messages_modulepath = '_'.join([api_name, api_version, 'messages']) base_url = '' client_full_classpath = class_path + '.' + client_classpath try: client_classpath_def = _GetClientClassFromDef(client_full_classpath) base_url = client_classpath_def.BASE_URL except Exception: # pylint: disable=broad-except # unreleased api or test not in "googlecloudsdk.generated_clients.apis" pass apitools_def = api_def.ApitoolsClientDef( class_path=class_path, client_classpath=client_classpath, messages_modulepath=messages_modulepath, base_url=base_url) return apitools_def def _GetClientClassFromDef(client_full_classpath): """Returns the client class for the API definition specified in the args.""" module_path, client_class_name = client_full_classpath.rsplit('.', 1) module_obj = __import__(module_path, fromlist=[client_class_name]) return getattr(module_obj, client_class_name) def _MakeGapicClientDef(root_package, api_name, api_version): """Makes a GapicClientDef.""" gapic_root_package = '.'.join(root_package.split('.')[:-1]) class_path = '.'.join( [gapic_root_package, 'gapic_wrappers', api_name, api_version]) return api_def.GapicClientDef( class_path=class_path) def _MakeApiMap(apis_config_map): """Combines package_map and api_config maps into ApiDef map. Args: apis_config_map: {api_name->api_version->ApiConfig}, description of each api. Returns: {api_name->api_version->ApiDef()}. Raises: NoDefaultApiError: if for some api with multiple versions default was not specified. """ # Validate that each API has exactly one default version configured. default_versions_map = {} for api_name, api_version_config in apis_config_map.items(): for api_version, api_config in api_version_config.items(): if api_config.default or len(api_version_config) == 1: if api_name in default_versions_map: raise NoDefaultApiError( 'Multiple default client versions found for [{}]!' .format(api_name)) default_versions_map[api_name] = api_version apis_without_default = ( set(apis_config_map.keys()).difference(default_versions_map.keys())) if apis_without_default: raise NoDefaultApiError('No default client versions found for [{0}]!' .format(', '.join(sorted(apis_without_default)))) apis_map = collections.defaultdict(dict) for api_name, api_version_config in apis_config_map.items(): for api_version, api_config in api_version_config.items(): if doc_data := api_config.discovery_doc: apitools_client = _MakeApitoolsClientDef( api_config.root_package, api_name, api_version ) discovery_doc = resource_generator.DiscoveryDoc.FromJson(doc_data) regional_endpoints = discovery_doc.endpoints else: apitools_client = None regional_endpoints = None if api_config.gcloud_gapic_library: gapic_client = _MakeGapicClientDef( api_config.root_package, api_name, api_version) else: gapic_client = None default = (api_version == default_versions_map[api_name]) enable_mtls = api_config.enable_mtls mtls_endpoint_override = api_config.mtls_endpoint_override or '' apis_map[api_name][api_version] = api_def.APIDef( apitools_client, gapic_client, default, enable_mtls, mtls_endpoint_override, regional_endpoints) return apis_map @dataclasses.dataclass(frozen=True) class ApiConfig: """Configuration for an API. root_package: dot notation of where client is generated discovery_doc: full path to where discovery doc is located gcloud_gapic_library: build target of gcloud gapic library enable_mtls: whether to enable mtls mtls_endpoint_override: mtls endpoint override default: whether this is the default version of the API """ root_package: str discovery_doc: str | None gcloud_gapic_library: str | None enable_mtls: bool mtls_endpoint_override: str | None default: bool @classmethod def FromData(cls, data, root_dir, discovery_doc_dir): """Creates an ApiConfig from regen config data. Args: data: yaml data from regen config root_dir: where the api clients are generated discovery_doc_dir: where the discovery docs are generated """ if ((doc_name := data.get('discovery_doc')) and discovery_doc_dir is not None): discovery_doc = os.path.join(discovery_doc_dir, doc_name) else: discovery_doc = None return cls( root_package=root_dir.replace('/', '.'), discovery_doc=discovery_doc, gcloud_gapic_library=data.get('gcloud_gapic_library'), enable_mtls=data.get('enable_mtls', True), mtls_endpoint_override=data.get('mtls_endpoint_override'), default=data.get('default', False), ) def GenerateApiMap(output_file, apis_config_map): """Create an apis_map.py file for the given packages and api_config. Args: output_file: Path of the output apis map file. apis_config_map: {api_name->api_version->ApiConfig}, regeneration config for all apis. """ api_def_filename, _ = os.path.splitext(api_def.__file__) api_def_source = files.ReadFileContents(api_def_filename + '.py') tpl = template.Template( filename=os.path.join(os.path.dirname(__file__), 'template.tpl') ) logging.debug('Generating api map at %s', output_file) api_map = _MakeApiMap(apis_config_map) logging.debug('Creating following api map %s', api_map) with files.FileWriter(output_file) as apis_map_file: ctx = runtime.Context( apis_map_file, api_def_source=api_def_source, apis_map=api_map ) tpl.render_context(ctx) def GenerateApitoolsResourceModule( discovery_doc, output_dir, api_name, api_version, custom_resources, ): """Create resource.py file for given api and its discovery doc. Args: discovery_doc: str, Path to the discovery doc. output_dir: str, Path to the base output directory (module will be generated underneath here in api_name/api_version subdir). api_name: str, name of the api. api_version: str, the version for the api. custom_resources: dict, dictionary of custom resource collections. Raises: WrongDiscoveryDocError: if discovery doc api name/version does not match. """ discovery_doc = resource_generator.DiscoveryDoc.FromJson(discovery_doc) if discovery_doc.api_version != api_version: logging.warning( 'Discovery api version %s does not match %s, ' 'this client will be accessible via new alias.', discovery_doc.api_version, api_version) if discovery_doc.api_name != api_name: raise WrongDiscoveryDocError('api name {0}, expected {1}'.format( discovery_doc.api_name, api_name)) resource_collections = discovery_doc.GetResourceCollections( custom_resources, api_version) if custom_resources: # Check if this is redefining one of the existing collections. matched_resources = set([]) for collection in resource_collections: if collection.name in custom_resources: apitools_compatible = custom_resources[collection.name].get( 'apitools_compatible', True ) if not apitools_compatible: continue matched_resources.add(collection.name) custom_path = custom_resources[collection.name]['path'] if isinstance(custom_path, dict): collection.flat_paths.update(custom_path) elif isinstance(custom_path, six.string_types): collection.flat_paths[ resource_generator.DEFAULT_PATH_NAME] = custom_path # Remaining must be new custom resources. for collection_name in set(custom_resources.keys()) - matched_resources: collection_def = custom_resources[collection_name] collection_path = collection_def['path'] apitools_compatible = collection_def.get( 'apitools_compatible', True ) if not apitools_compatible: continue enable_uri_parsing = collection_def.get('enable_uri_parsing', True) collection_info = discovery_doc.MakeResourceCollection( collection_name, collection_path, enable_uri_parsing, api_version) resource_collections.append(collection_info) api_dir = os.path.join(output_dir, api_name, api_version) if not os.path.exists(api_dir): os.makedirs(api_dir) resource_file_name = os.path.join(api_dir, 'resources.py') if resource_collections: logging.debug('Generating resource module at %s', resource_file_name) tpl = template.Template(filename=os.path.join(os.path.dirname(__file__), 'resources.tpl')) with files.FileWriter(resource_file_name) as output_file: ctx = runtime.Context(output_file, collections=sorted(resource_collections), base_url=resource_collections[0].base_url, docs_url=discovery_doc.docs_url) tpl.render_context(ctx) elif os.path.isfile(resource_file_name): logging.debug('Removing existing resource module at %s', resource_file_name) os.remove(resource_file_name)