# -*- coding: utf-8 -*- # # Copyright 2014 Google LLC. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Module to parse .yaml files for an appengine app.""" from __future__ import absolute_import from __future__ import division from __future__ import unicode_literals import os from googlecloudsdk.api_lib.app import env from googlecloudsdk.appengine.api import appinfo from googlecloudsdk.appengine.api import appinfo_errors from googlecloudsdk.appengine.api import appinfo_includes from googlecloudsdk.appengine.api import croninfo from googlecloudsdk.appengine.api import dispatchinfo from googlecloudsdk.appengine.api import queueinfo from googlecloudsdk.appengine.api import validation from googlecloudsdk.appengine.api import yaml_errors from googlecloudsdk.appengine.datastore import datastore_index from googlecloudsdk.core import exceptions from googlecloudsdk.core import log from googlecloudsdk.core.util import files HINT_PROJECT = ('This field is not used by gcloud and must be removed. ' 'Project name should instead be specified either by ' '`gcloud config set project MY_PROJECT` or by setting the ' '`--project` flag on individual command executions.') HINT_VERSION = ('This field is not used by gcloud and must be removed. ' 'Versions are generated automatically by default but can also ' 'be manually specified by setting the `--version` flag on ' 'individual command executions.') HINT_THREADSAFE = ('This field is not supported with runtime [{}] and can ' 'safely be removed.') HINT_READABLE = ('This field is not configurable with runtime [{}] since ' 'static files are always readable by the application. It ' 'can safely be removed.') MANAGED_VMS_DEPRECATION_WARNING = """\ Deployments using `vm: true` have been deprecated. Please update your \ app.yaml to use `env: flex`. To learn more, please visit \ https://cloud.google.com/appengine/docs/flexible/migration. """ UPGRADE_FLEX_PYTHON_URL = ( 'https://cloud.google.com/appengine/docs/flexible/python/migrating') APP_ENGINE_APIS_DEPRECATION_WARNING = ( 'Support for the compat runtimes and their base images ' '(enable_app_engine_apis: true) has been deprecated. Please migrate to a ' 'new base image, or use a Google managed runtime. ' 'To learn more, visit {}.').format(UPGRADE_FLEX_PYTHON_URL) PYTHON_SSL_WARNING = ( 'You are using an outdated version [2.7] of the Python SSL library. ' 'Please update your app.yaml file to specify SSL library [latest] to ' 'avoid security risks. On April 2, 2018, version 2.7 will be ' 'decommissioned and your app will be blocked from deploying until you ' 'you specify SSL library [latest] or [2.7.11].' 'For more information, visit {}.' ).format('https://cloud.google.com/appengine/docs/deprecations/python-ssl-27') FLEX_PY34_WARNING = ( 'You are using a deprecated version [3.4] of Python on the App ' 'Engine Flexible environment. Please update your app.yaml file to specify ' '[python_version: latest]. Python 3.4 will be decommissioned on March 29, ' '2019. After this date, new deployments will fail. For more information ' 'about this deprecation, visit {}.' ).format('https://cloud.google.com/appengine/docs/deprecations/python34') DEFAULT_MAX_INSTANCES_FORWARD_CHANGE_ZERO_WARNING = ( 'You might have set automatic_scaling.max_instances to 0. Starting from' ' March, 2025, App Engine sets the automatic scaling maximum' ' instances default for standard environment deployments to 20. This change' " doesn't impact existing apps. To disable the maximum instances default" ' configuration setting, specify the maximum permitted value 2147483647.' ' For more information, see {}. \n' ).format( 'https://cloud.google.com/appengine/docs/standard/reference/app-yaml.md#scaling_elements' ) DEFAULT_MAX_INSTANCES_FORWARD_CHANGE_WARNING = ( 'You might be using automatic scaling for a standard environment' ' deployment, without providing a value for' ' automatic_scaling.max_instances. Starting from March, 2025, App' ' Engine sets the automatic scaling maximum instances default for standard' " environment deployments to 20. This change doesn't impact existing apps." ' To override the default, specify the new max_instances value in your' ' app.yaml file, and deploy a new version or redeploy over an existing' ' version. For details on max_instances, see {}. \n' ).format( 'https://cloud.google.com/appengine/docs/standard/reference/app-yaml.md#scaling_elements' ) # This is the equivalent of the following in app.yaml: # skip_files: # - ^(.*/)?#.*#$ # - ^(.*/)?.*~$ # - ^(.*/)?.*\.py[co]$ # - ^(.*/)?.*/RCS/.*$ # - ^(.*/)?.git(ignore|/.*)$ # - ^(.*/)?node_modules/.* DEFAULT_SKIP_FILES_FLEX = (r'^(.*/)?#.*#$|' r'^(.*/)?.*~$|' r'^(.*/)?.*\.py[co]$|' r'^(.*/)?.*/RCS/.*$|' r'^(.*/)?.git(ignore|/.*)$|' r'^(.*/)?node_modules/.*$') class Error(exceptions.Error): """A base error for this module.""" pass class YamlParseError(Error): """An exception for when a specific yaml file is not well formed.""" def __init__(self, file_path, e): """Creates a new Error. Args: file_path: str, The full path of the file that failed to parse. e: Exception, The exception that was originally raised. """ super(YamlParseError, self).__init__( 'An error occurred while parsing file: [{file_path}]\n{err}' .format(file_path=file_path, err=e)) class YamlValidationError(Error): """An exception for when a specific yaml file has invalid info.""" pass class AppConfigError(Error): """Errors in Application Config.""" class _YamlInfo(object): """A base class for holding some basic attributes of a parsed .yaml file.""" def __init__(self, file_path, parsed): """Creates a new _YamlInfo. Args: file_path: str, The full path the file that was parsed. parsed: The parsed yaml data as one of the *_info objects. """ self.file = file_path self.parsed = parsed @staticmethod def _ParseYaml(file_path, parser): """Parses the given file using the given parser. Args: file_path: str, The full path of the file to parse. parser: str, The parser to use to parse this yaml file. Returns: The result of the parse. """ with files.FileReader(file_path) as fp: return parser(fp) class ConfigYamlInfo(_YamlInfo): """A class for holding some basic attributes of a parsed config .yaml file.""" CRON = 'cron' DISPATCH = 'dispatch' INDEX = 'index' QUEUE = 'queue' CONFIG_YAML_PARSERS = { CRON: croninfo.LoadSingleCron, DISPATCH: dispatchinfo.LoadSingleDispatch, INDEX: datastore_index.ParseIndexDefinitions, QUEUE: queueinfo.LoadSingleQueue, } def __init__(self, file_path, config, parsed): """Creates a new ConfigYamlInfo. Args: file_path: str, The full path the file that was parsed. config: str, The name of the config that was parsed (i.e. 'cron') parsed: The parsed yaml data as one of the *_info objects. """ super(ConfigYamlInfo, self).__init__(file_path, parsed) self.config = config @property def name(self): """Name of the config file without extension, e.g. `cron`.""" (base, _) = os.path.splitext(os.path.basename(self.file)) return base @staticmethod def FromFile(file_path): """Parses the given config file. Args: file_path: str, The full path to the config file. Raises: Error: If a user tries to parse a dos.yaml file. YamlParseError: If the file is not valid. Returns: A ConfigYamlInfo object for the parsed file. """ base, ext = os.path.splitext(os.path.basename(file_path)) if base == 'dos': raise Error( '`gcloud app deploy dos.yaml` is no longer supported. Please use' ' `gcloud app firewall-rules` instead.' ) parser = (ConfigYamlInfo.CONFIG_YAML_PARSERS.get(base) if os.path.isfile(file_path) and ext.lower() in ['.yaml', '.yml'] else None) if not parser: return None try: parsed = _YamlInfo._ParseYaml(file_path, parser) if not parsed: raise YamlParseError(file_path, 'The file is empty') except (yaml_errors.Error, validation.Error) as e: raise YamlParseError(file_path, e) _CheckIllegalAttribute( name='application', yaml_info=parsed, extractor_func=lambda yaml: yaml.application, file_path=file_path, msg=HINT_PROJECT) if base == 'dispatch': return DispatchConfigYamlInfo(file_path, config=base, parsed=parsed) return ConfigYamlInfo(file_path, config=base, parsed=parsed) class DispatchConfigYamlInfo(ConfigYamlInfo): """Provides methods for getting 1p-ready representation.""" def _HandlerToDict(self, handler): """Converst a dispatchinfo handler into a 1p-ready dict.""" parsed_url = dispatchinfo.ParsedURL(handler.url) dispatch_domain = parsed_url.host if not parsed_url.host_exact: dispatch_domain = '*' + dispatch_domain dispatch_path = parsed_url.path if not parsed_url.path_exact: trailing_matcher = '/*' if dispatch_path.endswith('/') else '*' dispatch_path = dispatch_path.rstrip('/') + trailing_matcher return { 'domain': dispatch_domain, 'path': dispatch_path, 'service': handler.service, } def GetRules(self): """Get dispatch rules on a format suitable for Admin API. Returns: [{'service': str, 'domain': str, 'path': str}], rules. """ return [self._HandlerToDict(h) for h in self.parsed.dispatch or []] class ServiceYamlInfo(_YamlInfo): """A class for holding some basic attributes of a parsed service yaml file.""" DEFAULT_SERVICE_NAME = 'default' def __init__(self, file_path, parsed): """Creates a new ServiceYamlInfo. Args: file_path: str, The full path the file that was parsed. parsed: appinfo.AppInfoExternal, parsed Application Configuration. """ super(ServiceYamlInfo, self).__init__(file_path, parsed) self.module = parsed.service or ServiceYamlInfo.DEFAULT_SERVICE_NAME if parsed.env in ['2', 'flex', 'flexible']: self.env = env.FLEX elif parsed.vm or parsed.runtime == 'vm': self.env = env.MANAGED_VMS else: self.env = env.STANDARD # All `env: flex` apps are hermetic. All `env: standard` apps are not # hermetic. All `vm: true` apps are hermetic IFF they don't use static # files. if self.env is env.FLEX: self.is_hermetic = True elif parsed.vm: for urlmap in parsed.handlers: if urlmap.static_dir or urlmap.static_files: self.is_hermetic = False break else: self.is_hermetic = True else: self.is_hermetic = False self._InitializeHasExplicitSkipFiles(file_path, parsed) self._UpdateSkipFiles(parsed) if (self.env is env.MANAGED_VMS) or self.is_hermetic: self.runtime = parsed.GetEffectiveRuntime() self._UpdateVMSettings() else: self.runtime = parsed.runtime # New "Ti" style runtimes self.is_ti_runtime = env.GetTiRuntimeRegistry().Get(self.runtime, self.env) @staticmethod def FromFile(file_path): """Parses the given service file. Args: file_path: str, The full path to the service file. Raises: YamlParseError: If the file is not a valid Yaml-file. YamlValidationError: If validation of parsed info fails. Returns: A ServiceYamlInfo object for the parsed file. """ try: parsed = _YamlInfo._ParseYaml(file_path, appinfo_includes.Parse) except (yaml_errors.Error, appinfo_errors.Error) as e: raise YamlParseError(file_path, e) info = ServiceYamlInfo(file_path, parsed) info.Validate() return info def Validate(self): """Displays warnings and raises exceptions for non-schema errors. Raises: YamlValidationError: If validation of parsed info fails. """ if self.parsed.runtime == 'vm': vm_runtime = self.parsed.GetEffectiveRuntime() else: vm_runtime = None if self.parsed.runtime == 'python': raise YamlValidationError( 'Service [{service}] uses unsupported Python 2.5 runtime. ' 'Please use [runtime: python27] instead.'.format( service=(self.parsed.service or ServiceYamlInfo.DEFAULT_SERVICE_NAME))) elif self.parsed.runtime == 'python-compat': raise YamlValidationError( '"python-compat" is not a supported runtime.') elif self.parsed.runtime == 'custom' and not self.parsed.env: raise YamlValidationError( 'runtime "custom" requires that env be explicitly specified.') if self.env is env.MANAGED_VMS: log.warning(MANAGED_VMS_DEPRECATION_WARNING) if (self.env is env.FLEX and self.parsed.beta_settings and self.parsed.beta_settings.get('enable_app_engine_apis')): log.warning(APP_ENGINE_APIS_DEPRECATION_WARNING) if self.env is env.FLEX and vm_runtime == 'python27': raise YamlValidationError( 'The "python27" is not a valid runtime in env: flex. ' 'Please use [python] instead.') if self.env is env.FLEX and vm_runtime == 'python-compat': log.warning('[runtime: {}] is deprecated. Please use [runtime: python] ' 'instead. See {} for more info.' .format(vm_runtime, UPGRADE_FLEX_PYTHON_URL)) # TODO: b/388712720 - Cleanup warning once backend experiments are cleaned # Raise warning about default max instances forward change for GAE Standard # when the user has selected AutomaticScaling without providing a # max_instances value or will use AutomaticScaling by default. if ( self.env is not env.FLEX and not self.parsed.basic_scaling and not self.parsed.manual_scaling and ( not self.parsed.automatic_scaling or ( self.parsed.automatic_scaling and not self.parsed.automatic_scaling.max_instances and self.parsed.automatic_scaling.max_instances != 0 ) ) ): log.warning(DEFAULT_MAX_INSTANCES_FORWARD_CHANGE_WARNING) # TODO: b/388712720 - Cleanup warning once backend experiments are cleaned # Raise warning about default max instances forward change for GAE Standard # when the user has selected AutomaticScaling and explicitly provided a # value of zero for max_instances. if ( self.env is not env.FLEX and self.parsed.automatic_scaling and self.parsed.automatic_scaling.max_instances == 0 ): log.warning(DEFAULT_MAX_INSTANCES_FORWARD_CHANGE_ZERO_WARNING) for warn_text in self.parsed.GetWarnings(): log.warning('In file [{0}]: {1}'.format(self.file, warn_text)) if (self.env is env.STANDARD and self.parsed.runtime == 'python27' and HasLib(self.parsed, 'ssl', '2.7')): log.warning(PYTHON_SSL_WARNING) if (self.env is env.FLEX and vm_runtime == 'python' and GetRuntimeConfigAttr(self.parsed, 'python_version') == '3.4'): log.warning(FLEX_PY34_WARNING) _CheckIllegalAttribute( name='application', yaml_info=self.parsed, extractor_func=lambda yaml: yaml.application, file_path=self.file, msg=HINT_PROJECT) _CheckIllegalAttribute( name='version', yaml_info=self.parsed, extractor_func=lambda yaml: yaml.version, file_path=self.file, msg=HINT_VERSION) self._ValidateTi() def _ValidateTi(self): """Validation specifically for Ti-runtimes.""" if not self.is_ti_runtime: return _CheckIllegalAttribute( name='threadsafe', yaml_info=self.parsed, extractor_func=lambda yaml: yaml.threadsafe, file_path=self.file, msg=HINT_THREADSAFE.format(self.runtime)) # pylint: disable=cell-var-from-loop for handler in self.parsed.handlers: _CheckIllegalAttribute( name='application_readable', yaml_info=handler, extractor_func=lambda yaml: handler.application_readable, file_path=self.file, msg=HINT_READABLE.format(self.runtime)) def RequiresImage(self): """Returns True if we'll need to build a docker image.""" return self.env is env.MANAGED_VMS or self.is_hermetic def _UpdateVMSettings(self): """Overwrites vm_settings for App Engine services with VMs. Also sets module_yaml_path which is needed for some runtimes. Raises: AppConfigError: if the function was called for a standard service """ if self.env not in [env.MANAGED_VMS, env.FLEX]: raise AppConfigError( 'This is not an App Engine Flexible service. Please set `env` ' 'field to `flex`.') if not self.parsed.vm_settings: self.parsed.vm_settings = appinfo.VmSettings() self.parsed.vm_settings['module_yaml_path'] = os.path.basename(self.file) def GetAppYamlBasename(self): """Returns the basename for the app.yaml file for this service.""" return os.path.basename(self.file) def HasExplicitSkipFiles(self): """Returns whether user has explicitly defined skip_files in app.yaml.""" return self._has_explicit_skip_files def _InitializeHasExplicitSkipFiles(self, file_path, parsed): """Read app.yaml to determine whether user explicitly defined skip_files.""" if getattr(parsed, 'skip_files', None) == appinfo.DEFAULT_SKIP_FILES: # Make sure that this was actually a default, not from the file. try: contents = files.ReadFileContents(file_path) except files.Error: # If the class was initiated with a non-existent file contents = '' self._has_explicit_skip_files = 'skip_files' in contents else: self._has_explicit_skip_files = True def _UpdateSkipFiles(self, parsed): """Resets skip_files field to Flex default if applicable.""" if self.RequiresImage() and not self.HasExplicitSkipFiles(): # pylint:disable=protected-access parsed.skip_files = validation._RegexStrValue( validation.Regex(DEFAULT_SKIP_FILES_FLEX), DEFAULT_SKIP_FILES_FLEX, 'skip_files') # pylint:enable=protected-access def HasLib(parsed, name, version=None): """Check if the parsed yaml has specified the given library. Args: parsed: parsed from yaml to python object name: str, Name of the library version: str, If specified, also matches against the version of the library. Returns: True if library with optionally the given version is present. """ libs = parsed.libraries or [] if version: return any(lib.name == name and lib.version == version for lib in libs) else: return any(lib.name == name for lib in libs) def GetRuntimeConfigAttr(parsed, attr): """Retrieve an attribute under runtime_config section. Args: parsed: parsed from yaml to python object attr: str, Attribute name, e.g. `runtime_version` Returns: The value of runtime_config.attr or None if the attribute isn't set. """ return (parsed.runtime_config or {}).get(attr) def _CheckIllegalAttribute(name, yaml_info, extractor_func, file_path, msg=''): """Validates that an illegal attribute is not set. Args: name: str, The name of the attribute in the yaml files. yaml_info: AppInfoExternal, The yaml to validate. extractor_func: func(AppInfoExternal)->str, A function to extract the value of the attribute from a _YamlInfo object. file_path: str, The path of file from which yaml_info was parsed. msg: str, Message to couple with the error Raises: YamlValidationError: If illegal attribute is set. """ attribute = extractor_func(yaml_info) if attribute is not None: # Disallow use of the given attribute. raise YamlValidationError( 'The [{0}] field is specified in file [{1}]. '.format(name, file_path) + msg)