# -*- coding: utf-8 -*- # # Copyright 2017 Google LLC. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Utilities for deriving services and configs from paths. Paths are typically given as positional params, like `gcloud app deploy ...`. """ from __future__ import absolute_import from __future__ import division from __future__ import unicode_literals import collections import os from googlecloudsdk.api_lib.app import env from googlecloudsdk.api_lib.app import yaml_parsing from googlecloudsdk.command_lib.app import exceptions from googlecloudsdk.core import log from googlecloudsdk.core.util import files _STANDARD_APP_YAML_URL = ( 'https://cloud.google.com/appengine/docs/standard/reference/app-yaml') _FLEXIBLE_APP_YAML_URL = ( 'https://cloud.google.com/appengine/docs/flexible/reference/app-yaml') APP_YAML_INSTRUCTIONS = ( 'using the directions at {flex} (App Engine flexible environment) or {std} ' '(App Engine standard environment) under the tab for your language.' ).format(flex=_FLEXIBLE_APP_YAML_URL, std=_STANDARD_APP_YAML_URL) FINGERPRINTING_WARNING = ( 'As an alternative, create an app.yaml file yourself ' + APP_YAML_INSTRUCTIONS) NO_YAML_ERROR = ( 'An app.yaml (or appengine-web.xml) file is required to deploy this ' 'directory as an App Engine application. Create an app.yaml file ' + APP_YAML_INSTRUCTIONS) class Service(object): """Represents data around a deployable service. Attributes: descriptor: str, File path to the original deployment descriptor, which is either a `.yaml` or an `appengine-web.xml`. source: str, Path to the original deployable artifact or directory, which is typically the original source directory, but could also be an artifact such as a fat JAR file. service_info: yaml_parsing.ServiceYamlInfo, Info parsed from the `.yaml` file. Note that service_info.file may point to a file in a staged directory. upload_dir: str, Path to the source directory. If staging is required, this points to the staged directory. service_id: str, the service id. path: str, File path to the staged deployment `.yaml` descriptor or to the original one, if no staging is used. """ def __init__(self, descriptor, source, service_info, upload_dir): self.descriptor = descriptor self.source = source self.service_info = service_info self.upload_dir = upload_dir @property def service_id(self): return self.service_info.module @property def path(self): return self.service_info.file @classmethod def FromPath(cls, path, stager, path_matchers, appyaml): """Return a Service from a path using staging if necessary. Args: path: str, Unsanitized absolute path, may point to a directory or a file of any type. There is no guarantee that it exists. stager: staging.Stager, stager that will be invoked if there is a runtime and environment match. path_matchers: List[Function], ordered list of functions on the form fn(path, stager), where fn returns a Service or None if no match. appyaml: str or None, the app.yaml location to used for deployment. Returns: Service, if one can be derived, else None. """ for matcher in path_matchers: service = matcher(path, stager, appyaml) if service: return service return None def ServiceYamlMatcher(path, stager, appyaml): """Generate a Service from an .yaml source path. This function is a path matcher that returns if and only if: - `path` points to either a `.yaml` or `` where `/app.yaml` exists. - the yaml-file is a valid .yaml file. If the runtime and environment match an entry in the stager, the service will be staged into a directory. Args: path: str, Unsanitized absolute path, may point to a directory or a file of any type. There is no guarantee that it exists. stager: staging.Stager, stager that will be invoked if there is a runtime and environment match. appyaml: str or None, the app.yaml location to used for deployment. Raises: staging.StagingCommandFailedError, staging command failed. Returns: Service, fully populated with entries that respect a potentially staged deployable service, or None if the path does not match the pattern described. """ descriptor = path if os.path.isfile(path) else os.path.join(path, 'app.yaml') _, ext = os.path.splitext(descriptor) if os.path.exists(descriptor) and ext in ['.yaml', '.yml']: app_dir = os.path.dirname(descriptor) service_info = yaml_parsing.ServiceYamlInfo.FromFile(descriptor) staging_dir = stager.Stage(descriptor, app_dir, service_info.runtime, service_info.env, appyaml) # If staging, stage, get stage_dir return Service(descriptor, app_dir, service_info, staging_dir or app_dir) return None def JarMatcher(jar_path, stager, appyaml): """Generate a Service from a Java fatjar path. This function is a path matcher that returns if and only if: - `jar_path` points to a jar file . The service will be staged according to the stager as a jar runtime, which is defined in staging.py. Args: jar_path: str, Unsanitized absolute path pointing to a file of jar type. stager: staging.Stager, stager that will be invoked if there is a runtime and environment match. appyaml: str or None, the app.yaml location to used for deployment. Raises: staging.StagingCommandFailedError, staging command failed. Returns: Service, fully populated with entries that respect a staged deployable service, or None if the path does not match the pattern described. """ _, ext = os.path.splitext(jar_path) if os.path.exists(jar_path) and ext in ['.jar']: app_dir = os.path.abspath(os.path.join(jar_path, os.pardir)) descriptor = jar_path staging_dir = stager.Stage(descriptor, app_dir, 'java-jar', env.STANDARD, appyaml) yaml_path = os.path.join(staging_dir, 'app.yaml') service_info = yaml_parsing.ServiceYamlInfo.FromFile(yaml_path) return Service(descriptor, app_dir, service_info, staging_dir) return None def PomXmlMatcher(path, stager, appyaml): """Generate a Service from an Maven project source path. This function is a path matcher that returns true if and only if: - `path` points to either a Maven `pom.xml` or `` where `/pom.xml` exists. If the runtime and environment match an entry in the stager, the service will be staged into a directory. Args: path: str, Unsanitized absolute path, may point to a directory or a file of any type. There is no guarantee that it exists. stager: staging.Stager, stager that will be invoked if there is a runtime and environment match. appyaml: str or None, the app.yaml location to used for deployment. Raises: staging.StagingCommandFailedError, staging command failed. Returns: Service, fully populated with entries that respect a potentially staged deployable service, or None if the path does not match the pattern described. """ descriptor = path if os.path.isfile(path) else os.path.join(path, 'pom.xml') filename = os.path.basename(descriptor) if os.path.exists(descriptor) and filename == 'pom.xml': app_dir = os.path.dirname(descriptor) staging_dir = stager.Stage(descriptor, app_dir, 'java-maven-project', env.STANDARD, appyaml) yaml_path = os.path.join(staging_dir, 'app.yaml') service_info = yaml_parsing.ServiceYamlInfo.FromFile(yaml_path) return Service(descriptor, app_dir, service_info, staging_dir) return None def BuildGradleMatcher(path, stager, appyaml): """Generate a Service from an Gradle project source path. This function is a path matcher that returns true if and only if: - `path` points to either a Gradle `build.gradle` or `` where `/build.gradle` exists. If the runtime and environment match an entry in the stager, the service will be staged into a directory. Args: path: str, Unsanitized absolute path, may point to a directory or a file of any type. There is no guarantee that it exists. stager: staging.Stager, stager that will be invoked if there is a runtime and environment match. appyaml: str or None, the app.yaml location to used for deployment. Raises: staging.StagingCommandFailedError, staging command failed. Returns: Service, fully populated with entries that respect a potentially staged deployable service, or None if the path does not match the pattern described. """ descriptor = path if os.path.isfile(path) else os.path.join( path, 'build.gradle') filename = os.path.basename(descriptor) if os.path.exists(descriptor) and filename == 'build.gradle': app_dir = os.path.dirname(descriptor) staging_dir = stager.Stage(descriptor, app_dir, 'java-gradle-project', env.STANDARD, appyaml) yaml_path = os.path.join(staging_dir, 'app.yaml') service_info = yaml_parsing.ServiceYamlInfo.FromFile(yaml_path) return Service(descriptor, app_dir, service_info, staging_dir) return None def AppengineWebMatcher(path, stager, appyaml): """Generate a Service from an appengine-web.xml source path. This function is a path matcher that returns if and only if: - `path` points to either `.../WEB-INF/appengine-web.xml` or `` where `/WEB-INF/appengine-web.xml` exists. - the xml-file is a valid appengine-web.xml file according to the Java stager. The service will be staged according to the stager as a java-xml runtime, which is defined in staging.py. Args: path: str, Unsanitized absolute path, may point to a directory or a file of any type. There is no guarantee that it exists. stager: staging.Stager, stager that will be invoked if there is a runtime and environment match. appyaml: str or None, the app.yaml location to used for deployment. Raises: staging.StagingCommandFailedError, staging command failed. Returns: Service, fully populated with entries that respect a staged deployable service, or None if the path does not match the pattern described. """ suffix = os.path.join(os.sep, 'WEB-INF', 'appengine-web.xml') app_dir = path[:-len(suffix)] if path.endswith(suffix) else path descriptor = os.path.join(app_dir, 'WEB-INF', 'appengine-web.xml') if not os.path.isfile(descriptor): return None xml_file = files.ReadFileContents(descriptor) if '' in xml_file or '' in xml_file: log.warning(' and elements in ' + '`appengine-web.xml` are not respected') staging_dir = stager.Stage(descriptor, app_dir, 'java-xml', env.STANDARD, appyaml) if not staging_dir: # After GA launch of appengine-web.xml support, this should never occur. return None yaml_path = os.path.join(staging_dir, 'app.yaml') service_info = yaml_parsing.ServiceYamlInfo.FromFile(yaml_path) return Service(descriptor, app_dir, service_info, staging_dir) def ExplicitAppYamlMatcher(path, stager, appyaml): """Use optional app.yaml with a directory or a file the user wants to deploy. Args: path: str, Unsanitized absolute path, may point to a directory or a file of any type. There is no guarantee that it exists. stager: staging.Stager, stager that will not be invoked. appyaml: str or None, the app.yaml location to used for deployment. Returns: Service, fully populated with entries that respect a staged deployable service, or None if there is no optional --appyaml flag usage. """ if appyaml: service_info = yaml_parsing.ServiceYamlInfo.FromFile(appyaml) staging_dir = stager.Stage(appyaml, path, 'generic-copy', service_info.env, appyaml) return Service(appyaml, path, service_info, staging_dir) return None def UnidentifiedDirMatcher(path, stager, appyaml): """Points out to the user that they need an app.yaml to deploy. Args: path: str, Unsanitized absolute path, may point to a directory or a file of any type. There is no guarantee that it exists. stager: staging.Stager, stager that will not be invoked. appyaml: str or None, the app.yaml location to used for deployment. Returns: None """ del stager, appyaml if os.path.isdir(path): log.error(NO_YAML_ERROR) return None def GetPathMatchers(): """Get list of path matchers ordered by descending precedence. Returns: List[Function], ordered list of functions on the form fn(path, stager), where fn returns a Service or None if no match. """ return [ ServiceYamlMatcher, AppengineWebMatcher, JarMatcher, PomXmlMatcher, BuildGradleMatcher, ExplicitAppYamlMatcher, UnidentifiedDirMatcher ] class Services(object): """Collection of deployable services.""" def __init__(self, services=None): """Instantiate a set of deployable services. Args: services: List[Service], optional list of services for quick initialization. Raises: DuplicateServiceError: Two or more services have the same service id. """ self._services = collections.OrderedDict() if services: for d in services: self.Add(d) def Add(self, service): """Add a deployable service to the set. Args: service: Service, to add. Raises: DuplicateServiceError: Two or more services have the same service id. """ existing = self._services.get(service.service_id) if existing: raise exceptions.DuplicateServiceError(existing.path, service.path, service.service_id) self._services[service.service_id] = service def GetAll(self): """Retrieve the service info objects in the order they were added. Returns: List[Service], list of services. """ return list(self._services.values()) class Configs(object): """Collection of config files.""" def __init__(self): self._configs = collections.OrderedDict() def Add(self, config): """Add a ConfigYamlInfo to the set of configs. Args: config: ConfigYamlInfo, the config to add. Raises: exceptions.DuplicateConfigError, the config type is already in the set. """ config_type = config.config existing = self._configs.get(config_type) if existing: raise exceptions.DuplicateConfigError(existing.file, config.file, config_type) self._configs[config_type] = config def GetAll(self): """Retreive the config file objects in the order they were added. Returns: List[ConfigYamlInfo], list of config file objects. """ return list(self._configs.values()) def GetDeployables(args, stager, path_matchers, appyaml=None): """Given a list of args, infer the deployable services and configs. Given a deploy command, e.g. `gcloud app deploy ./dir other/service.yaml cron.yaml WEB-INF/appengine-web.xml`, the deployables can be on multiple forms. This method pre-processes and infers yaml descriptors from the various formats accepted. The rules are as following: This function is a context manager, and should be used in conjunction with the `with` keyword. 1. If `args` is an empty list, add the current directory to it. 2. For each arg: - If arg refers to a config file, add it to the configs set. - Else match the arg against the path matchers. The first match will win. The match will be added to the services set. Matchers may run staging. Args: args: List[str], positional args as given on the command-line. stager: staging.Stager, stager that will be invoked on sources that have entries in the stager's registry. path_matchers: List[Function], list of functions on the form fn(path, stager) ordered by descending precedence, where fn returns a Service or None if no match. appyaml: str or None, the app.yaml location to used for deployment. Raises: FileNotFoundError: One or more argument does not point to an existing file or directory. UnknownSourceError: Could not infer a config or service from an arg. DuplicateConfigError: Two or more config files have the same type. DuplicateServiceError: Two or more services have the same service id. Returns: Tuple[List[Service], List[ConfigYamlInfo]], lists of deployable services and configs. """ if not args: args = ['.'] paths = [os.path.abspath(arg) for arg in args] configs = Configs() services = Services() if appyaml: if len(paths) > 1: raise exceptions.MultiDeployError() if not os.path.exists(os.path.abspath(appyaml)): raise exceptions.FileNotFoundError('File {0} referenced by --appyaml ' 'does not exist.'.format(appyaml)) if not os.path.exists(paths[0]): raise exceptions.FileNotFoundError(paths[0]) for path in paths: if not os.path.exists(path): raise exceptions.FileNotFoundError(path) config = yaml_parsing.ConfigYamlInfo.FromFile(path) if config: configs.Add(config) continue service = Service.FromPath(path, stager, path_matchers, appyaml) if service: services.Add(service) continue raise exceptions.UnknownSourceError(path) return services.GetAll(), configs.GetAll()