# -*- coding: utf-8 -*- # # Copyright 2016 Google LLC. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """General formatting utils, App Engine specific formatters.""" from __future__ import absolute_import from __future__ import division from __future__ import unicode_literals from googlecloudsdk.api_lib.logging import util from googlecloudsdk.core import log from googlecloudsdk.core import resources from googlecloudsdk.core.util import times import six LOG_LEVELS = ['critical', 'error', 'warning', 'info', 'debug', 'any'] # Request logs come from different sources if the app is Flex or Standard. FLEX_REQUEST = 'nginx.request' STANDARD_REQUEST = 'request_log' DEFAULT_LOGS = ['stderr', 'stdout', 'crash.log', FLEX_REQUEST, STANDARD_REQUEST] NGINX_LOGS = [ 'appengine.googleapis.com/nginx.request', 'appengine.googleapis.com/nginx.health_check'] def GetFilters(project, log_sources, service=None, version=None, level='any'): """Returns filters for App Engine app logs. Args: project: string name of project ID. log_sources: List of streams to fetch logs from. service: String name of service to fetch logs from. version: String name of version to fetch logs from. level: A string representing the severity of logs to fetch. Returns: A list of filter strings. """ filters = ['resource.type="gae_app"'] if service: filters.append('resource.labels.module_id="{0}"'.format(service)) if version: filters.append('resource.labels.version_id="{0}"'.format(version)) if level != 'any': filters.append('severity>={0}'.format(level.upper())) log_ids = [] for log_type in sorted(log_sources): log_ids.append('appengine.googleapis.com/{0}'.format(log_type)) if log_type in ('stderr', 'stdout'): log_ids.append(log_type) res = resources.REGISTRY.Parse( project, collection='appengine.projects').RelativeName() filters.append(_LogFilterForIds(log_ids, res)) return filters def _LogFilterForIds(log_ids, parent): """Constructs a log filter expression from the log_ids and parent name.""" if not log_ids: return None log_names = ['"{0}"'.format(util.CreateLogResourceName(parent, log_id)) for log_id in log_ids] log_names = ' OR '.join(log_names) if len(log_ids) > 1: log_names = '(%s)' % log_names return 'logName=%s' % log_names def FormatAppEntry(entry): """App Engine formatter for `LogPrinter`. Args: entry: A log entry message emitted from the V2 API client. Returns: A string representing the entry or None if there was no text payload. """ # TODO(b/36056460): Output others than text here too? if entry.resource.type != 'gae_app': return None if entry.protoPayload: text = six.text_type(entry.protoPayload) elif entry.jsonPayload: text = six.text_type(entry.jsonPayload) else: text = entry.textPayload service, version = _ExtractServiceAndVersion(entry) return '{service}[{version}] {text}'.format(service=service, version=version, text=text) def FormatRequestLogEntry(entry): """App Engine request_log formatter for `LogPrinter`. Args: entry: A log entry message emitted from the V2 API client. Returns: A string representing the entry if it is a request entry. """ if entry.resource.type != 'gae_app': return None log_id = util.ExtractLogId(entry.logName) if log_id != 'appengine.googleapis.com/request_log': return None service, version = _ExtractServiceAndVersion(entry) def GetStr(key): return next((x.value.string_value for x in entry.protoPayload.additionalProperties if x.key == key), '-') def GetInt(key): return next((x.value.integer_value for x in entry.protoPayload.additionalProperties if x.key == key), '-') msg = ('"{method} {resource} {http_version}" {status}' .format( method=GetStr('method'), resource=GetStr('resource'), http_version=GetStr('httpVersion'), status=GetInt('status'))) return '{service}[{version}] {msg}'.format(service=service, version=version, msg=msg) def FormatNginxLogEntry(entry): """App Engine nginx.* formatter for `LogPrinter`. Args: entry: A log entry message emitted from the V2 API client. Returns: A string representing the entry if it is a request entry. """ if entry.resource.type != 'gae_app': return None log_id = util.ExtractLogId(entry.logName) if log_id not in NGINX_LOGS: return None service, version = _ExtractServiceAndVersion(entry) msg = ('"{method} {resource}" {status}' .format( method=entry.httpRequest.requestMethod or '-', resource=entry.httpRequest.requestUrl or '-', status=entry.httpRequest.status or '-')) return '{service}[{version}] {msg}'.format(service=service, version=version, msg=msg) def _ExtractServiceAndVersion(entry): """Extract service and version from a App Engine log entry. Args: entry: An App Engine log entry. Returns: A 2-tuple of the form (service_id, version_id) """ # TODO(b/36051034): If possible, extract instance ID too ad_prop = entry.resource.labels.additionalProperties service = next(x.value for x in ad_prop if x.key == 'module_id') version = next(x.value for x in ad_prop if x.key == 'version_id') return (service, version) class LogPrinter(object): """Formats V2 API log entries to human readable text on a best effort basis. A LogPrinter consists of a collection of formatter functions which attempts to format specific log entries in a human readable form. The `Format` method safely returns a human readable string representation of a log entry, even if the provided formatters fails. The output format is `{timestamp} {log_text}`, where `timestamp` has a configurable but consistent format within a LogPrinter whereas `log_text` is emitted from one of its formatters (and truncated if necessary). See https://cloud.google.com/logging/docs/api/introduction_v2 Attributes: api_time_format: str, the output format to print. See datetime.strftime() max_length: The maximum length of a formatted log entry after truncation. """ def __init__(self, api_time_format='%Y-%m-%d %H:%M:%S', max_length=None): self.formatters = [] self.api_time_format = api_time_format self.max_length = max_length def Format(self, entry): """Safely formats a log entry into human readable text. Args: entry: A log entry message emitted from the V2 API client. Returns: A string without line breaks respecting the `max_length` property. """ text = self._LogEntryToText(entry) text = text.strip().replace('\n', ' ') try: time = times.FormatDateTime(times.ParseDateTime(entry.timestamp), self.api_time_format) except times.Error: log.warning('Received timestamp [{0}] does not match expected' ' format.'.format(entry.timestamp)) time = '????-??-?? ??:??:??' out = '{timestamp} {log_text}'.format( timestamp=time, log_text=text) if self.max_length and len(out) > self.max_length: out = out[:self.max_length - 3] + '...' return out def RegisterFormatter(self, formatter): """Attach a log entry formatter function to the printer. Note that if multiple formatters are attached to the same printer, the first added formatter that successfully formats the entry will be used. Args: formatter: A formatter function which accepts a single argument, a log entry. The formatter must either return the formatted log entry as a string, or None if it is unable to format the log entry. The formatter is allowed to raise exceptions, which will be caught and ignored by the printer. """ self.formatters.append(formatter) def _LogEntryToText(self, entry): """Use the formatters to convert a log entry to unprocessed text.""" out = None for fn in self.formatters + [self._FallbackFormatter]: # pylint:disable=bare-except try: out = fn(entry) if out: break except KeyboardInterrupt as e: raise e except: pass if not out: log.debug('Could not format log entry: %s %s %s', entry.timestamp, entry.logName, entry.insertId) out = ('< UNREADABLE LOG ENTRY {0}. OPEN THE DEVELOPER CONSOLE TO ' 'INSPECT. >'.format(entry.insertId)) return out def _FallbackFormatter(self, entry): # TODO(b/36057358): Is there better serialization for messages than # six.text_type()? if entry.protoPayload: return six.text_type(entry.protoPayload) elif entry.jsonPayload: return six.text_type(entry.jsonPayload) else: return entry.textPayload