499 lines
20 KiB
Python
499 lines
20 KiB
Python
# Copyright 2016 Google LLC. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""Library with a variant of appengine_rpc using httplib2.
|
|
|
|
The httplib2 module offers some of the features in appengine_rpc, with
|
|
one important one being a simple integration point for OAuth2 integration.
|
|
"""
|
|
|
|
from __future__ import absolute_import
|
|
# pylint: disable=g-bad-name,g-import-not-at-top
|
|
|
|
import io
|
|
import logging
|
|
import os
|
|
import random
|
|
import re
|
|
import time
|
|
import urllib
|
|
|
|
import httplib2
|
|
|
|
from oauth2client import client
|
|
from oauth2client import file as oauth2client_file
|
|
from oauth2client import tools
|
|
from googlecloudsdk.core.util import encoding
|
|
from googlecloudsdk.appengine.tools.value_mixin import ValueMixin
|
|
from googlecloudsdk.appengine._internal import six_subset
|
|
|
|
|
|
# pylint:disable=g-import-not-at-top
|
|
# pylint:disable=invalid-name
|
|
# Inline these directly rather than placing in six_subset since importing
|
|
# urllib into six_subset seems to mess with the overridden version of
|
|
# urllib/httplib that the NaCl runtime sandbox inserts for SSL purposes.
|
|
if six_subset.PY3:
|
|
HTTPError = urllib.error.HTTPError
|
|
urlencode_fn = urllib.parse.urlencode
|
|
else:
|
|
import urllib2
|
|
HTTPError = urllib2.HTTPError
|
|
urlencode_fn = urllib.urlencode
|
|
# pylint:disable=g-import-not-at-top
|
|
# pylint:disable=invalid-name
|
|
|
|
logger = logging.getLogger('googlecloudsdk.appengine.tools.appengine_rpc')
|
|
|
|
|
|
_TIMEOUT_WAIT_TIME = 5
|
|
|
|
|
|
class Error(Exception):
|
|
pass
|
|
|
|
|
|
class AuthPermanentFail(Error):
|
|
"""Authentication will not succeed in the current context."""
|
|
|
|
|
|
class MemoryCache(object):
|
|
"""httplib2 Cache implementation which only caches locally."""
|
|
|
|
def __init__(self):
|
|
self.cache = {}
|
|
|
|
def get(self, key):
|
|
return self.cache.get(key)
|
|
|
|
def set(self, key, value):
|
|
self.cache[key] = value
|
|
|
|
def delete(self, key):
|
|
self.cache.pop(key, None)
|
|
|
|
|
|
def RaiseHttpError(url, response_info, response_body, extra_msg=''):
|
|
"""Raise a urllib2.HTTPError based on an httplib2 response tuple."""
|
|
if response_body is not None:
|
|
stream = io.BytesIO()
|
|
stream.write(response_body)
|
|
stream.seek(0)
|
|
else:
|
|
stream = None
|
|
if not extra_msg:
|
|
msg = response_info.reason
|
|
else:
|
|
msg = response_info.reason + ' ' + extra_msg
|
|
raise HTTPError(url, response_info.status, msg, response_info, stream)
|
|
|
|
|
|
class HttpRpcServerHttpLib2(object):
|
|
"""A variant of HttpRpcServer which uses httplib2.
|
|
|
|
This follows the same interface as appengine_rpc.AbstractRpcServer,
|
|
but is a totally separate implementation.
|
|
"""
|
|
|
|
def __init__(self, host, auth_function, user_agent, source,
|
|
host_override=None, extra_headers=None, save_cookies=False,
|
|
auth_tries=None, account_type=None, debug_data=True, secure=True,
|
|
ignore_certs=False, rpc_tries=3, conflict_max_errors=10,
|
|
timeout_max_errors=2, http_class=None, http_object=None):
|
|
"""Creates a new HttpRpcServerHttpLib2.
|
|
|
|
Args:
|
|
host: The host to send requests to.
|
|
auth_function: Saved but ignored; may be used by subclasses.
|
|
user_agent: The user-agent string to send to the server. Specify None to
|
|
omit the user-agent header.
|
|
source: Saved but ignored; may be used by subclasses.
|
|
host_override: The host header to send to the server (defaults to host).
|
|
extra_headers: A dict of extra headers to append to every request. Values
|
|
supplied here will override other default headers that are supplied.
|
|
save_cookies: Saved but ignored; may be used by subclasses.
|
|
auth_tries: The number of times to attempt auth_function before failing.
|
|
account_type: Saved but ignored; may be used by subclasses.
|
|
debug_data: Whether debugging output should include data contents.
|
|
secure: If the requests sent using Send should be sent over HTTPS.
|
|
ignore_certs: If the certificate mismatches should be ignored.
|
|
rpc_tries: The number of rpc retries upon http server error (i.e.
|
|
Response code >= 500 and < 600) before failing.
|
|
conflict_max_errors: The number of rpc retries upon http server error
|
|
(i.e. Response code 409) before failing.
|
|
timeout_max_errors: The number of rpc retries upon http server timeout
|
|
(i.e. Response code 408) before failing.
|
|
http_class: the httplib2.Http subclass to use. Defaults to httplib2.Http.
|
|
http_object: an httlib2.Http object to use to make requests. If this is
|
|
provided, http_class is ignored.
|
|
"""
|
|
self.host = host
|
|
self.auth_function = auth_function
|
|
self.user_agent = user_agent
|
|
self.source = source
|
|
self.host_override = host_override
|
|
self.extra_headers = extra_headers or {}
|
|
self.save_cookies = save_cookies
|
|
self.auth_max_errors = auth_tries
|
|
self.account_type = account_type
|
|
self.debug_data = debug_data
|
|
self.secure = secure
|
|
self.ignore_certs = ignore_certs
|
|
self.rpc_max_errors = rpc_tries
|
|
self.scheme = secure and 'https' or 'http'
|
|
self.conflict_max_errors = conflict_max_errors
|
|
self.timeout_max_errors = timeout_max_errors
|
|
self.http_class = http_class if http_class is not None else httplib2.Http
|
|
self.http_object = http_object
|
|
|
|
self.certpath = None
|
|
self.cert_file_available = False
|
|
if not self.ignore_certs:
|
|
# Use the App Engine managed cacerts file instead of the 'httplib2' copy.
|
|
# should probably let the httplib2 copy get used.
|
|
self.certpath = os.path.normpath(os.path.join(
|
|
os.path.dirname(__file__), '..', '..', '..', 'lib', 'cacerts',
|
|
'cacerts.txt'))
|
|
self.cert_file_available = os.path.exists(self.certpath)
|
|
|
|
self.memory_cache = MemoryCache()
|
|
|
|
def _Authenticate(self, http, saw_error):
|
|
"""Pre or Re-auth stuff...
|
|
|
|
Args:
|
|
http: An 'Http' object from httplib2.
|
|
saw_error: If the user has already tried to contact the server.
|
|
If they have, it's OK to prompt them. If not, we should not be asking
|
|
them for auth info--it's possible it'll suceed w/o auth.
|
|
"""
|
|
# Note that this should possibly have direct access to headers, etc.
|
|
# But most of the 'httplib2' methods work on the 'Http' object.
|
|
raise NotImplementedError()
|
|
|
|
def Send(self, request_path, payload='',
|
|
content_type='application/octet-stream',
|
|
timeout=None,
|
|
**kwargs):
|
|
"""Sends an RPC and returns the response.
|
|
|
|
Args:
|
|
request_path: The path to send the request to, eg /api/appversion/create.
|
|
payload: The body of the request, or None to send an empty request.
|
|
content_type: The Content-Type header to use.
|
|
timeout: timeout in seconds; default None i.e. no timeout.
|
|
(Note: for large requests on OS X, the timeout doesn't work right.)
|
|
Any keyword arguments are converted into query string parameters.
|
|
|
|
Returns:
|
|
The response body, as a string.
|
|
|
|
Raises:
|
|
AuthPermanentFail: If authorization failed in a permanent way.
|
|
urllib2.HTTPError: On most HTTP errors.
|
|
"""
|
|
# TODO(user): To prevent raising httplib2.CertificateValidationUnsupported
|
|
# we need to track self.cert_file_available and send
|
|
# disable_ssl_certificate_validation on the Http(constructor).
|
|
# Though perhaps we should require proper SSL support if you're using oauth.
|
|
|
|
self.http = self.http_object or self.http_class(
|
|
cache=self.memory_cache, ca_certs=self.certpath,
|
|
disable_ssl_certificate_validation=(not self.cert_file_available))
|
|
self.http.follow_redirects = False
|
|
self.http.timeout = timeout
|
|
url = '%s://%s%s' % (self.scheme, self.host, request_path)
|
|
if kwargs:
|
|
url += '?' + urlencode_fn(sorted(kwargs.items()))
|
|
headers = {}
|
|
if self.extra_headers:
|
|
headers.update(self.extra_headers)
|
|
# This header is necessary to prevent XSRF attacks, since the browser
|
|
# cannot include this header, that means the request had to come from
|
|
# another agent like appcfg.py.
|
|
headers['X-appcfg-api-version'] = '1'
|
|
# POST if there's a payload (which may be empty). GET if there's no payload.
|
|
if payload is not None:
|
|
method = 'POST'
|
|
# For some reason, content-length is not sent automatically.
|
|
headers['content-length'] = str(len(payload))
|
|
headers['Content-Type'] = content_type
|
|
else:
|
|
method = 'GET'
|
|
if self.host_override:
|
|
headers['Host'] = self.host_override
|
|
|
|
rpc_errors = 0
|
|
auth_errors = [0]
|
|
conflict_errors = 0
|
|
timeout_errors = 0
|
|
|
|
def NeedAuth():
|
|
"""Marker that we need auth; it'll actually be tried next time around."""
|
|
auth_errors[0] += 1
|
|
logger.debug('Attempting to auth. This is try %s of %s.',
|
|
auth_errors[0], self.auth_max_errors)
|
|
if auth_errors[0] > self.auth_max_errors:
|
|
RaiseHttpError(url, response_info, response, 'Too many auth attempts.')
|
|
|
|
while (rpc_errors < self.rpc_max_errors and
|
|
conflict_errors < self.conflict_max_errors and
|
|
timeout_errors < self.timeout_max_errors):
|
|
self._Authenticate(self.http, auth_errors[0] > 0)
|
|
logger.debug('Sending request to %s headers=%s body=%s',
|
|
url, headers,
|
|
self.debug_data and payload or payload and 'ELIDED' or '')
|
|
try:
|
|
response_info, response = self.http.request(
|
|
url, method=method, body=payload, headers=headers)
|
|
except client.AccessTokenRefreshError as e:
|
|
# Consider this a 401.
|
|
logger.info('Got access token error', exc_info=1)
|
|
response_info = httplib2.Response({'status': 401})
|
|
response_info.reason = str(e)
|
|
response = ''
|
|
|
|
status = response_info.status
|
|
if status == 200:
|
|
return response
|
|
logger.debug('Got http error %s.', response_info.status)
|
|
if status == 401:
|
|
NeedAuth()
|
|
continue
|
|
elif status == 408:
|
|
timeout_errors += 1
|
|
logger.debug('Got timeout error %s of %s. Retrying in %s seconds',
|
|
timeout_errors, self.timeout_max_errors,
|
|
_TIMEOUT_WAIT_TIME)
|
|
time.sleep(_TIMEOUT_WAIT_TIME)
|
|
continue
|
|
elif status == 409:
|
|
conflict_errors += 1
|
|
# Retry with jitter.
|
|
wait_time = random.randint(0, 10)
|
|
logger.debug('Got conflict error %s of %s. Retrying in %s seconds.',
|
|
conflict_errors, self.conflict_max_errors, wait_time)
|
|
time.sleep(wait_time)
|
|
continue
|
|
elif status >= 500 and status < 600:
|
|
# Server Error - try again.
|
|
rpc_errors += 1
|
|
logger.debug('Retrying. This is attempt %s of %s.',
|
|
rpc_errors, self.rpc_max_errors)
|
|
continue
|
|
elif status == 302:
|
|
# Server may also return a 302 redirect to indicate authentication
|
|
# is required.
|
|
loc = response_info.get('location')
|
|
logger.debug('Got 302 redirect. Location: %s', loc)
|
|
if (loc.startswith('https://www.google.com/accounts/ServiceLogin') or
|
|
re.match(r'https://www\.google\.com/a/[a-z0-9.-]+/ServiceLogin',
|
|
loc)):
|
|
NeedAuth()
|
|
continue
|
|
elif loc.startswith('http://%s/_ah/login' % (self.host,)):
|
|
# We can probably stuff a fake header in here.
|
|
RaiseHttpError(url, response_info, response,
|
|
'dev_appserver login not supported')
|
|
else:
|
|
RaiseHttpError(url, response_info, response,
|
|
'Unexpected redirect to %s' % loc)
|
|
else:
|
|
logger.debug('Unexpected results: %s', response_info)
|
|
RaiseHttpError(url, response_info, response,
|
|
'Unexpected HTTP status %s' % status)
|
|
logging.info('Too many retries for url %s', url)
|
|
RaiseHttpError(url, response_info, response)
|
|
|
|
|
|
class NoStorage(client.Storage):
|
|
"""A no-op implementation of storage."""
|
|
|
|
def locked_get(self):
|
|
return None
|
|
|
|
def locked_put(self, credentials):
|
|
pass
|
|
|
|
|
|
class HttpRpcServerOAuth2(HttpRpcServerHttpLib2):
|
|
"""A variant of HttpRpcServer which uses oauth2.
|
|
|
|
This variant is specifically meant for interactive command line usage,
|
|
as it will attempt to open a browser and ask the user to enter
|
|
information from the resulting web page.
|
|
"""
|
|
|
|
class OAuth2Parameters(ValueMixin):
|
|
"""Class encapsulating parameters related to OAuth2 authentication."""
|
|
|
|
def __init__(self, access_token, client_id, client_secret, scope,
|
|
refresh_token, credential_file, token_uri=None,
|
|
credentials=None):
|
|
self.access_token = access_token
|
|
self.client_id = client_id
|
|
self.client_secret = client_secret
|
|
self.scope = scope
|
|
self.refresh_token = refresh_token
|
|
self.credential_file = credential_file
|
|
self.token_uri = token_uri
|
|
self.credentials = credentials
|
|
|
|
class FlowFlags(object):
|
|
|
|
def __init__(self, options):
|
|
self.logging_level = logging.getLevelName(logging.getLogger().level)
|
|
self.noauth_local_webserver = (not options.auth_local_webserver
|
|
if options else True)
|
|
self.auth_host_port = [8080, 8090]
|
|
self.auth_host_name = 'localhost'
|
|
|
|
def __init__(self, host, oauth2_parameters, user_agent, source,
|
|
host_override=None, extra_headers=None, save_cookies=False,
|
|
auth_tries=None, account_type=None, debug_data=True, secure=True,
|
|
ignore_certs=False, rpc_tries=3, timeout_max_errors=2,
|
|
options=None, http_class=None, http_object=None):
|
|
"""Creates a new HttpRpcServerOAuth2.
|
|
|
|
Args:
|
|
host: The host to send requests to.
|
|
oauth2_parameters: An object of type OAuth2Parameters (defined above)
|
|
that specifies all parameters related to OAuth2 authentication. (This
|
|
replaces the auth_function parameter in the parent class.)
|
|
user_agent: The user-agent string to send to the server. Specify None to
|
|
omit the user-agent header.
|
|
source: Saved but ignored.
|
|
host_override: The host header to send to the server (defaults to host).
|
|
extra_headers: A dict of extra headers to append to every request. Values
|
|
supplied here will override other default headers that are supplied.
|
|
save_cookies: If the refresh token should be saved.
|
|
auth_tries: The number of times to attempt auth_function before failing.
|
|
account_type: Ignored.
|
|
debug_data: Whether debugging output should include data contents.
|
|
secure: If the requests sent using Send should be sent over HTTPS.
|
|
ignore_certs: If the certificate mismatches should be ignored.
|
|
rpc_tries: The number of rpc retries upon http server error (i.e.
|
|
Response code >= 500 and < 600) before failing.
|
|
timeout_max_errors: The number of rpc retries upon http server timeout
|
|
(i.e. Response code 408) before failing.
|
|
options: the command line options.
|
|
http_class: the httplib2.Http subclass to use. Defaults to httplib2.Http.
|
|
http_object: an httlib2.Http object to use to make requests. If this is
|
|
provided, http_class is ignored.
|
|
"""
|
|
super(HttpRpcServerOAuth2, self).__init__(
|
|
host, None, user_agent, source, host_override=host_override,
|
|
extra_headers=extra_headers, auth_tries=auth_tries,
|
|
debug_data=debug_data, secure=secure, ignore_certs=ignore_certs,
|
|
rpc_tries=rpc_tries, timeout_max_errors=timeout_max_errors,
|
|
save_cookies=save_cookies, http_class=http_class,
|
|
http_object=http_object)
|
|
|
|
if not isinstance(oauth2_parameters, self.OAuth2Parameters):
|
|
raise TypeError('oauth2_parameters must be an OAuth2Parameters: %r' %
|
|
oauth2_parameters)
|
|
self.oauth2_parameters = oauth2_parameters
|
|
|
|
if save_cookies:
|
|
oauth2_credential_file = (oauth2_parameters.credential_file
|
|
or '~/.appcfg_oauth2_tokens')
|
|
self.storage = oauth2client_file.Storage(
|
|
os.path.expanduser(oauth2_credential_file))
|
|
else:
|
|
self.storage = NoStorage()
|
|
|
|
if oauth2_parameters.credentials:
|
|
self.credentials = oauth2_parameters.credentials
|
|
elif any((oauth2_parameters.access_token, oauth2_parameters.refresh_token,
|
|
oauth2_parameters.token_uri)):
|
|
token_uri = (oauth2_parameters.token_uri or
|
|
('https://%s/o/oauth2/token' %
|
|
encoding.GetEncodedValue(
|
|
os.environ, 'APPENGINE_AUTH_SERVER',
|
|
'accounts.google.com')))
|
|
self.credentials = client.OAuth2Credentials(
|
|
oauth2_parameters.access_token,
|
|
oauth2_parameters.client_id,
|
|
oauth2_parameters.client_secret,
|
|
oauth2_parameters.refresh_token,
|
|
None,
|
|
token_uri,
|
|
self.user_agent)
|
|
else:
|
|
self.credentials = self.storage.get()
|
|
|
|
self.flags = self.FlowFlags(options)
|
|
|
|
def _Authenticate(self, http, needs_auth):
|
|
"""Pre or Re-auth stuff...
|
|
|
|
This will attempt to avoid making any OAuth related HTTP connections or
|
|
user interactions unless it's needed.
|
|
|
|
Args:
|
|
http: An 'Http' object from httplib2.
|
|
needs_auth: If the user has already tried to contact the server.
|
|
If they have, it's OK to prompt them. If not, we should not be asking
|
|
them for auth info--it's possible it'll suceed w/o auth, but if we have
|
|
some credentials we'll use them anyway.
|
|
|
|
Raises:
|
|
AuthPermanentFail: The user has requested non-interactive auth but
|
|
the token is invalid.
|
|
"""
|
|
if needs_auth and (not self.credentials or self.credentials.invalid):
|
|
# If we were given either an access token or a refresh token on the
|
|
# command line then we assume that we don't want interactive login.
|
|
# Likewise we interpret a non-default token_uri as meaning we are using
|
|
# something like the GCE Metadata Service so again it is not interactive.
|
|
if self.oauth2_parameters.access_token:
|
|
logger.debug('_Authenticate skipping auth because user explicitly '
|
|
'supplied an access token.')
|
|
raise AuthPermanentFail('Access token is invalid.')
|
|
if self.oauth2_parameters.refresh_token:
|
|
logger.debug('_Authenticate skipping auth because user explicitly '
|
|
'supplied a refresh token.')
|
|
raise AuthPermanentFail('Refresh token is invalid.')
|
|
if self.oauth2_parameters.token_uri:
|
|
logger.debug('_Authenticate skipping auth because user explicitly '
|
|
'supplied a Token URI, for example for service account '
|
|
'authentication with Compute Engine')
|
|
raise AuthPermanentFail('Token URI did not yield a valid token: ' +
|
|
self.oauth_parameters.token_uri)
|
|
logger.debug('_Authenticate requesting auth')
|
|
flow = client.OAuth2WebServerFlow(
|
|
client_id=self.oauth2_parameters.client_id,
|
|
client_secret=self.oauth2_parameters.client_secret,
|
|
scope=_ScopesToString(self.oauth2_parameters.scope),
|
|
user_agent=self.user_agent)
|
|
self.credentials = tools.run_flow(flow, self.storage, self.flags)
|
|
if self.credentials and not self.credentials.invalid:
|
|
# We will configure this automatically if either we think the access token
|
|
# is valid, or we think we need a token and have a refresh token
|
|
if not self.credentials.access_token_expired or needs_auth:
|
|
logger.debug('_Authenticate configuring auth; needs_auth=%s',
|
|
needs_auth)
|
|
self.credentials.authorize(http)
|
|
return
|
|
logger.debug('_Authenticate skipped auth; needs_auth=%s', needs_auth)
|
|
|
|
|
|
def _ScopesToString(scopes):
|
|
"""Converts scope value to a string."""
|
|
# TODO(user): replace with oauth2client.util.scopes_to_string when we
|
|
# have a more recent oauth2client.
|
|
if isinstance(scopes, six_subset.string_types):
|
|
return scopes
|
|
else:
|
|
return ' '.join(scopes)
|