1029 lines
38 KiB
Python
1029 lines
38 KiB
Python
# -*- coding: utf-8 -*- #
|
|
# Copyright 2020 Google LLC. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""A module to get an unauthenticated requests.Session object."""
|
|
|
|
from __future__ import absolute_import
|
|
from __future__ import division
|
|
from __future__ import unicode_literals
|
|
|
|
import abc
|
|
import atexit
|
|
import collections
|
|
import inspect
|
|
import io
|
|
import json
|
|
import os
|
|
import secrets
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
from google.auth.transport import requests as google_auth_requests
|
|
from google.auth.transport.requests import _MutualTlsOffloadAdapter
|
|
from googlecloudsdk.core import context_aware
|
|
from googlecloudsdk.core import execution_utils
|
|
from googlecloudsdk.core import log
|
|
from googlecloudsdk.core import properties
|
|
from googlecloudsdk.core import transport
|
|
from googlecloudsdk.core.util import encoding
|
|
from googlecloudsdk.core.util import http_proxy_types
|
|
from googlecloudsdk.core.util import platforms
|
|
import httplib2
|
|
import requests
|
|
import six
|
|
from six.moves import http_client as httplib
|
|
from six.moves import urllib
|
|
import socks
|
|
from urllib3.util.ssl_ import create_urllib3_context
|
|
|
|
try:
|
|
import urllib.request as urllib_request # pylint: disable=g-import-not-at-top
|
|
except ImportError: # PY2
|
|
import urllib as urllib_request # pylint: disable=g-import-not-at-top
|
|
|
|
|
|
_INVALID_HTTPS_PROXY_ENV_VAR_WARNING = (
|
|
'It appears that the current proxy configuration is using an HTTPS scheme '
|
|
'for contacting the proxy server, which likely indicates an error in your '
|
|
'HTTPS_PROXY environment variable setting. This can usually be resolved '
|
|
'by setting HTTPS_PROXY=http://... instead of HTTPS_PROXY=https://... '
|
|
'See https://cloud.google.com/sdk/docs/proxy-settings for more information.'
|
|
)
|
|
_invalid_https_proxy_env_var_warning_shown = False
|
|
|
|
|
|
def GetSession(timeout='unset',
|
|
ca_certs=None,
|
|
session=None,
|
|
streaming_response_body=False,
|
|
redact_request_body_reason=None,
|
|
client_certificate=None,
|
|
client_key=None,):
|
|
"""Get a requests.Session that is properly configured for use by gcloud.
|
|
|
|
This method does not add credentials to the client. For a requests.Session
|
|
that has been authenticated, use core.credentials.requests.GetSession().
|
|
|
|
Args:
|
|
timeout: double, The timeout in seconds. This is the
|
|
socket level timeout. If timeout is None, timeout is infinite. If
|
|
default argument 'unset' is given, a sensible default is selected using
|
|
transport.GetDefaultTimeout().
|
|
ca_certs: str, absolute filename of a ca_certs file that overrides the
|
|
default. The gcloud config property for ca_certs, in turn, overrides
|
|
this argument.
|
|
session: requests.Session instance
|
|
streaming_response_body: bool, True indicates that the response body will
|
|
be a streaming body.
|
|
redact_request_body_reason: str, the reason why the request body must be
|
|
redacted if --log-http is used. If None, the body is not redacted.
|
|
client_certificate: str, absolute filename of a client_certificate file that
|
|
is set explicitly for client certificate authentication
|
|
client_key: str, absolute filename of a client_key file that
|
|
is set explicitly for client certificate authentication
|
|
|
|
Returns:
|
|
A requests.Session object configured with all the required settings
|
|
for gcloud.
|
|
"""
|
|
http_client = _CreateRawSession(timeout, ca_certs, session,
|
|
client_certificate, client_key)
|
|
http_client = RequestWrapper().WrapWithDefaults(
|
|
http_client,
|
|
streaming_response_body=streaming_response_body,
|
|
redact_request_body_reason=redact_request_body_reason)
|
|
return http_client
|
|
|
|
|
|
class ClientSideCertificate(
|
|
collections.namedtuple('ClientSideCertificate',
|
|
['certfile', 'keyfile', 'password'])):
|
|
"""Holds information about a client side certificate.
|
|
|
|
Attributes:
|
|
certfile: str, path to a cert file.
|
|
keyfile: str, path to a key file.
|
|
password: str, password to the private key.
|
|
"""
|
|
|
|
def __new__(cls, certfile, keyfile, password=None):
|
|
return super(ClientSideCertificate, cls).__new__(
|
|
cls, certfile, keyfile, password)
|
|
|
|
|
|
def CreateSSLContext():
|
|
"""Returns a urrlib3 SSL context."""
|
|
return create_urllib3_context()
|
|
|
|
|
|
class HTTPAdapter(requests.adapters.HTTPAdapter):
|
|
"""Transport adapter for requests.
|
|
|
|
Transport adapters provide an interface to extend the default behavior of the
|
|
requests library using the full power of the underlying urrlib3 library.
|
|
|
|
See https://requests.readthedocs.io/en/master/user/advanced/
|
|
#transport-adapters for more information about adapters.
|
|
"""
|
|
|
|
def __init__(self, client_side_certificate, *args, **kwargs):
|
|
self._cert_info = client_side_certificate
|
|
super(HTTPAdapter, self).__init__(*args, **kwargs)
|
|
|
|
def init_poolmanager(self, *args, **kwargs):
|
|
self._add_ssl_context(kwargs)
|
|
return super(HTTPAdapter, self).init_poolmanager(*args, **kwargs)
|
|
|
|
def proxy_manager_for(self, *args, **kwargs):
|
|
self._add_ssl_context(kwargs)
|
|
return super(HTTPAdapter, self).proxy_manager_for(*args, **kwargs)
|
|
|
|
def _add_ssl_context(self, kwargs):
|
|
if not self._cert_info:
|
|
return
|
|
|
|
context = CreateSSLContext()
|
|
context.load_default_certs()
|
|
|
|
cert_chain_kwargs = {}
|
|
if self._cert_info.keyfile:
|
|
cert_chain_kwargs['keyfile'] = self._cert_info.keyfile
|
|
if self._cert_info.password:
|
|
cert_chain_kwargs['password'] = self._cert_info.password
|
|
|
|
context.load_cert_chain(self._cert_info.certfile, **cert_chain_kwargs)
|
|
|
|
kwargs['ssl_context'] = context
|
|
|
|
|
|
def GetProxyInfo():
|
|
"""Returns the proxy string for use by requests from gcloud properties.
|
|
|
|
See https://requests.readthedocs.io/en/master/user/advanced/#proxies.
|
|
"""
|
|
proxy_type = properties.VALUES.proxy.proxy_type.Get()
|
|
proxy_address = properties.VALUES.proxy.address.Get()
|
|
proxy_port = properties.VALUES.proxy.port.GetInt()
|
|
|
|
proxy_prop_set = len(
|
|
[f for f in (proxy_type, proxy_address, proxy_port) if f])
|
|
if proxy_prop_set > 0 and proxy_prop_set != 3:
|
|
raise properties.InvalidValueError(
|
|
'Please set all or none of the following properties: '
|
|
'proxy/type, proxy/address and proxy/port')
|
|
|
|
if not proxy_prop_set:
|
|
return
|
|
|
|
proxy_rdns = properties.VALUES.proxy.rdns.GetBool()
|
|
proxy_user = properties.VALUES.proxy.username.Get()
|
|
proxy_pass = properties.VALUES.proxy.password.Get()
|
|
|
|
http_proxy_type = http_proxy_types.PROXY_TYPE_MAP[proxy_type]
|
|
if http_proxy_type == socks.PROXY_TYPE_SOCKS4:
|
|
proxy_scheme = 'socks4a' if proxy_rdns else 'socks4'
|
|
elif http_proxy_type == socks.PROXY_TYPE_SOCKS5:
|
|
proxy_scheme = 'socks5h' if proxy_rdns else 'socks5'
|
|
elif http_proxy_type == socks.PROXY_TYPE_HTTP:
|
|
proxy_scheme = 'http'
|
|
else:
|
|
raise ValueError('Unsupported proxy type: {}'.format(proxy_type))
|
|
|
|
if proxy_user or proxy_pass:
|
|
proxy_auth = ':'.join(
|
|
urllib.parse.quote(x) or '' for x in (proxy_user, proxy_pass))
|
|
proxy_auth += '@'
|
|
else:
|
|
proxy_auth = ''
|
|
return '{}://{}{}:{}'.format(proxy_scheme, proxy_auth, proxy_address,
|
|
proxy_port)
|
|
|
|
|
|
_GOOGLER_BUNDLED_PYTHON_WARNING = (
|
|
'Please use the installed gcloud CLI (`apt install google-cloud-cli`)\n'
|
|
' This version of gcloud you are currently using will encounter issues due'
|
|
' to\n changes in internal security policy enforcement in the near'
|
|
' future.\n\n If this is not possible due to dev requirements, please apply'
|
|
' for\n policy exemption at go/gcloud-cba-exemption-internal-version-gcloud'
|
|
' using this error message to self-exempt or reach out to\n'
|
|
' go/gcloud-cba-investigation for investigation.\n'
|
|
)
|
|
|
|
|
|
class ECPProxyError(Exception):
|
|
"""Custom exception for errors related to the ECP proxy communication.
|
|
|
|
For example, startup failures or receiving a specific proxy error header.
|
|
"""
|
|
|
|
def __init__(self, message, original_exception=None):
|
|
self.original_exception = original_exception
|
|
super().__init__(
|
|
f'{message}: {original_exception}' if original_exception else message
|
|
)
|
|
|
|
|
|
class _LocalECPProxyAdapter(requests.adapters.HTTPAdapter):
|
|
"""A requests adapter that routes HTTPS requests through a local ECP proxy.
|
|
|
|
This adapter starts the ECP proxy as a background process upon instantiation
|
|
and manages its lifecycle, terminating it when the adapter is closed.
|
|
This avoids the overhead of starting a new process for every request.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
certificate_config_file_path: str,
|
|
gcloud_proxy_url: str = None,
|
|
startup_timeout: int = 5,
|
|
**kwargs,
|
|
):
|
|
"""Initializes the adapter and starts the local ECP proxy process.
|
|
|
|
Args:
|
|
certificate_config_file_path: Path to the JSON certificate config file.
|
|
gcloud_proxy_url: Optional URL for proxy chaining.
|
|
startup_timeout: Seconds to wait for the proxy to become available.
|
|
**kwargs: Additional arguments for the HTTPAdapter.
|
|
"""
|
|
self.certificate_config_file_path = certificate_config_file_path
|
|
self.gcloud_proxy_url = gcloud_proxy_url
|
|
|
|
super().__init__(**kwargs)
|
|
|
|
# Register a cleanup function to terminate the proxy process on exit.
|
|
# This is necessary to ensure the proxy process is terminated in case of
|
|
# unexpected termination, such as a crash or keyboard interrupt.
|
|
atexit.register(self.close)
|
|
|
|
self.proxy_process = None
|
|
self.proxy_host = 'localhost'
|
|
self.nonce_token = secrets.token_hex(16)
|
|
self.proxy_port = self._start_ecp_proxy_with_retries(
|
|
startup_timeout, max_retries=1
|
|
)
|
|
|
|
def _find_free_port(self) -> int:
|
|
"""Dynamically finds and returns an available TCP port."""
|
|
with socket.socket() as s:
|
|
s.bind(('', 0))
|
|
return s.getsockname()[1]
|
|
|
|
def _start_ecp_proxy_with_retries(
|
|
self, timeout: int, max_retries: int = 1
|
|
) -> int:
|
|
"""Attempts to start the ECP Proxy, retrying on failure.
|
|
|
|
This method orchestrates the proxy startup by finding an available port,
|
|
launching the proxy process, and waiting for it to become responsive.
|
|
If the proxy fails to start, it retries the process until either succeeds
|
|
or the maximum number of retries is reached.
|
|
|
|
Args:
|
|
timeout: The maximum time in seconds to wait for the proxy to start
|
|
max_retries: The maximum number of times to retry starting the proxy.
|
|
|
|
Returns:
|
|
The port number on which the proxy successfully started.
|
|
|
|
Raises:
|
|
ECPProxyError: If the proxy fails to start after all retry attempts.
|
|
"""
|
|
|
|
cert_config = context_aware.GetCertificateConfig(
|
|
self.certificate_config_file_path
|
|
)
|
|
ecp_http_proxy = cert_config.get('libs', {}).get('ecp_http_proxy')
|
|
if not ecp_http_proxy:
|
|
raise ECPProxyError(
|
|
'ECP HTTP proxy binary path is not specified in enterprise'
|
|
' certificate config file. Cannot use mTLS with ECP if the ECP HTTP'
|
|
' proxy binary does not exist. Please check the ECP configuration.'
|
|
' See `gcloud topic client-certificate` to learn more about ECP. \nIf'
|
|
' this error is unexpected either delete {} or generate a new'
|
|
' configuration with `$ gcloud auth enterprise-certificate-config'
|
|
' create --help` '.format(self.certificate_config_file_path)
|
|
)
|
|
|
|
for attempt in range(max_retries + 1):
|
|
proxy_port = self._find_free_port()
|
|
self._start_ecp_proxy(
|
|
ecp_http_proxy=ecp_http_proxy, proxy_port=proxy_port
|
|
)
|
|
|
|
try:
|
|
self._wait_for_proxy(proxy_port=proxy_port, timeout=timeout)
|
|
return proxy_port
|
|
except ECPProxyError as e:
|
|
if self.proxy_process and self.proxy_process.poll() is None:
|
|
self.proxy_process.terminate()
|
|
if attempt < max_retries:
|
|
log.warning(f'ECP proxy failed to start on port {proxy_port}: {e}')
|
|
continue
|
|
log.error(f'ECP proxy failed to start after {max_retries} retries: {e}')
|
|
raise
|
|
|
|
def _start_ecp_proxy(self, *, ecp_http_proxy: str, proxy_port: int) -> None:
|
|
"""Launches the local ECP proxy executable as a subprocess.
|
|
|
|
Constructs the necessary command-line arguments, including the ECP config
|
|
path, port, nonce token, and an optional upstream proxy URL. It then uses
|
|
subprocess.Popen to start the proxy process in the background.
|
|
|
|
Args:
|
|
ecp_http_proxy: The path to the ECP HTTP proxy binary.
|
|
proxy_port: The TCP port for the proxy to listen on.
|
|
|
|
Raises:
|
|
ECPProxyError: If the subprocess fails to start due to OSError or
|
|
ValueError.
|
|
"""
|
|
log.debug(f'Starting local ECP proxy server on port {proxy_port}')
|
|
|
|
args = [
|
|
'-enterprise_certificate_file_path',
|
|
self.certificate_config_file_path,
|
|
'-port',
|
|
str(proxy_port),
|
|
'-nonce_token',
|
|
self.nonce_token,
|
|
]
|
|
|
|
if self.gcloud_proxy_url:
|
|
args.extend(
|
|
['-gcloud_configured_upstream_proxy_url', self.gcloud_proxy_url]
|
|
)
|
|
|
|
proxy_args = execution_utils.ArgsForExecutableTool(ecp_http_proxy, *args)
|
|
try:
|
|
self.proxy_process = subprocess.Popen(
|
|
proxy_args,
|
|
stdout=None,
|
|
stderr=None,
|
|
)
|
|
except (OSError, ValueError) as e:
|
|
log.error(f'Failed to start ECP proxy executable: {e}')
|
|
raise ECPProxyError(
|
|
'Failed to start ECP proxy process', original_exception=e
|
|
) from e
|
|
|
|
def _wait_for_proxy(self, *, proxy_port: int, timeout: int) -> None:
|
|
"""Waits for the proxy to become available and verifies its identity.
|
|
|
|
This method first waits for the proxy's TCP port to accept connections.
|
|
Once the port is open, it sends a request to the `/readyz` endpoint to
|
|
confirm that the proxy is fully operational and to verify a security nonce,
|
|
ensuring that gcloud is communicating with the correct proxy instance.
|
|
|
|
Args:
|
|
proxy_port: The port where the proxy is expected to be listening.
|
|
timeout: The maximum time in seconds to wait for the proxy.
|
|
|
|
Raises:
|
|
ECPProxyError: If the proxy process terminates unexpectedly, fails to
|
|
respond within the timeout, or returns an invalid nonce.
|
|
"""
|
|
|
|
log.debug(f'Waiting for the proxy to be ready on port {proxy_port}...')
|
|
start_time = time.monotonic()
|
|
|
|
# Check to see if the port is open.
|
|
while time.monotonic() - start_time < timeout:
|
|
# This is a fast-fail. If the process is dead, stop trying.
|
|
if self.proxy_process.poll() is not None:
|
|
raise ECPProxyError(
|
|
'Proxy process terminated unexpectedly with code '
|
|
f'{self.proxy_process.returncode} while waiting for it to start.'
|
|
)
|
|
|
|
try:
|
|
with socket.create_connection(
|
|
(self.proxy_host, proxy_port), timeout=0.1
|
|
):
|
|
# Proxy is ready, break.
|
|
break
|
|
except OSError:
|
|
time.sleep(0.1)
|
|
continue
|
|
else:
|
|
# The 'while' loop finished without a 'break'.
|
|
# This means we timed out while trying to connect to the socket.
|
|
self.close() # Clean up the zombie process.
|
|
raise ECPProxyError(
|
|
f'ECP Proxy on {self.proxy_host}:{proxy_port} did not become ready in'
|
|
f' {timeout} seconds.'
|
|
)
|
|
|
|
# If we're here, the socket is open.
|
|
# Now we do a one-time verification using the /readyz endpoint.
|
|
try:
|
|
readyz_url = f'http://{self.proxy_host}:{proxy_port}/readyz'
|
|
response = requests.get(readyz_url, timeout=1)
|
|
if response.status_code != 200:
|
|
raise ECPProxyError(
|
|
f'Proxy /readyz endpoint returned status {response.status_code}.'
|
|
)
|
|
|
|
server_nonce = response.text
|
|
if server_nonce != self.nonce_token:
|
|
raise ECPProxyError('Nonce mismatch from proxy /readyz endpoint.')
|
|
|
|
log.debug('Proxy is ready and nonce verified.')
|
|
except requests.exceptions.RequestException as e:
|
|
# Failed during the HTTP call itself (e.g., connection reset)
|
|
raise ECPProxyError(
|
|
'Failed to verify proxy readiness via /readyz endpoint.',
|
|
original_exception=e,
|
|
) from e
|
|
|
|
def send(self, request, **kwargs):
|
|
"""Intercepts an outgoing request and reroutes it through the local ECP proxy.
|
|
|
|
This method modifies the request's URL to point to the local proxy and
|
|
adds a custom header (`x-goog-ecpproxy-target-host`) to inform the proxy
|
|
of the original destination. It then sends the modified request and passes
|
|
the response to `_handle_proxy_response` for inspection.
|
|
|
|
Args:
|
|
request: The `requests.PreparedRequest` object to send.
|
|
**kwargs: Additional arguments passed to the underlying `send` method.
|
|
|
|
Returns:
|
|
The `requests.Response` object from the proxy.
|
|
|
|
Raises:
|
|
ECPProxyError: If the proxy returns an error or fails to send the request.
|
|
"""
|
|
# Rewrite the request to target the proxy
|
|
original_url = urllib.parse.urlsplit(request.url)
|
|
request.headers['x-goog-ecpproxy-target-host'] = original_url.hostname
|
|
|
|
# The new URL for the request is the proxy address + original path/query
|
|
# urlunsplit takes (scheme, netloc, path, query, fragment).
|
|
# The empty strings for scheme, netloc, and fragment ensure that only
|
|
# the path and query components from the original URL are used to form
|
|
# the part of the URL that comes after the proxy address.
|
|
proxy_path = urllib.parse.urlunsplit(
|
|
('', '', original_url.path, original_url.query, '')
|
|
)
|
|
request.url = f'http://{self.proxy_host}:{self.proxy_port}{proxy_path}'
|
|
|
|
# We are connecting to the proxy over HTTP, not HTTPS.
|
|
kwargs['verify'] = False
|
|
|
|
log.debug(
|
|
f'Redirecting request for {original_url.geturl()} to proxy at'
|
|
f' {request.url}'
|
|
)
|
|
try:
|
|
response = super().send(request, **kwargs)
|
|
except requests.exceptions.RequestException as e:
|
|
raise ECPProxyError(
|
|
'Failed to send request to proxy', original_exception=e
|
|
) from e
|
|
|
|
return self._handle_proxy_response(response)
|
|
|
|
def _handle_proxy_response(self, response):
|
|
"""Inspects the proxy's response for custom error headers.
|
|
|
|
If the `x-goog-ecpproxy-error` header is present in the response, this
|
|
method assumes the proxy encountered an internal error. It attempts to parse
|
|
a detailed error message from the JSON response body and raises an
|
|
`ECPProxyError` with the relevant information.
|
|
|
|
Args:
|
|
response: The `requests.Response` object received from the proxy.
|
|
|
|
Returns:
|
|
The original `requests.Response` object if no error header is found.
|
|
|
|
Raises:
|
|
ECPProxyError: If the `x-goog-ecpproxy-error` header is present.
|
|
"""
|
|
proxy_error_header = response.headers.get('x-goog-ecpproxy-error')
|
|
if proxy_error_header:
|
|
log.error('ECP Proxy returned an error')
|
|
try:
|
|
# Attempt to parse a more detailed error from the JSON body.
|
|
error_details = response.json().get('message', 'No message in body.')
|
|
message = f'ECP Proxy indicated an internal error: {error_details}'
|
|
except json.JSONDecodeError:
|
|
message = (
|
|
'ECP Proxy indicated an internal error. Response body:'
|
|
f' {response.text}'
|
|
)
|
|
raise ECPProxyError(message)
|
|
|
|
return response
|
|
|
|
def close(self):
|
|
"""Terminates the background ECP proxy process to clean up resources.
|
|
|
|
This method ensures that the local proxy subprocess is properly shut down
|
|
when the session is closed. It first attempts a graceful termination and
|
|
waits briefly, then forcefully kills the process if it does not exit in
|
|
time. Finally, it calls the parent class's `close` method to complete
|
|
the cleanup.
|
|
"""
|
|
log.debug('Closing ECP Proxy Adapter and terminating proxy process...')
|
|
if self.proxy_process and self.proxy_process.poll() is None:
|
|
self.proxy_process.terminate()
|
|
try:
|
|
# Wait a moment for graceful shutdown before killing.
|
|
self.proxy_process.wait(timeout=0.5)
|
|
except subprocess.TimeoutExpired:
|
|
log.warning('Proxy process did not terminate gracefully, killing it.')
|
|
self.proxy_process.kill()
|
|
super().close()
|
|
|
|
|
|
def _CreateMutualTlsOffloadAdapter(
|
|
ca_config: context_aware._EnterpriseCertConfigImpl,
|
|
) -> requests.adapters.BaseAdapter:
|
|
"""Creates a requests adapter for mTLS offloading via ECP.
|
|
|
|
This function decides which adapter to use based on the provided
|
|
configuration:
|
|
- If `ca_config.use_local_proxy` is True, it returns a
|
|
`_LocalECPProxyAdapter`, which routes traffic through a local ECP proxy
|
|
subprocess.
|
|
- Otherwise, it returns a `_MutualTlsOffloadAdapter` from the google-auth
|
|
library, which uses the ECP binary for TLS offloading without a local proxy.
|
|
|
|
Args:
|
|
ca_config: The enterprise certificate configuration object.
|
|
|
|
Returns:
|
|
An instance of a requests adapter for mTLS offloading.
|
|
|
|
Raises:
|
|
ValueError: If the certificate configuration file path is not provided.
|
|
"""
|
|
if not ca_config or not ca_config.certificate_config_file_path:
|
|
raise ValueError('Certificate config file path must be provided.')
|
|
|
|
if ca_config.use_local_proxy:
|
|
return _LocalECPProxyAdapter(
|
|
certificate_config_file_path=ca_config.certificate_config_file_path,
|
|
)
|
|
else:
|
|
return _MutualTlsOffloadAdapter(ca_config.certificate_config_file_path)
|
|
|
|
|
|
def _LinuxNonbundledPythonAndGooglerCheck():
|
|
"""Warn users if running non-bundled Python on Linux and is a Googler.
|
|
|
|
Checks if the current OS is Linux, running Python that is not bundled and if
|
|
the user is a Googler. If all conditions are true, a warning message will be
|
|
emitted, along with returning true to bypass the mTLS code path.
|
|
|
|
Returns:
|
|
True if the conditions are met, False otherwise.
|
|
"""
|
|
is_linux = (
|
|
platforms.OperatingSystem.Current() == platforms.OperatingSystem.LINUX)
|
|
is_bundled_python = sys.executable and 'bundled' in sys.executable
|
|
is_internal_user = properties.IsInternalUserCheck()
|
|
if is_linux and not is_bundled_python and is_internal_user:
|
|
log.warning(_GOOGLER_BUNDLED_PYTHON_WARNING)
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
|
|
def Session(
|
|
timeout=None,
|
|
ca_certs=None,
|
|
disable_ssl_certificate_validation=False,
|
|
session=None,
|
|
client_certificate=None,
|
|
client_key=None):
|
|
"""Returns a requests.Session subclass.
|
|
|
|
Args:
|
|
timeout: float, Request timeout, in seconds.
|
|
ca_certs: str, absolute filename of a ca_certs file
|
|
disable_ssl_certificate_validation: bool, If true, disable ssl certificate
|
|
validation.
|
|
session: requests.Session instance. Otherwise, a new requests.Session will
|
|
be initialized.
|
|
client_certificate: str, absolute filename of a client_certificate file
|
|
client_key: str, absolute filename of a client_key file
|
|
|
|
Returns: A requests.Session subclass.
|
|
"""
|
|
session = session or requests.Session()
|
|
proxy_info = GetProxyInfo()
|
|
|
|
orig_request_method = session.request
|
|
def WrappedRequest(*args, **kwargs):
|
|
if 'timeout' not in kwargs:
|
|
kwargs['timeout'] = timeout
|
|
|
|
# Work around a proxy bug in Python's standard library on Windows.
|
|
if _HasBpo42627() and 'proxies' not in kwargs:
|
|
kwargs['proxies'] = _AdjustProxiesKwargForBpo42627(
|
|
proxy_info, urllib_request.getproxies_environment(),
|
|
orig_request_method, *args, **kwargs)
|
|
|
|
return orig_request_method(*args, **kwargs)
|
|
session.request = WrappedRequest
|
|
|
|
if proxy_info:
|
|
session.trust_env = False
|
|
session.proxies = {
|
|
'http': proxy_info,
|
|
'https': proxy_info,
|
|
}
|
|
elif _HasInvalidHttpsProxyEnvVarScheme():
|
|
# Requests (and by extension gcloud) currently only supports connecting to
|
|
# proxy servers via HTTP. Until that changes, provide a more informative
|
|
# message when attempting to connect via HTTPS (usually due to a
|
|
# misconfigured HTTPS_PROXY env var), since this now results in a (rather
|
|
# opaque) error as of newer versions of urllib3 (b/228647259#comment30).
|
|
global _invalid_https_proxy_env_var_warning_shown
|
|
if not _invalid_https_proxy_env_var_warning_shown:
|
|
# Just do this once per command invocation to avoid spamming the warning
|
|
# multiple times (we initialize multiple sessions per command).
|
|
_invalid_https_proxy_env_var_warning_shown = True
|
|
log.warning(_INVALID_HTTPS_PROXY_ENV_VAR_WARNING)
|
|
|
|
client_side_certificate = None
|
|
if client_certificate is not None and client_key is not None and ca_certs is not None:
|
|
log.debug(
|
|
'Using provided server certificate %s, client certificate %s, client certificate key %s',
|
|
ca_certs, client_certificate, client_key)
|
|
client_side_certificate = ClientSideCertificate(
|
|
client_certificate, client_key)
|
|
adapter = HTTPAdapter(client_side_certificate)
|
|
else:
|
|
ca_config = context_aware.Config()
|
|
if ca_config:
|
|
_LinuxNonbundledPythonAndGooglerCheck()
|
|
if ca_config.config_type == context_aware.ConfigType.ENTERPRISE_CERTIFICATE:
|
|
adapter = _CreateMutualTlsOffloadAdapter(ca_config)
|
|
elif ca_config.config_type == context_aware.ConfigType.ON_DISK_CERTIFICATE:
|
|
log.debug('Using client certificate %s',
|
|
ca_config.encrypted_client_cert_path)
|
|
client_side_certificate = ClientSideCertificate(
|
|
ca_config.encrypted_client_cert_path,
|
|
ca_config.encrypted_client_cert_path,
|
|
ca_config.encrypted_client_cert_password)
|
|
adapter = HTTPAdapter(client_side_certificate)
|
|
else:
|
|
adapter = HTTPAdapter(None)
|
|
else:
|
|
adapter = HTTPAdapter(None)
|
|
|
|
if disable_ssl_certificate_validation:
|
|
session.verify = False
|
|
elif ca_certs:
|
|
session.verify = ca_certs
|
|
|
|
session.mount('https://', adapter)
|
|
return session
|
|
|
|
|
|
def _CreateRawSession(timeout='unset', ca_certs=None, session=None,
|
|
client_certificate=None, client_key=None):
|
|
"""Create a requests.Session matching the appropriate gcloud properties."""
|
|
# Compared with setting the default timeout in the function signature (i.e.
|
|
# timeout=300), this lets you test with short default timeouts by mocking
|
|
# GetDefaultTimeout.
|
|
if timeout != 'unset':
|
|
effective_timeout = timeout
|
|
else:
|
|
effective_timeout = transport.GetDefaultTimeout()
|
|
|
|
no_validate = properties.VALUES.auth.disable_ssl_validation.GetBool() or False
|
|
ca_certs_property = properties.VALUES.core.custom_ca_certs_file.Get()
|
|
# Believe an explicitly-set ca_certs property over anything we added.
|
|
if ca_certs_property:
|
|
ca_certs = ca_certs_property
|
|
if no_validate:
|
|
ca_certs = None
|
|
return Session(timeout=effective_timeout,
|
|
ca_certs=ca_certs,
|
|
disable_ssl_certificate_validation=no_validate,
|
|
session=session,
|
|
client_certificate=client_certificate,
|
|
client_key=client_key)
|
|
|
|
|
|
def _GetURIFromRequestArgs(url, params):
|
|
"""Gets the complete URI by merging url and params from the request args."""
|
|
url_parts = urllib.parse.urlsplit(url)
|
|
query_params = urllib.parse.parse_qs(url_parts.query, keep_blank_values=True)
|
|
for param, value in six.iteritems(params or {}):
|
|
query_params[param] = value
|
|
# Need to do this to convert a SplitResult into a list so it can be modified.
|
|
url_parts = list(url_parts)
|
|
# pylint:disable=redundant-keyword-arg, this is valid syntax for this lib
|
|
url_parts[3] = urllib.parse.urlencode(query_params, doseq=True)
|
|
|
|
# pylint:disable=too-many-function-args, This is just bogus.
|
|
return urllib.parse.urlunsplit(url_parts)
|
|
|
|
|
|
class Request(transport.Request):
|
|
"""Encapsulates parameters for making a general HTTP request.
|
|
|
|
This implementation does additional manipulation to ensure that the request
|
|
parameters are specified in the same way as they were specified by the
|
|
caller. That is, if the user calls:
|
|
request('URI', 'GET', None, {'header': '1'})
|
|
|
|
After modifying the request, we will call request using positional
|
|
parameters, instead of transforming the request into:
|
|
request('URI', method='GET', body=None, headers={'header': '1'})
|
|
"""
|
|
|
|
@classmethod
|
|
def FromRequestArgs(cls, *args, **kwargs):
|
|
return cls(*args, **kwargs)
|
|
|
|
def __init__(self, method, url, params=None, data=None, headers=None,
|
|
**kwargs):
|
|
self._kwargs = kwargs
|
|
uri = _GetURIFromRequestArgs(url, params)
|
|
super(Request, self).__init__(uri, method, headers or {}, data)
|
|
|
|
def ToRequestArgs(self):
|
|
args = [self.method, self.uri]
|
|
kwargs = dict(self._kwargs)
|
|
kwargs['headers'] = self.headers
|
|
if self.body:
|
|
kwargs['data'] = self.body
|
|
return args, kwargs
|
|
|
|
|
|
class Response(transport.Response):
|
|
"""Encapsulates responses from making a general HTTP request."""
|
|
|
|
@classmethod
|
|
def FromResponse(cls, response):
|
|
return cls(response.status_code, response.headers, response.content)
|
|
|
|
|
|
class RequestWrapper(transport.RequestWrapper):
|
|
"""Class for wrapping request.Session requests."""
|
|
|
|
request_class = Request
|
|
response_class = Response
|
|
|
|
def DecodeResponse(self, response, response_encoding):
|
|
"""Returns the response without decoding."""
|
|
del response_encoding # unused
|
|
# The response decoding is handled by the _ApitoolsRequests.request method.
|
|
return response
|
|
|
|
|
|
def GoogleAuthRequest():
|
|
"""Returns a gcloud's requests session to refresh google-auth credentials."""
|
|
session = GetSession()
|
|
|
|
# Ensure requests to the GCE metadata server are not proxied. We respect the
|
|
# same env vars for overriding the metadata server hostname/IP as google-auth:
|
|
# https://googleapis.dev/python/google-auth/latest/reference/google.auth.environment_vars.html
|
|
metadata_root = encoding.GetEncodedValue(
|
|
os.environ,
|
|
'GCE_METADATA_HOST',
|
|
encoding.GetEncodedValue(
|
|
os.environ,
|
|
'GCE_METADATA_ROOT',
|
|
'metadata.google.internal'))
|
|
metadata_ip_root = encoding.GetEncodedValue(
|
|
os.environ, 'GCE_METADATA_IP', '169.254.169.254')
|
|
# Ideally we would set 'no_proxy' in the proxies dict, but requests doesn't
|
|
# actually handle this correctly: https://github.com/psf/requests/issues/5000.
|
|
# Instead we specify the hostnames/addresses not to proxy as individual keys
|
|
# with empty values, which works because requests also consider these keys
|
|
# when evaluating whether to proxy a URL.
|
|
session.proxies.update({
|
|
f'http://{metadata_root}': '',
|
|
f'http://{metadata_ip_root}': '',
|
|
})
|
|
|
|
return google_auth_requests.Request(session=session)
|
|
|
|
|
|
class _GoogleAuthApitoolsCredentials():
|
|
|
|
def __init__(self, credentials):
|
|
self.credentials = credentials
|
|
|
|
def refresh(self, http_client): # pylint: disable=invalid-name
|
|
del http_client # unused
|
|
auth_request = GoogleAuthRequest()
|
|
self.credentials.refresh(auth_request)
|
|
|
|
|
|
def GetApitoolsRequests(session, response_handler=None, response_encoding=None):
|
|
"""Returns an authenticated httplib2.Http-like object for use by apitools."""
|
|
http_client = _ApitoolsRequests(session, response_handler, response_encoding)
|
|
# apitools needs this attribute to do credential refreshes during batch API
|
|
# requests.
|
|
if hasattr(session, '_googlecloudsdk_credentials'):
|
|
creds = _GoogleAuthApitoolsCredentials(session._googlecloudsdk_credentials) # pylint: disable=protected-access
|
|
|
|
orig_request_method = http_client.request
|
|
|
|
# The closure that will replace 'httplib2.Http.request'.
|
|
def HttpRequest(*args, **kwargs):
|
|
return orig_request_method(*args, **kwargs)
|
|
|
|
http_client.request = HttpRequest
|
|
setattr(http_client.request, 'credentials', creds)
|
|
|
|
return http_client
|
|
|
|
|
|
class ResponseHandler(six.with_metaclass(abc.ABCMeta)):
|
|
"""Handler to process the Http Response.
|
|
|
|
Attributes:
|
|
use_stream: bool, if True, the response body gets returned as a stream
|
|
of data instead of returning the entire body at once.
|
|
"""
|
|
|
|
def __init__(self, use_stream):
|
|
"""Initializes ResponseHandler.
|
|
|
|
Args:
|
|
use_stream: bool, if True, the response body gets returned as a stream of
|
|
data instead of returning the entire body at once.
|
|
"""
|
|
self.use_stream = use_stream
|
|
|
|
@abc.abstractmethod
|
|
def handle(self, response_stream):
|
|
"""Handles the http response."""
|
|
|
|
|
|
class _ApitoolsRequests():
|
|
"""A httplib2.Http-like object for use by apitools."""
|
|
|
|
def __init__(self, session, response_handler=None, response_encoding=None):
|
|
self.session = session
|
|
# Mocks the dictionary of connection instances that apitools iterates over
|
|
# to modify the underlying connection.
|
|
self.connections = {}
|
|
if response_handler:
|
|
if not isinstance(response_handler, ResponseHandler):
|
|
raise ValueError('response_handler should be of type ResponseHandler.')
|
|
self._response_handler = response_handler
|
|
self._response_encoding = response_encoding
|
|
|
|
def ResponseHook(self, response, *args, **kwargs):
|
|
"""Response hook to be used if response_handler has been set."""
|
|
del args, kwargs # Unused.
|
|
if response.status_code not in (httplib.OK, httplib.PARTIAL_CONTENT):
|
|
log.debug('Skipping response_handler as response is invalid.')
|
|
return
|
|
|
|
if (self._response_handler.use_stream and
|
|
properties.VALUES.core.log_http.GetBool() and
|
|
properties.VALUES.core.log_http_streaming_body.GetBool()):
|
|
# The response_handler uses streaming body, but since a request was
|
|
# made to log the response body, we should retain a copy of the response
|
|
# data. A call to response.content would read the entire data in-memory.
|
|
stream = io.BytesIO(response.content)
|
|
else:
|
|
stream = response.raw
|
|
self._response_handler.handle(stream)
|
|
|
|
def request(
|
|
self,
|
|
uri,
|
|
method='GET',
|
|
body=None,
|
|
headers=None,
|
|
redirections=0,
|
|
connection_type=None,
|
|
): # pylint: disable=invalid-name
|
|
"""Makes an HTTP request using httplib2 semantics."""
|
|
del connection_type # Unused
|
|
|
|
if redirections > 0:
|
|
self.session.max_redirects = redirections
|
|
|
|
hooks = {}
|
|
if self._response_handler is not None:
|
|
hooks['response'] = self.ResponseHook
|
|
use_stream = self._response_handler.use_stream
|
|
else:
|
|
use_stream = False
|
|
|
|
response = self.session.request(
|
|
method, uri, data=body, headers=headers, stream=use_stream, hooks=hooks)
|
|
headers = dict(response.headers)
|
|
headers['status'] = response.status_code
|
|
|
|
if use_stream:
|
|
# If use_stream is True, we assume that the data will be read from the
|
|
# response_handler
|
|
content = b''
|
|
elif self._response_encoding is not None:
|
|
# We update response.encoding before calling response.text because
|
|
# response.text property will try to make an educated guess about the
|
|
# encoding based on the response header, which might be different from
|
|
# the self._response_encoding set by the caller.
|
|
response.encoding = self._response_encoding
|
|
content = response.text
|
|
else:
|
|
content = response.content
|
|
|
|
return httplib2.Response(headers), content
|
|
|
|
|
|
def _HasInvalidHttpsProxyEnvVarScheme():
|
|
"""Returns whether the HTTPS proxy env var is using an HTTPS scheme."""
|
|
# We call urllib.getproxies_environment instead of checking os.environ
|
|
# ourselves to ensure we match the semantics of what the requests library ends
|
|
# up doing.
|
|
env_proxies = urllib_request.getproxies_environment()
|
|
return env_proxies.get('https', '').startswith('https://')
|
|
|
|
|
|
def _HasBpo42627():
|
|
"""Returns whether Python is affected by https://bugs.python.org/issue42627.
|
|
|
|
Due to a bug in Python's standard library, urllib.request misparses the
|
|
Windows registry proxy settings and assumes that HTTPS URLs should use an
|
|
HTTPS proxy, when in fact they should use an HTTP proxy.
|
|
|
|
This bug affects PY<3.9, as well as lower patch versions of 3.9, 3.10, and
|
|
3.11.
|
|
|
|
Returns:
|
|
True if proxies read from the Windows registry are being parsed incorrectly.
|
|
"""
|
|
return (
|
|
platforms.OperatingSystem.Current() == platforms.OperatingSystem.WINDOWS
|
|
and hasattr(urllib_request, 'getproxies_registry')
|
|
and urllib_request.getproxies_registry().get('https', '').startswith(
|
|
'https://')
|
|
)
|
|
|
|
|
|
def _AdjustProxiesKwargForBpo42627(
|
|
gcloud_proxy_info, environment_proxies,
|
|
orig_request_method, *args, **kwargs):
|
|
"""Returns proxies to workaround https://bugs.python.org/issue42627 if needed.
|
|
|
|
Args:
|
|
gcloud_proxy_info: str, Proxy info from gcloud properties.
|
|
environment_proxies: dict, Proxy config from http/https_proxy env vars.
|
|
orig_request_method: function, The original requests.Session.request method.
|
|
*args: Positional arguments to the original request method.
|
|
**kwargs: Keyword arguments to the original request method.
|
|
Returns:
|
|
Optional[dict], Adjusted proxies to pass to the request method, or None if
|
|
no adjustment is necessary.
|
|
"""
|
|
# Proxy precedence:
|
|
# gcloud properties > http/https/no_proxy env vars > registry settings
|
|
# So if proxy settings come from either of the first two, then there's no need
|
|
# to adjust anything.
|
|
if gcloud_proxy_info or environment_proxies:
|
|
return None
|
|
|
|
# We want to correct proxies incorrectly parsed from the registry by sending a
|
|
# tweaked 'proxies' kwarg to the requests.Session.request method. However,
|
|
# proxies passed in this manner apply unconditionally, and we still wish to
|
|
# respect the "ProxyOverride" settings from the registry. So we extract the
|
|
# URL passed to the method, and only pass a corrected HTTPS proxy if requests
|
|
# would end up using the proxy for that URL when taking "ProxyOverride"
|
|
# settings into account.
|
|
url = inspect.getcallargs(orig_request_method, *args, **kwargs)['url'] # pylint: disable=deprecated-method, for PY2 compatibility
|
|
proxies = requests.utils.get_environ_proxies(url) # Respects ProxyOverride.
|
|
https_proxy = proxies.get('https')
|
|
if not https_proxy:
|
|
return None
|
|
|
|
if not https_proxy.startswith('https://'):
|
|
# This should theoretically never happen, since
|
|
# requests.utils.get_environ_proxies should have returned proxies from the
|
|
# registry if we got here, and those will have been bugged. But just in case
|
|
# some implementation detail changes, don't try to adjust anything.
|
|
return None
|
|
|
|
return {
|
|
'https': https_proxy.replace('https://', 'http://', 1)
|
|
}
|