feat: Add new gcloud commands, API clients, and third-party libraries across various services.

This commit is contained in:
2026-01-01 20:26:35 +01:00
parent 5e23cbece0
commit a19e592eb7
25221 changed files with 8324611 additions and 0 deletions

View File

@@ -0,0 +1,367 @@
# -*- coding: utf-8 -*- #
# Copyright 2025 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Utility for validating Firestore Mongo connection strings."""
import dataclasses
import re
import socket
import ssl
import time
from typing import List
from googlecloudsdk.core import log
from googlecloudsdk.core.console import console_attr
@dataclasses.dataclass(frozen=True)
class ValidationResults:
"""Container class for results of validating a connection string."""
headers: List[str]
info: List[str]
warnings: List[str]
errors: List[str]
footers: List[str]
def __str__(self):
return '\n'.join(
self.headers + self.info + self.warnings + self.errors + self.footers
)
def ValidateConnectionString(
connection_string,
db_uid=None,
db_location_id=None,
database_id=None,
):
"""Validate the specified connection_string for the specified database."""
headers = [
'-' * 80,
f'Evaluating connection string: {connection_string}',
'-' * 80,
]
info = []
warnings = []
errors = []
footers = ['-' * 80]
user = None
password = None
# Helper method for checking k=v params
def CheckParam(param_name, expected_value, error_description=''):
if param_name not in extra_params:
errors.append(
f'{error_description}The connection string must specify'
f' {param_name}={expected_value}.'
)
else:
actual_value = extra_params[param_name]
del extra_params[param_name]
if actual_value != expected_value:
errors.append(
f'{error_description}The parameter {param_name} is set to '
f'{actual_value}. The connection string must specify '
f'{param_name}={expected_value}.'
)
else:
info.append(f'{param_name}={expected_value}.')
# Scan the connection string left-to-right and emit recommendations.
while True:
# Check that the connection string starts with the appropriate prefix
if not connection_string.startswith('mongodb://'):
errors.append('The connection string must start with mongodb://')
break
# Strip off mongodb:// and continue evaluation
connection_string = connection_string[len('mongodb://') :]
# Check for the presense of a user/password (optional)
match = re.match(r'^([^:]*):([^@]*)@', connection_string)
if match:
user = match.group(1)
password = '*' * len(match.group(2))
info.append(
f'The connection string specifies user: {user} '
f'and password: {password}'
)
# Strip off the user+password and continue evaluation
connection_string = connection_string[len(user) + len(password) + 2 :]
# Check that the database address begins with a valid UUID
match = re.match(
r'^([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.',
connection_string,
)
if not match:
errors.append((
'The database address must start with a valid UUID. '
f'For database {database_id}, use the value {db_uid}'
))
errors.append(
'NOTE: for password based authentication, the connection string '
'can also start with mongodb://username:password@UUID.'
)
break
if match.group(1) != db_uid:
errors.append(
f'the UUID {match.group(1)} in the connection string does not '
f'match the UUID {db_uid} for the current database {database_id}.'
)
else:
info.append(f'The UUID {db_uid} is correct.')
# Strip off the 36 characters of the UUID + . and continue evaluation
connection_string = connection_string[37:]
# Check that the UUID is followed by a valid database location
match = re.match(
r'^([^\.]+)\.',
connection_string,
)
if not match:
errors.append(
'The database address must have the form: '
'UUID.location.firestore.goog:443'
)
break
if match.group(1) != db_location_id:
errors.append(
f'The location {match.group(1)} in the connection string does '
f'not match the location {db_location_id} for the '
f'current database {database_id}.'
)
else:
info.append(f'The location {db_location_id} is correct.')
# Strip off the location + . and continue evaluation
connection_string = connection_string[len(match.group(1)) + 1 :]
# Check that the location is followed by the valid domain and port number
if not connection_string.startswith('firestore.goog:443/'):
errors.append(
'The database address must end with firestore.goog:443 as '
'the domain name and port.'
)
break
# Strip off the rest of the address and continue evaluation
connection_string = connection_string[len('firestore.goog:443/') :]
# Check that the string contains a valid database name.
match = re.match(r'^([^\?]*)\?', connection_string)
if not match:
errors.append(
'The connection string must specify the database id. '
f'For the current database {database_id} it should have the form '
f'UUID.location.firestore.goog:443/{database_id}?'
)
break
if match.group(1) != database_id:
if match.group(1):
errors.append(
f'The database name {match.group(1)} in the connection'
f' string does not match the current database {database_id}.'
)
else:
errors.append(
'The database name in the connection string is empty. '
'It is recommended to explicitly specify the database name '
f'{database_id}, e.g. firestore.goog:443/{database_id}?'
)
else:
info.append(f'The database name {database_id} is correct.')
# Stip off the rest of database id + '?' and continue evaluation
connection_string = connection_string[len(match.group(1)) + 1 :]
# Validate additional parameters, which should come in as k=v pairs.
extra_params = {}
entries = connection_string.split('&')
for entry in entries:
if not entry:
continue
parts = entry.split('=')
if len(parts) != 2:
errors.append(
f'The parameter {entry} appears malformed. Expected'
' something in the form key=value.'
)
else:
extra_params[parts[0]] = parts[1]
# Check for always-required params
CheckParam('loadBalanced', 'true')
CheckParam('tls', 'true')
CheckParam('retryWrites', 'false')
# Check for params that require extra validation
if 'authMechanism' in extra_params:
auth_mechanism = extra_params['authMechanism']
del extra_params['authMechanism']
if auth_mechanism == 'PLAIN':
CheckParam(
'authSource',
'$external',
error_description='Using PLAIN authentication. ',
)
if not user:
errors.append(
'The username and an access token should be specified in '
'the connection string when PLAIN authentication is enabled.'
)
else:
info.append(
'username and access token specified for PLAIN authentication.'
)
elif auth_mechanism == 'SCRAM-SHA-256':
if not user:
errors.append(
'The username and password should be specified in the '
'connection string when SCRAM-SHA-256 is enabled.'
)
else:
info.append('username and password specified for SCRAM-SHA-256.')
elif auth_mechanism == 'MONGODB-OIDC':
if user:
errors.append(
'The username should not be specified when using the '
'MONGODB-OIDC authentication mechanism.'
)
CheckParam(
'authMechanismProperties',
'ENVIRONMENT:gcp,TOKEN_RESOURCE:FIRESTORE',
error_description='Using MONGODB-OIDC authentication. ',
)
else:
errors.append(f'Unsupported authentication mechanism {auth_mechanism}.')
else:
if user:
errors.append(
f'Since the connection string specified user: {user} and'
f' password: {password}, the connection must also be configured'
' with an appropriate authentication mechanism, e.g.'
' authMechanism=SCRAM-SHA-256'
)
else:
errors.append(
'No authMechanism specified. The connection string must '
'specify one of the supported authentication mechanisms.'
)
# Check for any unconsumed parameters
for k, v in extra_params.items():
# Emit these was warnings. We don't know how they'll affect the client.
warnings.append(f'Unknown parameter {k}={v}.')
break
if not errors:
footers.append('Did not detect any errors in this connection string.')
else:
footers.append(
"TIP: You can use 'gcloud firestore databases connection-string "
f"--database={database_id}' to construct valid connection strings."
)
return ValidationResults(
headers=headers,
info=info,
warnings=warnings,
errors=errors,
footers=footers,
)
def PrettyPrintValidationResults(validation_results: ValidationResults):
"""Renders the connection string validation results to the console."""
con = console_attr.GetConsoleAttr()
for header in validation_results.headers:
log.status.Print(header)
for info in validation_results.info:
log.status.Print(f"{con.Colorize('INFO:', 'green')} {info}")
for warning in validation_results.warnings:
log.status.Print(f"{con.Colorize('WARNING:', 'yellow')} {warning}")
for error in validation_results.errors:
log.status.Print(f"{con.Colorize('ERROR:', 'red')} {error}")
for footer in validation_results.footers:
log.status.Print(footer)
# Byte encoding of the Bson "hello" command document:
# {"hello": 1, "helloOk": True, "loadBalanced": True}
_HELLO_HEX = (
'340000000100000000000000DD07000000000000001F0000001068656C6C6'
'F0001000000086C6F616442616C616E636564000100'
)
# Byte encoding of the Bson "ping" command document: {"ping": 1}
_PING_HEX = (
'240000000000000000000000dd07000000000000000f0000001070696e67000100000000'
)
_MAX_CONNECTION_WAIT_TIME = 20.0
_MAX_PING_WAIT_TIME = 5.0
def Ping(ssock):
"""Sends a Mongo ping message via specified socket."""
ping_complete = False
ping_start = time.perf_counter()
ssock.sendall(bytes.fromhex(_PING_HEX))
while True:
data = ssock.recv(1024)
ping_time = time.perf_counter() - ping_start
if not data:
break
if data.find(b'ok') != -1:
ping_complete = True
break
# Give up after enough time has passed.
if ping_time > _MAX_PING_WAIT_TIME:
break
if ping_complete:
print(f'{ping_time:.3f}s ', end='')
else:
print('N/A ', end='')
return ping_time if ping_complete else None
def Hello(ssock):
"""Sends a Mongo hello message via specified socket."""
handshake_complete = False
connection_start = time.perf_counter()
ssock.sendall(bytes.fromhex(_HELLO_HEX))
while True:
data = ssock.recv(1024)
connect_time = time.perf_counter() - connection_start
if not data:
break
if data.find(b'isWritablePrimary') != -1:
print(f'Connection established in {connect_time:.3f} seconds')
handshake_complete = True
break
# Give up after enough time has passed.
if connect_time > _MAX_CONNECTION_WAIT_TIME:
break
return connect_time if handshake_complete else None
def ConnectAndPing(hostname, num_pings):
"""Opens an SSL connection and sends timed Mongo commands to the server."""
context = ssl.create_default_context()
ping_times = []
with socket.create_connection((hostname, 443)) as sock:
with context.wrap_socket(sock, server_hostname=hostname) as ssock:
connect_time = Hello(ssock)
print(f'Sending {num_pings} pings ...: ', end='')
for _ in range(num_pings):
ping_times.append(Ping(ssock))
print()
return (connect_time, ping_times)

View File

@@ -0,0 +1,103 @@
# -*- coding: utf-8 -*- #
# Copyright 2019 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Utilities for database creation."""
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from apitools.base.py import exceptions as apitools_exceptions
from googlecloudsdk.api_lib.app import appengine_api_client
from googlecloudsdk.calliope import base
from googlecloudsdk.core import log
from googlecloudsdk.core import properties
class AppEngineAppDoesNotExist(apitools_exceptions.Error):
"""An App Engine app must be created first."""
class AppEngineAppRegionDoesNotMatch(apitools_exceptions.Error):
"""An App Engine app must have a matching region."""
class RegionNotSpecified(apitools_exceptions.Error):
"""Must specify a region to use this command."""
def create(region, product_name, enum_value):
"""Helper for implementing Firestore database create comamnds.
Guides the user through the gcloud app creation process and then updates the
database type to the requested type.
Args:
region: The region of Firestore database.
product_name: The product name of the database trying to be created.
enum_value: Enum value representing the product name in the API.
Raises:
AppEngineAppDoesNotExist: If no app has been created.
AppEngineAppRegionDoesNotMatch: If app created but region doesn't match the
--region flag.
RegionNotSpecified: User didn't specify --region.
"""
api_client = appengine_api_client.GetApiClientForTrack(base.ReleaseTrack.GA)
app = None
try:
app = api_client.GetApplication()
except apitools_exceptions.HttpNotFoundError:
if region is None:
raise AppEngineAppDoesNotExist(
'You must first create a Google App Engine app by running:\n'
'gcloud app create\n'
'The region you create the App Engine app in is '
'the same region that the Firestore database will be created in. '
'Once an App Engine region has been chosen it cannot be changed.')
else:
raise AppEngineAppDoesNotExist(
'You must first create an Google App Engine app in the '
'corresponding region by running:\n'
'gcloud app create --region={region}'.format(region=region))
current_region = app.locationId
if not region:
raise RegionNotSpecified(
'You must specify a region using the --region flag to use this '
'command. The region needs to match the Google App Engine region: '
'--region={current_region}'.format(current_region=current_region))
if current_region != region:
raise AppEngineAppRegionDoesNotMatch(
'The app engine region is {current_region} which is not the same as '
'{region}. Right now the Firestore region must match the App Engine '
'region.\n'
'Try running this command with --region={current_region}'.format(
current_region=current_region, region=region))
project = properties.VALUES.core.project.Get(required=True)
# Set the DB Type to the desired type (if needed)
if app.databaseType != enum_value:
api_client.UpdateDatabaseType(enum_value)
else:
log.status.Print(
'Success! Confirmed selection of a {product_name} database for {project}'
.format(product_name=product_name, project=project))
return
log.status.Print(
'Success! Selected {product_name} database for {project}'.format(
product_name=product_name, project=project))

View File

@@ -0,0 +1,408 @@
# -*- coding: utf-8 -*- #
# Copyright 2016 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Flags and helpers for the firestore related commands."""
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
import string
import textwrap
from googlecloudsdk.calliope import arg_parsers
def AddCollectionGroupIdsFlag(parser):
"""Adds flag for collection group ids to the given parser.
Args:
parser: The argparse parser.
"""
parser.add_argument(
'--collection-ids',
metavar='COLLECTION_GROUP_IDS',
type=arg_parsers.ArgList(),
help="""
List specifying which collection groups will be included in the operation.
When omitted, all collection groups are included.
For example, to operate on only the `customers` and `orders`
collections groups:
$ {command} --collection-ids='customers','orders'
""",
)
def AddDatabaseIdFlag(parser, required=False, hidden=False):
"""Adds flag for database id to the given parser.
Args:
parser: The argparse parser.
required: Whether the flag must be set for running the command, a bool.
hidden: Whether the flag is hidden, a bool.
"""
if not required:
helper_text = """\
The database to operate on. The default value is `(default)`.
For example, to operate on database `foo`:
$ {command} --database='foo'
"""
else:
helper_text = """\
The database to operate on.
For example, to operate on database `foo`:
$ {command} --database='foo'
"""
parser.add_argument(
'--database',
metavar='DATABASE',
type=str,
default='(default)' if not required else None,
required=required,
hidden=hidden,
help=helper_text,
)
def AddNamespaceIdsFlag(parser):
"""Adds flag for namespace ids to the given parser."""
parser.add_argument(
'--namespace-ids',
metavar='NAMESPACE_IDS',
type=arg_parsers.ArgList(),
help="""
List specifying which namespaces will be included in the operation.
When omitted, all namespaces are included.
This is only supported for Datastore Mode databases.
For example, to operate on only the `customers` and `orders` namespaces:
$ {command} --namespaces-ids='customers','orders'
""",
)
def AddSnapshotTimeFlag(parser):
"""Adds flag for snapshot time to the given parser.
Args:
parser: The argparse parser.
"""
parser.add_argument(
'--snapshot-time',
metavar='SNAPSHOT_TIME',
type=str,
default=None,
required=False,
help="""
The version of the database to export.
The timestamp must be in the past, rounded to the minute and not older
than `earliestVersionTime`. If specified, then the exported documents will
represent a consistent view of the database at the provided time.
Otherwise, there are no guarantees about the consistency of the exported
documents.
For example, to operate on snapshot time `2023-05-26T10:20:00.00Z`:
$ {command} --snapshot-time='2023-05-26T10:20:00.00Z'
""",
)
def AddLocationFlag(
parser, required=False, hidden=False, suggestion_aliases=None
):
"""Adds flag for location to the given parser.
Args:
parser: The argparse parser.
required: Whether the flag must be set for running the command, a bool.
hidden: Whether the flag is hidden in document. a bool.
suggestion_aliases: A list of flag name aliases. A list of string.
"""
parser.add_argument(
'--location',
metavar='LOCATION',
required=required,
hidden=hidden,
type=str,
suggestion_aliases=suggestion_aliases,
help="""
The location to operate on. Available locations are listed at
https://cloud.google.com/firestore/docs/locations.
For example, to operate on location `us-east1`:
$ {command} --location='us-east1'
""",
)
def AddBackupFlag(parser):
"""Adds flag for backup to the given parser.
Args:
parser: The argparse parser.
"""
parser.add_argument(
'--backup',
metavar='BACKUP',
required=True,
type=str,
help="""
The backup to operate on.
For example, to operate on backup `cf9f748a-7980-4703-b1a1-d1ffff591db0`:
$ {command} --backup='cf9f748a-7980-4703-b1a1-d1ffff591db0'
""",
)
def AddBackupScheduleFlag(parser):
"""Adds flag for backup schedule id to the given parser.
Args:
parser: The argparse parser.
"""
parser.add_argument(
'--backup-schedule',
metavar='BACKUP_SCHEDULE',
required=True,
type=str,
help="""
The backup schedule to operate on.
For example, to operate on backup schedule `091a49a0-223f-4c98-8c69-a284abbdb26b`:
$ {command} --backup-schedule='091a49a0-223f-4c98-8c69-a284abbdb26b'
""",
)
def AddRetentionFlag(parser, required=False):
"""Adds flag for retention to the given parser.
Args:
parser: The argparse parser.
required: Whether the flag must be set for running the command, a bool.
"""
parser.add_argument(
'--retention',
metavar='RETENTION',
required=required,
type=arg_parsers.Duration(),
help=textwrap.dedent("""\
The rention of the backup. At what relative time in the future,
compared to the creation time of the backup should the backup be
deleted, i.e. keep backups for 7 days.
For example, to set retention as 7 days.
$ {command} --retention=7d
"""),
)
def AddRecurrenceFlag(parser):
"""Adds flag for recurrence to the given parser.
Args:
parser: The argparse parser.
"""
group = parser.add_group(
help='Recurrence settings of a backup schedule.',
required=True,
)
help_text = """\
The recurrence settings of a backup schedule.
Currently only daily and weekly backup schedules are supported.
When a weekly backup schedule is created, day-of-week is needed.
For example, to create a weekly backup schedule which creates backups on
Monday.
$ {command} --recurrence=weekly --day-of-week=MON
"""
group.add_argument('--recurrence', type=str, help=help_text, required=True)
help_text = """\
The day of week (UTC time zone) of when backups are created.
The available values are: `MON`, `TUE`, `WED`, `THU`, `FRI`, `SAT`,`SUN`.
Values are case insensitive.
This is required when creating a weekly backup schedule.
"""
group.add_argument(
'--day-of-week',
choices=arg_parsers.DayOfWeek.DAYS,
type=arg_parsers.DayOfWeek.Parse,
help=help_text,
required=False,
)
def AddEncryptionConfigGroup(parser, source_type):
"""Adds flags for the database's encryption configuration to the given parser.
Args:
parser: The argparse parser.
source_type: "backup" if a restore; "database" if a clone
"""
encryption_config = parser.add_argument_group(
required=False,
help=textwrap.dedent(string.Template("""\
The encryption configuration of the new database being created from the $source_type.
If not specified, the same encryption settings as the $source_type will be used.
To create a CMEK-enabled database:
$$ {command} --encryption-type=customer-managed-encryption --kms-key-name=projects/PROJECT_ID/locations/LOCATION_ID/keyRings/KEY_RING_ID/cryptoKeys/CRYPTO_KEY_ID
To create a Google-default-encrypted database:
$$ {command} --encryption-type=google-default-encryption
To create a database using the same encryption settings as the $source_type:
$$ {command} --encryption-type=use-source-encryption
""").substitute(source_type=source_type)),
)
encryption_config.add_argument(
'--encryption-type',
metavar='ENCRYPTION_TYPE',
type=str,
required=True,
choices=[
'use-source-encryption',
'customer-managed-encryption',
'google-default-encryption',
],
help=textwrap.dedent("""\
The encryption type of the destination database.
"""),
)
AddKmsKeyNameFlag(
encryption_config,
'This flag must only be specified when encryption-type is'
' `customer-managed-encryption`.',
)
def AddKmsKeyNameFlag(parser, additional_help_text=None):
"""Adds flag for KMS Key Name to the given parser.
Args:
parser: The argparse parser.
additional_help_text: Additional help text to be added to the flag.
"""
help_text = textwrap.dedent("""
The resource ID of a Cloud KMS key. If set, the database created will be a Customer-Managed Encryption Key (CMEK) database encrypted with this key.
This feature is allowlist only in initial launch.
Only a key in the same location as this database is allowed to be used for encryption.
For Firestore's nam5 multi-region, this corresponds to Cloud KMS location us.
For Firestore's eur3 multi-region, this corresponds to Cloud KMS location europe.
See https://cloud.google.com/kms/docs/locations.
This value should be the KMS key resource ID in the format of `projects/{project_id}/locations/{kms_location}/keyRings/{key_ring}/cryptoKeys/{crypto_key}`.
How to retrieve this resource ID is listed at https://cloud.google.com/kms/docs/getting-resource-ids#getting_the_id_for_a_key_and_version.
""")
if additional_help_text:
help_text = help_text + '\n\n' + additional_help_text
parser.add_argument(
'--kms-key-name',
metavar='KMS_KEY_NAME',
type=str,
required=False,
default=None,
help=help_text,
)
def AddDestinationDatabase(parser, action_name, source_type):
parser.add_argument(
'--destination-database',
metavar='DESTINATION_DATABASE',
type=str,
required=True,
help=textwrap.dedent(f"""\
Destination database to {action_name} to. Destination database will be created in the same location as the source {source_type}.
This value should be 4-63 characters. Valid characters are /[a-z][0-9]-/
with first character a letter and the last a letter or a number. Must
not be UUID-like /[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}/.
Using "(default)" database ID is also allowed.
For example, to {action_name} to database `testdb`:
$ {{command}} --destination-database=testdb
"""),
)
def AddTags(parser, resource_type):
"""Adds the --tags flag to the given parser.
Args:
parser: The parser to add the flag to.
resource_type: The resource type to use in the help text (e.g. 'database').
"""
parser.add_argument(
'--tags',
metavar='KEY=VALUE',
type=arg_parsers.ArgDict(),
default=None,
help=textwrap.dedent(f"""\
Tags to attach to the destination {resource_type}. Example: --tags=key1=value1,key2=value2
For example, to attach tags to a {resource_type}:
$ --tags=key1=value1,key2=value2
"""),
)
def AddUserCredsIdArg(parser):
"""Adds positional arg for user creds id to the given parser.
Args:
parser: The argparse parser.
"""
parser.add_argument(
'user_creds',
metavar='USER_CREDS',
type=str,
help="""
The user creds to operate on.
For example, to operate on user creds `creds-name-1`:
$ {command} creds-name-1
""",
)

View File

@@ -0,0 +1,203 @@
# Copyright 2018 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
type:
api_field: googleFirestoreAdminV1Database.type
arg_name: type
help_text: The database type.
required: false
choices:
- arg_value: firestore-native
enum_value: FIRESTORE_NATIVE
- arg_value: datastore-mode
enum_value: DATASTORE_MODE
enable_delete_protection:
api_field: googleFirestoreAdminV1Database.deleteProtectionState
arg_name: delete-protection
action: store_true
default: null
required: false
help_text: |
If set to true, the Firestore database will be updated to have database delete protection
enabled. A database with delete protection enabled cannot be deleted. You can disable the
delete protection via --no-delete-protection.
choices:
- arg_value: true
enum_value: DELETE_PROTECTION_ENABLED
- arg_value: false
enum_value: DELETE_PROTECTION_DISABLED
enable_pitr:
api_field: googleFirestoreAdminV1Database.pointInTimeRecoveryEnablement
arg_name: enable-pitr
action: store_true
default: null
required: false
help_text: |
If set to true, the Firestore database will be updated to enable Point In Time Recovery. You can
disable the this feature via --no-enable-pitr.
choices:
- arg_value: true
enum_value: POINT_IN_TIME_RECOVERY_ENABLED
- arg_value: false
enum_value: POINT_IN_TIME_RECOVERY_DISABLED
query_scope:
api_field: googleFirestoreAdminV1Index.queryScope
arg_name: query-scope
help_text: Query scope the index applies to.
default: collection
choices:
- arg_value: collection
enum_value: COLLECTION
- arg_value: collection-group
enum_value: COLLECTION_GROUP
- arg_value: collection-recursive
enum_value: COLLECTION_RECURSIVE
api_scope:
api_field: googleFirestoreAdminV1Index.apiScope
arg_name: api-scope
help_text: Api scope the index applies to.
default: any-api
choices:
- arg_value: any-api
enum_value: ANY_API
- arg_value: datastore-mode-api
enum_value: DATASTORE_MODE_API
- arg_value: mongodb-compatible-api
enum_value: MONGODB_COMPATIBLE_API
density:
api_field: googleFirestoreAdminV1Index.density
arg_name: density
required: false
help_text: Density of the index.
default: null
choices:
- arg_value: density-unspecified
enum_value: DENSITY_UNSPECIFIED
- arg_value: sparse-any
enum_value: SPARSE_ANY
- arg_value: sparse-all
enum_value: SPARSE_ALL
- arg_value: DENSE
enum_value: DENSE
field_config:
api_field: googleFirestoreAdminV1Index.fields
arg_name: field-config
help_text: Configuration for an index field.
type: arg_object
required: [field-path]
spec:
- api_field: fieldPath
json_name: field-path
help_text: Specifies the field path (e.g. 'address.city'). This is required.
- api_field: arrayConfig
json_name: array-config
help_text: |
Specifies the configuration for an array field. The only valid option
is 'contains'. Exactly one of 'order', 'array-config', or
'vector-config' must be specified.
- api_field: order
json_name: order
help_text: |
Specifies the order. Valid options are 'ascending', 'descending'.
Exactly one of 'order', 'array-config', or 'vector-config' must be
specified.
- api_field: vectorConfig
json_name: vector-config
help_text: |
Specifies the configuration for a vector field. Exactly one of
'order', 'array-config', or 'vector-config' must be specified.
index:
api_field: googleFirestoreAdminV1Field.indexConfig.indexes
arg_name: index
metavar: KEY=VALUE
processor: googlecloudsdk.command_lib.firestore.util:AddQueryScope
help_text: |
An index for the field.
This flag can be repeated to provide multiple indexes. Any existing indexes will
be overwritten with the ones provided. Any omitted indexes will be deleted if
they currently exist.
The following keys are allowed:
*order*:::: Specifies the order. Valid options are: 'ascending', 'descending'.
Exactly one of 'order' or 'array-config' must be specified.
*array-config*:::: Specifies the configuration for an array field. The only
valid option is 'contains'. Exactly one of 'order' or 'array-config' must be
specified.
type:
arg_dict:
flatten: false
spec:
- api_field: fields.arrayConfig
arg_name: array-config
type: str
required: false
choices:
- arg_value: contains
enum_value: CONTAINS
- api_field: fields.order
arg_name: order
type: str
required: false
choices:
- arg_value: ascending
enum_value: ASCENDING
- arg_value: descending
enum_value: DESCENDING
disable_indexes:
arg_name: disable-indexes
help_text: If provided, the field will no longer be indexed at all.
action: store_true
multikey:
api_field: googleFirestoreAdminV1Index.multikey
arg_name: multikey
required: false
help_text: |
Optional. Whether the index is multikey. By default, the index
is not multikey. For non-multikey indexes, none of the paths in the
index definition reach or traverse an array, except via an explicit
array index. For multikey indexes, at most one of the paths in the index
definition reach or traverse an array, except via an explicit array
index. Violations will result in errors. Note this field only applies to
index with MONGODB_COMPATIBLE_API ApiScope.
action: store_true
unique:
api_field: googleFirestoreAdminV1Index.unique
arg_name: unique
required: false
help_text: |
Optional. Whether it is an unique index. Unique index ensures all values for
the indexed field(s) are unique across documents.
action: store_true
clear_exemption:
arg_name: clear-exemption
help_text: |
If provided, the field's current index configuration will be
reverted to inherit from its ancestor index configurations.
action: store_true

View File

@@ -0,0 +1,129 @@
# Note: we have some duplicated resources here because of a limitation with declarative commands.
#
# In particular, some commands (e.g. composite create) need the collection group to be specified.
# But other commands (e.g. composite list) need to function if the collection group is not specified
# (a special value of '-' is used).
#
# Normally this would be configured with command level fallthroughs; however, the declarative schema
# currently only allows command level fallthroughs to be arg_fallthroughs (as opposed to generic
# python hook fallthroughs), which means we can't provide a default value in this manner.
#
# The workaround for now is just to have different commands use different resource args with
# appropriate attribute fallthroughs (which can be generic python hooks) depending on what the
# command needs.
project:
name: project
collection: firestore.projects
attributes:
- &project
parameter_name: projectsId
attribute_name: project
help: |
Project of the {resource}.
property: core/project
database:
name: database
collection: firestore.projects.databases
attributes:
- *project
- &database
parameter_name: databasesId
attribute_name: database
help: |
Database of the {resource}.
fallthroughs:
- value: (default)
hint: |-
the default value of argument [--database] is `(default)`
# This is for the '-' collection group which allows API calls to access index IDs without specifying
# their collection group, e.g.:
# projects/{projectsId}/databases/{databasesId}/collectionGroups/-/{indexesId}
collection_group_with_default:
name: collection group
collection: firestore.projects.databases.collectionGroups
attributes:
- *project
- *database
- &collection_group_with_default
parameter_name: collectionGroupsId
attribute_name: collection-group
help: |
Collection group of the {resource}.
fallthroughs:
- hook: googlecloudsdk.command_lib.firestore.util:GetCollectionGroupFallthrough
hint: |-
provide the argument [--collection-group] on the command line
collection_group:
name: collection group
collection: firestore.projects.databases.collectionGroups
attributes:
- *project
- *database
- &collection_group
parameter_name: collectionGroupsId
attribute_name: collection-group
help: |
Collection group of the {resource}.
# Composite index IDs are unique across all collection groups; we can use the special '-' collection
# group in API calls so that the user doesn't need to enter the actual one on the command line.
index:
name: composite index
collection: firestore.projects.databases.collectionGroups.indexes
attributes:
- *project
- *database
- *collection_group_with_default
- &index
parameter_name: indexesId
attribute_name: index
help: |
Index of the {resource}.
# This resource lets us describe the special field that contains database-wide index settings. Note
# that the collection group for this field is '__default__' not '-', so we can't reuse the
# collection_group_with_default attribute.
field_with_default:
name: field
collection: firestore.projects.databases.collectionGroups.fields
attributes:
- *project
- *database
- parameter_name: collectionGroupsId
attribute_name: collection-group
help: |
Collection group of the {resource}.
fallthroughs:
- hook: googlecloudsdk.command_lib.firestore.util:GetDefaultFieldCollectionGroupFallthrough
hint: ' '
- &field_with_default
parameter_name: fieldsId
attribute_name: field
help: |
Field of the {resource}.
The field may be a simple field name (e.g. address) or a path to a field in a key-value map
(e.g. address.city.neighborhood).
fallthroughs:
- hook: googlecloudsdk.command_lib.firestore.util:GetDefaultFieldPathFallthrough
hint: ' '
field:
name: field
collection: firestore.projects.databases.collectionGroups.fields
attributes:
- *project
- *database
- *collection_group
- &field
parameter_name: fieldsId
attribute_name: field
help: |
Field of the {resource}.
The field may be a simple field name (e.g. address) or a path to a field in a key-value map
(e.g. address.city.neighborhood).

View File

@@ -0,0 +1,315 @@
# -*- coding: utf-8 -*- #
# Copyright 2018 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Utilities for Cloud Firestore commands."""
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from typing import Optional
from apitools.base.py import encoding
from googlecloudsdk.api_lib.firestore import api_utils as fs_api
from googlecloudsdk.api_lib.util import apis
from googlecloudsdk.calliope import exceptions
from googlecloudsdk.command_lib.util.apis import arg_utils
from googlecloudsdk.core.util import text
FIRESTORE_INDEX_API_VERSION = 'v1'
def GetMessagesModule():
return apis.GetMessagesModule('firestore', FIRESTORE_INDEX_API_VERSION)
def GetDatabaseFallthrough():
"""Python hook to get the value for the default database.
Firestore currently only supports one database called '(default)'.
Returns:
The name of the default database.
"""
return '(default)'
def GetCollectionGroupFallthrough():
"""Python hook to get the value for the '-' collection group.
See details at:
https://cloud.google.com/apis/design/design_patterns#list_sub-collections
This allows us to describe or delete an index by specifying just its ID,
without needing to know which collection group it belongs to.
Returns:
The value of the wildcard collection group.
"""
return '-'
def GetDefaultFieldCollectionGroupFallthrough():
return '__default__'
def GetDefaultFieldPathFallthrough():
return '*'
def ValidateFieldConfig(unused_ref, args, request):
"""Python hook to validate the field configuration of the given request.
Note that this hook is only called after the request has been formed based on
the spec. Thus, the validation of the user's choices for order and
array-config, as well as the check for the required field-path attribute, have
already been performed. As such the only remaining things to verify are that
the user has specified at least 2 fields, and that exactly one of order or
array-config was specified for each field.
Args:
unused_ref: The resource ref (unused).
args: The parsed arg namespace.
request: The request formed based on the spec.
Returns:
The original request assuming the field configuration is valid.
Raises:
InvalidArgumentException: If the field configuration is invalid.
"""
invalid_field_configs = []
for field_config in args.field_config:
# Because of the way declarative ArgDict parsing works, the type of
# field_config here is already an apitools message, as opposed to an
# ArgDict.
order = field_config.order
array_config = field_config.arrayConfig
if field_config.vectorConfig:
# TODO(b/302742966): enhance validation and error message once the vector
# config is in preview.
continue
if (order and array_config) or (not order and not array_config):
invalid_field_configs.append(field_config)
if invalid_field_configs:
raise exceptions.InvalidArgumentException(
'--field-config',
"Exactly one of 'order' or 'array-config' must be specified for the "
"{field_word} with the following {path_word}: [{paths}].".format(
field_word=text.Pluralize(len(invalid_field_configs), 'field'),
path_word=text.Pluralize(len(invalid_field_configs), 'path'),
paths=', '.join(field_config.fieldPath
for field_config in invalid_field_configs)))
return request
def ExtractOperationMetadata(response, unused_args):
"""Python hook to extract the operation metadata message.
This is needed because apitools gives us a MetadataValue message for the
operation metadata field, instead of the actual message that we want.
Args:
response: The response field in the operation returned by the API.
unused_args: The parsed arg namespace (unused).
Returns:
The metadata field converted to a
GoogleFirestoreAdminV1IndexOperationMetadata message.
"""
messages = GetMessagesModule()
return encoding.DictToMessage(
encoding.MessageToDict(response.metadata),
messages.GoogleFirestoreAdminV1IndexOperationMetadata)
def ValidateFieldArg(ref, unused_args, request):
"""Python hook to validate that the field reference is correctly specified.
The user should be able to describe database-wide settings as well as
collection-group wide settings; however it doesn't make sense to describe a
particular field path's settings unless the collection group was also
specified. The API will catch this but it's better to do it here for a clearer
error message.
Args:
ref: The field resource reference.
unused_args: The parsed arg namespace (unused).
request: The field describe request.
Returns:
The original request assuming the field configuration is valid.
Raises:
InvalidArgumentException: If the field resource is invalid.
"""
if (ref.fieldsId != GetDefaultFieldPathFallthrough() and
ref.collectionGroupsId == GetDefaultFieldCollectionGroupFallthrough()):
raise exceptions.InvalidArgumentException(
'FIELD',
'Collection group must be provided if the field path was specified.')
return request
def ValidateFieldIndexArgs(args):
"""Validates the repeated --index arg.
Args:
args: The parsed arg namespace.
Raises:
InvalidArgumentException: If the provided indexes are incorrectly specified.
"""
if not args.IsSpecified('index'):
return
for index in args.index:
for field in index.fields:
order = field.order
array_config = field.arrayConfig
if (order and array_config) or (not order and not array_config):
raise exceptions.InvalidArgumentException(
'--index',
"Exactly one of 'order' or 'array-config' must be specified "
"for each --index flag provided.")
def UpdateFieldRequestTtls(ref, args, request):
"""Update field request for TTL.
Args:
ref: The field resource reference(unused).
args: The parsed arg namespace.
request: The ttl field request.
Raises:
InvalidArgumentException: If the provided indexes are incorrectly specified.
Returns:
UpdateFieldRequest
"""
messages = GetMessagesModule()
request.updateMask = 'ttlConfig'
ttl_config = None
if args.enable_ttl:
ttl_config = messages.GoogleFirestoreAdminV1TtlConfig()
request.googleFirestoreAdminV1Field = messages.GoogleFirestoreAdminV1Field(
name=ref.RelativeName(),
ttlConfig=ttl_config)
return request
def AddQueryScope(indexes):
messages = GetMessagesModule()
scope = (
messages.GoogleFirestoreAdminV1Index.QueryScopeValueValuesEnum.COLLECTION)
for index in indexes:
index.queryScope = scope
return indexes
def ValidateFieldUpdateRequest(unused_ref, args, req):
ValidateFieldIndexArgs(args)
return req
def AddIndexConfigToUpdateRequest(unused_ref, args, req):
"""Update patch request to include indexConfig.
The mapping of index config message to API behavior is as follows:
None - Clears the exemption
indexes=[] - Disables all indexes
indexes=[...] - Sets the index config to the indexes provided
Args:
unused_ref: The field resource reference.
args: The parsed arg namespace.
req: The auto-generated patch request.
Returns:
FirestoreProjectsDatabasesCollectionGroupsFieldsPatchRequest
"""
messages = GetMessagesModule()
if args.disable_indexes:
index_config = messages.GoogleFirestoreAdminV1IndexConfig(indexes=[])
elif args.IsSpecified('index') and req.googleFirestoreAdminV1Field:
index_config = req.googleFirestoreAdminV1Field.indexConfig
else:
index_config = None
arg_utils.SetFieldInMessage(req,
'googleFirestoreAdminV1Field.indexConfig',
index_config)
return req
def ExtractEncryptionConfig(args):
"""Parses the args and returns the encryption configuration, or none.
Args:
args: The parsed arg namespace.
Returns:
The encryption configuration, or none.
"""
messages = fs_api.GetMessages()
encryption_type = _NormalizeString(args.encryption_type)
if encryption_type == 'google-default-encryption':
_ThrowIfKmsKeyNameSet(args.kms_key_name)
return messages.GoogleFirestoreAdminV1EncryptionConfig(
googleDefaultEncryption=messages.GoogleFirestoreAdminV1GoogleDefaultEncryptionOptions()
)
elif encryption_type == 'use-source-encryption':
_ThrowIfKmsKeyNameSet(args.kms_key_name)
return messages.GoogleFirestoreAdminV1EncryptionConfig(
useSourceEncryption=messages.GoogleFirestoreAdminV1SourceEncryptionOptions()
)
elif encryption_type == 'customer-managed-encryption':
_ThrowIfKmsKeyNameNotSet(args.kms_key_name)
return messages.GoogleFirestoreAdminV1EncryptionConfig(
customerManagedEncryption=messages.GoogleFirestoreAdminV1CustomerManagedEncryptionOptions(
kmsKeyName=args.kms_key_name
)
)
elif encryption_type is not None:
raise exceptions.InvalidArgumentException(
'Invalid encryption type: {}'.format(encryption_type),
'encryption-type',
)
return None
def _NormalizeString(value: Optional[str]):
if not value:
return None
return value.casefold()
def _ThrowIfKmsKeyNameSet(kms_key_name: Optional[str]):
if kms_key_name is not None:
raise exceptions.ConflictingArgumentsException(
'--kms-key-name',
'A KMS Key cannot be set when using an --encryption-type of'
' google-default-encryption or use-source-encryption.',
)
def _ThrowIfKmsKeyNameNotSet(kms_key_name: Optional[str]):
if kms_key_name is None:
raise exceptions.RequiredArgumentException(
'--kms-key-name',
'The KMS Key Name is required'
' when using customer-managed encryption (CMEK), please use'
' --kms-key-name to specify this value',
)