416 lines
19 KiB
YAML
416 lines
19 KiB
YAML
- release_tracks: [ALPHA, BETA, GA]
|
|
help_text:
|
|
brief: |
|
|
Update a Dataproc Metastore service.
|
|
description: |
|
|
Update the metadata and/or configuration parameters of a Dataproc Metastore service.
|
|
|
|
If run asynchronously with `--async`, exits after printing
|
|
one operation name that can be used to poll the status of the
|
|
update via:
|
|
|
|
{top_command} metastore operations describe
|
|
examples: |
|
|
To update a Dataproc Metastore service with the name `my-metastore-service` to
|
|
have the port number 8080, and add the two labels, `env` and `foo`, run:
|
|
|
|
$ {command} my-metastore-service --port=8080 --update-labels=env=test,foo=bar
|
|
|
|
request:
|
|
ALPHA:
|
|
api_version: v1alpha
|
|
modify_request_hooks:
|
|
- googlecloudsdk.command_lib.metastore.services.util:UpdateScalingConfig
|
|
- googlecloudsdk.command_lib.metastore.services.util:SetServiceRequestUpdateHiveMetastoreConfigs
|
|
- googlecloudsdk.command_lib.metastore.services.util:GenerateUpdateAuxiliaryVersionsConfigs
|
|
- googlecloudsdk.command_lib.metastore.services.util:UpdateServiceMaskHook
|
|
- googlecloudsdk.command_lib.metastore.validators:ValidateServiceMutexConfig
|
|
BETA:
|
|
api_version: v1beta
|
|
modify_request_hooks:
|
|
- googlecloudsdk.command_lib.metastore.services.util:UpdateScalingConfig
|
|
- googlecloudsdk.command_lib.metastore.services.util:SetServiceRequestUpdateHiveMetastoreConfigs
|
|
- googlecloudsdk.command_lib.metastore.services.util:GenerateUpdateAuxiliaryVersionsConfigs
|
|
- googlecloudsdk.command_lib.metastore.services.util:UpdateServiceMaskHook
|
|
- googlecloudsdk.command_lib.metastore.validators:ValidateServiceMutexConfig
|
|
GA:
|
|
api_version: v1
|
|
modify_request_hooks:
|
|
- googlecloudsdk.command_lib.metastore.services.util:UpdateScalingConfig
|
|
- googlecloudsdk.command_lib.metastore.services.util:SetServiceRequestUpdateHiveMetastoreConfigs
|
|
- googlecloudsdk.command_lib.metastore.services.util:GenerateUpdateAuxiliaryVersionsConfigs
|
|
- googlecloudsdk.command_lib.metastore.services.util:UpdateServiceMaskHook
|
|
- googlecloudsdk.command_lib.metastore.validators:ValidateServiceMutexConfig
|
|
collection: metastore.projects.locations.services
|
|
|
|
arguments:
|
|
resource:
|
|
help_text: |
|
|
Arguments and flags that specify the Dataproc Metastore service
|
|
you want to update.
|
|
spec: !REF googlecloudsdk.command_lib.metastore.resources:service
|
|
params:
|
|
- arg_name: port
|
|
api_field: service.port
|
|
processor: googlecloudsdk.command_lib.metastore.validators:ValidatePort
|
|
help_text: |
|
|
The TCP port on which the Metastore service will listen.
|
|
- arg_name: data-catalog-sync
|
|
default: false
|
|
api_field: service.metadataIntegration.dataCatalogConfig.enabled
|
|
help_text: |
|
|
Boolean flag to determine whether or not Dataproc Metastore metadata sync to Data Catalog
|
|
is enabled, false if unspecified.
|
|
Mutually exclusive with flag `--encryption-kms-key`.
|
|
Cannot be updated if the service uses customer-managed encryption keys.
|
|
- arg_name: deletion-protection
|
|
type: bool
|
|
api_field: service.deletionProtection
|
|
help_text: |
|
|
Flag that enables delete protection on Dataproc Metastore instance to prevent
|
|
accidental deletions of the instance.
|
|
Use --deletion-protection to enable and --no-deletion-protection to disable.
|
|
- group:
|
|
help_text: |
|
|
The instance size of the Dataproc Metastore instance.
|
|
mutex: true
|
|
params:
|
|
- arg_name: tier
|
|
api_field: service.tier
|
|
help_text: |
|
|
The tier of the service.
|
|
choices:
|
|
- arg_value: enterprise
|
|
enum_value: ENTERPRISE
|
|
help_text: |
|
|
The enterprise tier provides multi-zone high availability, and sufficient
|
|
scalability for enterprise-level Dataproc Metastore workloads.
|
|
- arg_value: developer
|
|
enum_value: DEVELOPER
|
|
help_text: |
|
|
The developer tier provides limited scalability and no fault tolerance.
|
|
Good for low-cost proof-of-concept.
|
|
- arg_name: scaling-factor
|
|
api_field: service.scalingConfig.scalingFactor
|
|
help_text: |
|
|
The scaling factor of the service.
|
|
Can be one of [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0].
|
|
processor: googlecloudsdk.command_lib.metastore.validators:ValidateScalingFactor
|
|
- arg_name: instance-size
|
|
api_field: service.scalingConfig.instanceSize
|
|
help_text: |
|
|
The instance size of the service.
|
|
choices:
|
|
- arg_value: extra-small
|
|
enum_value: EXTRA_SMALL
|
|
help_text: |
|
|
Extra small instance size, maps to a scaling factor of 0.1.
|
|
- arg_value: small
|
|
enum_value: SMALL
|
|
help_text: |
|
|
Small instance size, maps to a scaling factor of 0.5.
|
|
- arg_value: medium
|
|
enum_value: MEDIUM
|
|
help_text: |
|
|
Medium instance size, maps to a scaling factor of 1.0.
|
|
- arg_value: large
|
|
enum_value: LARGE
|
|
help_text: |
|
|
Large instance size, maps to a scaling factor of 3.0.
|
|
- arg_value: extra-large
|
|
enum_value: EXTRA_LARGE
|
|
help_text: |
|
|
Extra large instance size, maps to a scaling factor of 6.0.
|
|
- group:
|
|
help_text: |
|
|
The autoscaling configuration of the Dataproc Metastore instance.
|
|
params:
|
|
- arg_name: min-scaling-factor
|
|
api_field: service.scalingConfig.autoscalingConfig.limitConfig.minScalingFactor
|
|
help_text: |
|
|
The minimum scaling factor allowed by the autoscaler.
|
|
Can be one of [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0].
|
|
The default value is 0.1.
|
|
processor: googlecloudsdk.command_lib.metastore.validators:ValidateMinScalingFactor
|
|
- arg_name: max-scaling-factor
|
|
api_field: service.scalingConfig.autoscalingConfig.limitConfig.maxScalingFactor
|
|
help_text: |
|
|
The maximum scaling factor allowed by the autoscaler.
|
|
Can be one of [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0].
|
|
The default value is 6.0.
|
|
processor: googlecloudsdk.command_lib.metastore.validators:ValidateMaxScalingFactor
|
|
- arg_name: autoscaling-enabled
|
|
api_field: service.scalingConfig.autoscalingConfig.autoscalingEnabled
|
|
help_text: |
|
|
A boolean flag to determine whether Dataproc Metastore autoscaling should be enabled,
|
|
false if unspecified.
|
|
|
|
The default minimum and maximum scaling factors are 0.1 and 6.0, respectively.
|
|
|
|
The minimum and maximum scaling factors can be specified using --min-scaling-factor and
|
|
--max-scaling-factor.
|
|
- group:
|
|
help_text: |
|
|
Configuration properties specific to running Hive metastore
|
|
software as the metastore service.
|
|
mutex: true
|
|
params:
|
|
- arg_name: update-hive-metastore-configs-from-file
|
|
type: "googlecloudsdk.calliope.arg_parsers:FileContents:"
|
|
help_text: |
|
|
Path to a XML file containing a mapping of Hive metastore configuration key-value
|
|
pairs to apply to the Hive metastore.
|
|
|
|
For example:
|
|
|
|
hive-site.xml
|
|
<configuration>
|
|
<property>
|
|
<name>hive.metastore.warehouse.dir</name>
|
|
<value>${test.warehouse.dir}</value>
|
|
<description></description>
|
|
</property>
|
|
</configuration>
|
|
- group:
|
|
help_text: |
|
|
Modify the Hive metastore configuration properties by passing key-value pairs
|
|
in through the flags.
|
|
params:
|
|
- arg_name: update-hive-metastore-configs
|
|
metavar: KEY=VALUE
|
|
type: "googlecloudsdk.calliope.arg_parsers:ArgDict:"
|
|
help_text: |
|
|
Comma-separated list of Hive metastore configurations. Each configuration has
|
|
the form "NAME=VALUE".
|
|
- group:
|
|
mutex: true
|
|
params:
|
|
- arg_name: remove-hive-metastore-configs
|
|
type: "googlecloudsdk.calliope.arg_parsers:ArgList:"
|
|
help_text: |
|
|
Comma-separated list of configuration keys to remove with the form
|
|
"KEY1, KEY2". If a label does not exist it is silently ignored. If
|
|
--update-hive-metastore-configs is also specified, then
|
|
--remove-hive-metastore-configs is applied first.
|
|
- arg_name: clear-hive-metastore-configs
|
|
action: store_true
|
|
help_text: |
|
|
clear existing Hive metastore configurations. If --update-hive-metastore-configs
|
|
is also specified, then --clear-hive-metastore-configs is applied first.
|
|
- group:
|
|
help_text: |
|
|
Information used to configure the Hive metastore service as a service principal
|
|
in a Kerberos realm.
|
|
params:
|
|
- arg_name: kerberos-principal
|
|
api_field: service.hiveMetastoreConfig.kerberosConfig.principal
|
|
processor: googlecloudsdk.command_lib.metastore.validators:ValidateKerberosPrincipal
|
|
help_text: |
|
|
A Kerberos principal that exists in the KDC to authenticate as. A typical principal
|
|
is of the form "primary/instance@REALM", but there is no exact format.
|
|
- arg_name: krb5-config
|
|
api_field: service.hiveMetastoreConfig.kerberosConfig.krb5ConfigGcsUri
|
|
processor: googlecloudsdk.command_lib.metastore.validators:ValidateGcsUri:arg_name=--krb5-config
|
|
help_text: |
|
|
A Cloud Storage URI that specifies the path to a krb5.conf file. It is of the form
|
|
gs://{bucket_name}/path/krb5.conf, although the file does not need to be named
|
|
krb5.conf explicitly.
|
|
- arg_name: keytab
|
|
api_field: service.hiveMetastoreConfig.kerberosConfig.keytab.cloudSecret
|
|
processor: googlecloudsdk.command_lib.metastore.parsers:ParseSecretManagerSecretVersion
|
|
help_text: |
|
|
A Kerberos keytab file that can be used to authenticate a service principal
|
|
with a Kerberos Key Distribution Center. This is a Secret Manager secret version,
|
|
and can be fully-qualified URL, or relative name in the form
|
|
`projects/{project_id}/secrets/{secret_id}/versions/{version_id}`.
|
|
- group:
|
|
help_text: |
|
|
The one hour maintenance window that specifies when Dataproc
|
|
Metastore may perform system maintenance operation to the service, in
|
|
UTC time.
|
|
params:
|
|
- arg_name: maintenance-window-day
|
|
api_field: service.maintenanceWindow.dayOfWeek
|
|
required: true
|
|
choices:
|
|
- arg_value: mon
|
|
enum_value: MONDAY
|
|
- arg_value: tue
|
|
enum_value: TUESDAY
|
|
- arg_value: wed
|
|
enum_value: WEDNESDAY
|
|
- arg_value: thu
|
|
enum_value: THURSDAY
|
|
- arg_value: fri
|
|
enum_value: FRIDAY
|
|
- arg_value: sat
|
|
enum_value: SATURDAY
|
|
- arg_value: sun
|
|
enum_value: SUNDAY
|
|
help_text: |
|
|
The day of week when the window starts, e.g., `sun`.
|
|
- arg_name: maintenance-window-hour
|
|
api_field: service.maintenanceWindow.hourOfDay
|
|
required: true
|
|
processor: googlecloudsdk.command_lib.metastore.validators:ValidateHourOfDay
|
|
help_text: |
|
|
The hour of day (0-23) when the window starts.
|
|
- arg_name: endpoint-protocol
|
|
help_text: |
|
|
The protocol to use for the metastore service endpoint.
|
|
choices:
|
|
- arg_value: thrift
|
|
enum_value: THRIFT
|
|
help_text: |
|
|
The legacy Apache `THRIFT` protocol.
|
|
- arg_value: grpc
|
|
enum_value: GRPC
|
|
help_text: |
|
|
The modernized `GRPC` protocol.
|
|
api_field: service.hiveMetastoreConfig.endpointProtocol
|
|
- group:
|
|
help_text: |
|
|
Auxiliary versions configuration for the Dataproc Metastore service. When specified,
|
|
a secondary Hive metastore service is created along with the primary service.
|
|
mutex: true
|
|
params:
|
|
- arg_name: update-auxiliary-versions-from-file
|
|
api_field: service.hiveMetastoreConfig.auxiliaryVersions
|
|
type: "googlecloudsdk.calliope.arg_parsers:YAMLFileContents:"
|
|
processor: googlecloudsdk.command_lib.metastore.services.util:LoadAuxiliaryVersionsConfigsFromYamlFile
|
|
help_text: |
|
|
Path to a YAML file containing the auxiliary versions configuration for
|
|
Dataproc Metastore instance. The file should contain a unique auxiliary service name and
|
|
auxiliary version that is lower than the primary service version. The primary version's
|
|
Hive metastore configs are applied to the auxiliary version. Additional Hive metastore
|
|
configs can be specified for the auxiliary version using "config_overrides". If a
|
|
specified config property has already been overridden in the primary Hive metastore
|
|
version's configs, the auxiliary version's override takes precedence.
|
|
The contents of the file should be structured as follows:
|
|
|
|
YAML:
|
|
|
|
```
|
|
- name: aux-service1
|
|
version: x.y.z
|
|
config_overrides:
|
|
key1: value1
|
|
key2: value2
|
|
...
|
|
- name: aux-service2
|
|
version: x.y.z
|
|
config_overrides:
|
|
key1: value1
|
|
key2: value2
|
|
...
|
|
...
|
|
```
|
|
- group:
|
|
params:
|
|
- arg_name: add-auxiliary-versions
|
|
type: "arg_list"
|
|
help_text: |
|
|
Comma-separated list of auxiliary Hive metastore versions to deploy.
|
|
Auxiliary Hive metastore versions must be less than the primary Hive metastore
|
|
service's version.
|
|
- arg_name: clear-auxiliary-versions
|
|
action: store_true
|
|
help_text: |
|
|
Clears the existing auxiliary services attached to the primary Hive metastore
|
|
services. If --add-auxiliary-versions is also specified, --clear-auxiliary-versions
|
|
is applied first.
|
|
- group:
|
|
mutex: true
|
|
help_text: |
|
|
Scheduled backup configurations to enable a scheduled backup of the metastore service.
|
|
params:
|
|
- arg_name: scheduled-backup-configs-from-file
|
|
api_field: service.scheduledBackup
|
|
type: "googlecloudsdk.calliope.arg_parsers:FileContents:"
|
|
processor: googlecloudsdk.command_lib.metastore.services.util:LoadScheduledBackupConfigsFromJsonFile
|
|
help_text: |
|
|
Path to a JSON file containing a mapping of scheduled backup configurations key-value pairs to apply to the scheduled backup.
|
|
|
|
For example:
|
|
|
|
backup_configs.json
|
|
{
|
|
"enabled": true,
|
|
"cron_schedule": "0 * * * *",
|
|
"backup_location": "gs://example-bucket",
|
|
"time_zone": "UTC"
|
|
}
|
|
- group:
|
|
help_text: |
|
|
Scheduled backup configurations to enable a scheduled backup of the metastore service.
|
|
params:
|
|
- arg_name: enable-scheduled-backup
|
|
api_field: service.scheduledBackup.enabled
|
|
action: store_true
|
|
help_text: |
|
|
Enable or disable scheduled backups for the Metastore service.
|
|
- arg_name: scheduled-backup-cron
|
|
api_field: service.scheduledBackup.cronSchedule
|
|
help_text: |
|
|
Specify the frequency of a scheduled backup of the metastore service.
|
|
- arg_name: scheduled-backup-location
|
|
api_field: service.scheduledBackup.backupLocation
|
|
help_text: |
|
|
Set the Cloud Storage URI to store the scheduled backups of the metastore service.
|
|
- group:
|
|
release_tracks: [ALPHA, BETA]
|
|
hidden: true
|
|
help_text: |
|
|
Configuration required for migrating to BigQuery Metastore service.
|
|
params:
|
|
- arg_name: bigquery-project-id
|
|
api_field: service.bigqueryMetastoreMigrationConfig.bigqueryProjectId
|
|
help_text: |
|
|
Project ID where the BigQuery resources (e.g. datasets, tables, etc.)
|
|
should be created.
|
|
- arg_name: bigquery-dataset-location
|
|
api_field: service.bigqueryMetastoreMigrationConfig.bigqueryDatasetLocation
|
|
help_text: |
|
|
The location where the BigQuery resources (e.g. datasets, tables, etc.)
|
|
should be created (e.g. us-central1, us, eu, etc.).
|
|
- arg_name: desired-migration-state
|
|
api_field: service.bigqueryMetastoreMigrationConfig.desiredMigrationState
|
|
help_text: |
|
|
The desired state of the migration. Note that this also reflects the
|
|
current state of the migration. If an attempt to update to a new desired
|
|
state fails, the migration will revert to the previous state.
|
|
choices:
|
|
- arg_value: migrate
|
|
enum_value: MIGRATE
|
|
help_text: |
|
|
By setting the desired migration state to `MIGRATE`, metadata updates in
|
|
Dataproc Metastore will be replicated to the BigQuery Metastore service,
|
|
ensuring that it remains consistently synchronized with Dataproc
|
|
Metastore. Note that this includes initial backfill of existing metadata.
|
|
- arg_value: cutover
|
|
enum_value: CUTOVER
|
|
help_text: |
|
|
By setting the desired migration state to `CUTOVER`, all metadata
|
|
requests are routed to BigQuery Metastore service and Dataproc Metastore
|
|
only functions as a proxy. This state can be considered as the completion
|
|
of the migration.
|
|
- arg_value: cancel
|
|
enum_value: CANCEL
|
|
help_text: |
|
|
By setting the desired migration state to `CANCEL`, the migration is
|
|
effectively cancelled. If the previous migration state was `MIGRATE`,
|
|
then replication to BigQuery Metastore will be cancelled. If the previous
|
|
state was `CUTOVER`, then metadata requests will now be served from
|
|
Dataproc Metastore instead of BigQuery Metastore. Note that existing
|
|
metadata changes replicated to BigQuery Metastore service are not rolled
|
|
back.
|
|
labels:
|
|
api_field: service.labels
|
|
|
|
async:
|
|
collection: metastore.projects.locations.operations
|
|
update:
|
|
# b/136698204
|
|
read_modify_update: true
|
|
disable_auto_field_mask: true
|