530 lines
24 KiB
YAML
530 lines
24 KiB
YAML
- release_tracks: [ALPHA, BETA, GA]
|
|
help_text:
|
|
brief: |
|
|
Create a Dataproc Metastore service.
|
|
description: |
|
|
Create a new Dataproc Metastore service with the given name and configurations.
|
|
|
|
If run asynchronously with `--async`, exits after printing
|
|
one operation name that can be used to poll the status of the
|
|
creation via:
|
|
|
|
{top_command} metastore operations describe
|
|
examples: |
|
|
To create a Dataproc Metastore service with the name `my-metastore-service` in location
|
|
`us-central` using a non default port 9090, run:
|
|
|
|
$ {command} my-metastore-service --location=us-central1 --port=9090
|
|
|
|
To create a Dataproc Metastore service with the name `my-metastore-service` in location
|
|
`us-central` using a non default network foo, run:
|
|
|
|
$ {command} my-metastore-service --location=us-central1 --network=foo
|
|
|
|
request:
|
|
ALPHA:
|
|
api_version: v1alpha
|
|
modify_request_hooks:
|
|
- googlecloudsdk.command_lib.metastore.services.util:UpdateScalingConfigForCreate
|
|
- googlecloudsdk.command_lib.metastore.services.util:GenerateNetworkConfigFromSubnetList
|
|
- googlecloudsdk.command_lib.metastore.services.util:GenerateAuxiliaryVersionsConfigFromList
|
|
- googlecloudsdk.command_lib.metastore.validators:ValidateServiceMutexConfig
|
|
- googlecloudsdk.command_lib.metastore.validators:ValidateScheduledBackupConfigs
|
|
- googlecloudsdk.command_lib.metastore.validators:ValidateKmsKeys
|
|
BETA:
|
|
api_version: v1beta
|
|
modify_request_hooks:
|
|
- googlecloudsdk.command_lib.metastore.services.util:UpdateScalingConfigForCreate
|
|
- googlecloudsdk.command_lib.metastore.services.util:GenerateNetworkConfigFromSubnetList
|
|
- googlecloudsdk.command_lib.metastore.services.util:GenerateAuxiliaryVersionsConfigFromList
|
|
- googlecloudsdk.command_lib.metastore.validators:ValidateServiceMutexConfig
|
|
- googlecloudsdk.command_lib.metastore.validators:ValidateScheduledBackupConfigs
|
|
- googlecloudsdk.command_lib.metastore.validators:ValidateKmsKeys
|
|
GA:
|
|
api_version: v1
|
|
modify_request_hooks:
|
|
- googlecloudsdk.command_lib.metastore.services.util:UpdateScalingConfigForCreate
|
|
- googlecloudsdk.command_lib.metastore.services.util:GenerateNetworkConfigFromSubnetList
|
|
- googlecloudsdk.command_lib.metastore.services.util:GenerateAuxiliaryVersionsConfigFromList
|
|
- googlecloudsdk.command_lib.metastore.validators:ValidateServiceMutexConfigForV1
|
|
collection: metastore.projects.locations.services
|
|
|
|
arguments:
|
|
resource:
|
|
help_text: |
|
|
Arguments and flags that specify the Dataproc Metastore service
|
|
you want to create.
|
|
spec: !REF googlecloudsdk.command_lib.metastore.resources:service
|
|
params:
|
|
- arg_name: port
|
|
api_field: service.port
|
|
default: 9083
|
|
processor: googlecloudsdk.command_lib.metastore.validators:ValidatePort
|
|
help_text: |
|
|
The TCP port on which the Metastore service will listen.
|
|
If unspecified, the default port 9083 will be used.
|
|
- group:
|
|
help_text: |
|
|
The instance size of the Dataproc Metastore instance.
|
|
mutex: true
|
|
params:
|
|
- arg_name: tier
|
|
api_field: service.tier
|
|
help_text: |
|
|
The tier of the service.
|
|
choices:
|
|
- arg_value: enterprise
|
|
enum_value: ENTERPRISE
|
|
help_text: |
|
|
The enterprise tier provides multi-zone high availability, and sufficient
|
|
scalability for enterprise-level Dataproc Metastore workloads.
|
|
- arg_value: developer
|
|
enum_value: DEVELOPER
|
|
help_text: |
|
|
The developer tier provides limited scalability and no fault tolerance.
|
|
Good for low-cost proof-of-concept.
|
|
fallback: googlecloudsdk.core.properties:VALUES.metastore.tier.Get
|
|
- arg_name: scaling-factor
|
|
api_field: service.scalingConfig.scalingFactor
|
|
help_text: |
|
|
The scaling factor of the service.
|
|
Can be one of [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0].
|
|
processor: googlecloudsdk.command_lib.metastore.validators:ValidateScalingFactor
|
|
- arg_name: instance-size
|
|
api_field: service.scalingConfig.instanceSize
|
|
help_text: |
|
|
The instance size of the service.
|
|
choices:
|
|
- arg_value: extra-small
|
|
enum_value: EXTRA_SMALL
|
|
help_text: |
|
|
Extra small instance size, maps to a scaling factor of 0.1.
|
|
- arg_value: small
|
|
enum_value: SMALL
|
|
help_text: |
|
|
Small instance size, maps to a scaling factor of 0.5.
|
|
- arg_value: medium
|
|
enum_value: MEDIUM
|
|
help_text: |
|
|
Medium instance size, maps to a scaling factor of 1.0.
|
|
- arg_value: large
|
|
enum_value: LARGE
|
|
help_text: |
|
|
Large instance size, maps to a scaling factor of 3.0.
|
|
- arg_value: extra-large
|
|
enum_value: EXTRA_LARGE
|
|
help_text: |
|
|
Extra large instance size, maps to a scaling factor of 6.0.
|
|
- group:
|
|
help_text: |
|
|
The autoscaling configuration of the Dataproc Metastore instance.
|
|
params:
|
|
- arg_name: min-scaling-factor
|
|
api_field: service.scalingConfig.autoscalingConfig.limitConfig.minScalingFactor
|
|
help_text: |
|
|
The minimum scaling factor allowed by the autoscaler.
|
|
Can be one of [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0].
|
|
The default value is 0.1.
|
|
processor: googlecloudsdk.command_lib.metastore.validators:ValidateMinScalingFactor
|
|
- arg_name: max-scaling-factor
|
|
api_field: service.scalingConfig.autoscalingConfig.limitConfig.maxScalingFactor
|
|
help_text: |
|
|
The maximum scaling factor allowed by the autoscaler.
|
|
Can be one of [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0].
|
|
The default value is 6.0.
|
|
processor: googlecloudsdk.command_lib.metastore.validators:ValidateMaxScalingFactor
|
|
- arg_name: autoscaling-enabled
|
|
api_field: service.scalingConfig.autoscalingConfig.autoscalingEnabled
|
|
help_text: |
|
|
A boolean flag to determine whether Dataproc Metastore autoscaling should be enabled,
|
|
false if unspecified.
|
|
|
|
The default minimum and maximum scaling factors are 0.1 and 6.0, respectively.
|
|
|
|
The minimum and maximum scaling factors can be specified using --min-scaling-factor and
|
|
--max-scaling-factor.
|
|
- arg_name: hive-metastore-version
|
|
api_field: service.hiveMetastoreConfig.version
|
|
help_text: |
|
|
The Hive metastore schema version. The supported versions of a location are listed via:
|
|
|
|
{top_command} metastore locations describe
|
|
|
|
If unspecified, the default version chosen by the server will be used.
|
|
- arg_name: data-catalog-sync
|
|
default: false
|
|
api_field: service.metadataIntegration.dataCatalogConfig.enabled
|
|
help_text: |
|
|
A boolean flag to determine whether Dataproc Metastore metadata sync
|
|
to Data Catalog should be enabled, false if unspecified.
|
|
Mutually exclusive with flag `--encryption-kms-key`.
|
|
- arg_name: deletion-protection
|
|
type: bool
|
|
api_field: service.deletionProtection
|
|
help_text: |
|
|
Flag that enables delete protection on Dataproc Metastore instance to prevent
|
|
accidental deletions of the instance.
|
|
Use --deletion-protection to enable.
|
|
- arg_name: release-channel
|
|
help_text: |
|
|
The release channel of the service.
|
|
choices:
|
|
- arg_value: stable
|
|
enum_value: STABLE
|
|
help_text: |
|
|
The `STABLE` release channel contains features that are considered stable
|
|
and have been validated for production use.
|
|
- arg_value: canary
|
|
enum_value: CANARY
|
|
help_text: |
|
|
The `CANARY` release channel contains the newest features, which may be
|
|
unstable and subject to unresolved issues with no known workarounds.
|
|
Services using the `CANARY` release channel are not subject to any SLAs.
|
|
default: stable
|
|
api_field: service.releaseChannel
|
|
- arg_name: database-type
|
|
help_text: |
|
|
The type of database the Dataproc Metastore service will store data in.
|
|
choices:
|
|
- arg_value: mysql
|
|
enum_value: MYSQL
|
|
help_text: |
|
|
`MYSQL` database type is a Dataproc Metastore service backed by MySQL CloudSQL.
|
|
- arg_value: spanner
|
|
enum_value: SPANNER
|
|
help_text: |
|
|
`SPANNER` database type is a Dataproc Metastore service backed by Cloud Spanner.
|
|
default: mysql
|
|
api_field: service.databaseType
|
|
- group:
|
|
help_text: |
|
|
Configuration properties specific to running Hive metastore
|
|
software as the metastore service.
|
|
mutex: true
|
|
params:
|
|
- arg_name: hive-metastore-configs
|
|
api_field: service.hiveMetastoreConfig.configOverrides.additionalProperties
|
|
metavar: KEY=VALUE
|
|
help_text: |
|
|
A mapping of Hive metastore configuration key-value pairs to
|
|
apply to the Hive metastore.
|
|
type:
|
|
arg_dict:
|
|
flatten: true
|
|
spec:
|
|
- api_field: key
|
|
- api_field: value
|
|
- arg_name: hive-metastore-configs-from-file
|
|
api_field: service.hiveMetastoreConfig.configOverrides
|
|
type: "googlecloudsdk.calliope.arg_parsers:FileContents:"
|
|
processor: googlecloudsdk.command_lib.metastore.services.util:LoadHiveMetatsoreConfigsFromXmlFile
|
|
help_text: |
|
|
Path to a XML file containing a mapping of Hive metastore configuration key-value
|
|
pairs to apply to the Hive metastore.
|
|
|
|
For example:
|
|
|
|
hive-site.xml
|
|
<configuration>
|
|
<property>
|
|
<name>hive.metastore.warehouse.dir</name>
|
|
<value>${test.warehouse.dir}</value>
|
|
<description></description>
|
|
</property>
|
|
</configuration>
|
|
- group:
|
|
help_text: |
|
|
Information used to configure the Hive metastore service as a service principal
|
|
in a Kerberos realm.
|
|
params:
|
|
- arg_name: kerberos-principal
|
|
api_field: service.hiveMetastoreConfig.kerberosConfig.principal
|
|
processor: googlecloudsdk.command_lib.metastore.validators:ValidateKerberosPrincipal
|
|
help_text: |
|
|
A Kerberos principal that exists in the KDC to authenticate as. A typical principal
|
|
is of the form "primary/instance@REALM", but there is no exact format.
|
|
required: true
|
|
- arg_name: krb5-config
|
|
api_field: service.hiveMetastoreConfig.kerberosConfig.krb5ConfigGcsUri
|
|
processor: googlecloudsdk.command_lib.metastore.validators:ValidateGcsUri:arg_name=--krb5-config
|
|
help_text: |
|
|
A Cloud Storage URI that specifies the path to a krb5.conf file. It is of the form
|
|
gs://{bucket_name}/path/krb5.conf, although the file does not need to be named
|
|
krb5.conf explicitly.
|
|
required: true
|
|
- arg_name: keytab
|
|
api_field: service.hiveMetastoreConfig.kerberosConfig.keytab.cloudSecret
|
|
processor: googlecloudsdk.command_lib.metastore.parsers:ParseSecretManagerSecretVersion
|
|
help_text: |
|
|
A Kerberos keytab file that can be used to authenticate a service principal
|
|
with a Kerberos Key Distribution Center. This is a Secret Manager secret version,
|
|
and can be fully-qualified URL, or relative name in the form
|
|
`projects/{project_id}/secrets/{secret_id}/versions/{version_id}`.
|
|
required: true
|
|
- group:
|
|
help_text: |
|
|
The one hour maintenance window that specifies when Dataproc
|
|
Metastore may perform system maintenance operation to the service, in
|
|
UTC time.
|
|
params:
|
|
- arg_name: maintenance-window-day
|
|
api_field: service.maintenanceWindow.dayOfWeek
|
|
required: true
|
|
choices:
|
|
- arg_value: mon
|
|
enum_value: MONDAY
|
|
- arg_value: tue
|
|
enum_value: TUESDAY
|
|
- arg_value: wed
|
|
enum_value: WEDNESDAY
|
|
- arg_value: thu
|
|
enum_value: THURSDAY
|
|
- arg_value: fri
|
|
enum_value: FRIDAY
|
|
- arg_value: sat
|
|
enum_value: SATURDAY
|
|
- arg_value: sun
|
|
enum_value: SUNDAY
|
|
help_text: |
|
|
The day of week when the window starts, e.g., `sun`.
|
|
- arg_name: maintenance-window-hour
|
|
api_field: service.maintenanceWindow.hourOfDay
|
|
required: true
|
|
processor: googlecloudsdk.command_lib.metastore.validators:ValidateHourOfDay
|
|
help_text: |
|
|
The hour of day (0-23) when the window starts.
|
|
- group:
|
|
help_text: |
|
|
Encryption configuration for the metastore service.
|
|
mutex: true
|
|
params:
|
|
- arg_name: encryption-kms-keys
|
|
release_tracks: [ALPHA, BETA]
|
|
api_field: service.encryptionConfig.kmsKeys
|
|
help_text: |
|
|
Comma-separated list of the Cloud KMS keys to use for customer data encryption.
|
|
Cannot be changed once the service is created.
|
|
Mutually exclusive with flag `--data-catalog-sync` and `--encryption-kms-key`.
|
|
Each key can be provided as a fully-qualified URL, or a relative name in
|
|
the form `projects/{project_id}/locations/{location}/keyRings/{key_ring_id}/cryptoKeys/{crypto_key_id}`.
|
|
type: "arg_list"
|
|
- arg_name: encryption-kms-key
|
|
api_field: service.encryptionConfig.kmsKey
|
|
processor: googlecloudsdk.command_lib.metastore.parsers:ParseCloudKmsKey
|
|
help_text: |
|
|
The name of the Cloud KMS key to use for customer data encryption.
|
|
Cannot be changed once the service is created.
|
|
Mutually exclusive with flag `--data-catalog-sync`.
|
|
This can be provided as a fully-qualified URL, or a relative name in
|
|
the form `projects/{project_id}/locations/{location}/keyRings/{key_ring_id}/cryptoKeys/{crypto_key_id}`.
|
|
- group:
|
|
help_text: |
|
|
Network configuration for the metastore service.
|
|
mutex: true
|
|
params:
|
|
- arg_name: network
|
|
api_field: service.network
|
|
processor: googlecloudsdk.command_lib.metastore.parsers:ParseNetwork
|
|
help_text: |
|
|
The name of the network on which the service can be accessed.
|
|
This can be the network's ID, fully-qualified URL, or relative name in
|
|
the form `projects/{project_id}/global/networks/{network}`.
|
|
If "network", "consumer-subnetworks" and "network-config" are unspecified,
|
|
the "default" network will be used.
|
|
- arg_name: consumer-subnetworks
|
|
help_text: |
|
|
The list of subnetworks from which the service can be accessed.
|
|
This can be the subnetwork's ID, fully-qualified URL, or relative name in
|
|
the form `projects/{project_id}/regions/{region_id}/subnetworks/{subnetwork}`.
|
|
type: "arg_list"
|
|
- arg_name: network-config-from-file
|
|
api_field: service.networkConfig
|
|
type: "googlecloudsdk.calliope.arg_parsers:YAMLFileContents:"
|
|
help_text: |
|
|
Path to a YAML file containing the network configuration for Dataproc Metastore instance.
|
|
The contents of the file should be structured as follows:
|
|
|
|
YAML:
|
|
consumers:
|
|
- subnetwork: projects/{project_id}/regions/{region_id}/subnetworks/{subnetwork_id}
|
|
- subnetwork: projects/{project_id}/regions/{region_id}/subnetworks/{subnetwork_id}
|
|
...
|
|
- group:
|
|
help_text: |
|
|
Auxiliary versions configuration for the Dataproc Metastore service. When specified,
|
|
a secondary Hive metastore service is created along with the primary service.
|
|
mutex: true
|
|
params:
|
|
- arg_name: auxiliary-versions
|
|
type: "arg_list"
|
|
help_text: |
|
|
Comma-separated list of auxiliary Hive metastore versions to deploy. Auxiliary Hive
|
|
metastore versions must be less than the primary Hive metastore service's version.
|
|
- arg_name: auxiliary-versions-from-file
|
|
api_field: service.hiveMetastoreConfig.auxiliaryVersions
|
|
type: "googlecloudsdk.calliope.arg_parsers:YAMLFileContents:"
|
|
processor: googlecloudsdk.command_lib.metastore.services.util:LoadAuxiliaryVersionsConfigsFromYamlFile
|
|
help_text: |
|
|
Path to a YAML file containing the auxiliary versions configuration for
|
|
Dataproc Metastore instance. The file should contain a unique auxiliary service name and
|
|
auxiliary version that is lower than the primary service version. The primary version's
|
|
Hive metastore configs are applied to the auxiliary version. Additional Hive metastore
|
|
configs can be specified for the auxiliary version using "config_overrides". If a
|
|
specified config property has already been overridden in the primary Hive metastore
|
|
version's configs, the auxiliary version's override takes precedence.
|
|
The contents of the file should be structured as follows:
|
|
|
|
YAML:
|
|
|
|
```
|
|
- name: aux-service1
|
|
version: x.y.z
|
|
config_overrides:
|
|
key1: value1
|
|
key2: value2
|
|
...
|
|
- name: aux-service2
|
|
version: x.y.z
|
|
config_overrides:
|
|
key1: value1
|
|
key2: value2
|
|
...
|
|
...
|
|
```
|
|
- arg_name: endpoint-protocol
|
|
help_text: |
|
|
The protocol to use for the metastore service endpoint.
|
|
If unspecified, defaults to `THRIFT`.
|
|
choices:
|
|
- arg_value: thrift
|
|
enum_value: THRIFT
|
|
help_text: |
|
|
The legacy Apache `THRIFT` protocol.
|
|
- arg_value: grpc
|
|
enum_value: GRPC
|
|
help_text: |
|
|
The modernized `GRPC` protocol.
|
|
default: thrift
|
|
api_field: service.hiveMetastoreConfig.endpointProtocol
|
|
- group:
|
|
release_tracks: [ALPHA, BETA]
|
|
help_text: |
|
|
Custom region configuration for the metastore service.
|
|
params:
|
|
- arg_name: read-write-regions
|
|
api_field: service.multiRegionConfig.customRegionConfig.readWriteRegions
|
|
help_text: |
|
|
The list of read-write regions where the metastore service runs in. These regions
|
|
should be part (or subset) of the multi-region.
|
|
type: "arg_list"
|
|
- arg_name: read-only-regions
|
|
api_field: service.multiRegionConfig.customRegionConfig.readOnlyRegions
|
|
help_text: |
|
|
The list of read-only regions where the metastore service runs in. These regions
|
|
should be part (or subset) of the multi-region.
|
|
type: "arg_list"
|
|
- group:
|
|
mutex: true
|
|
help_text: |
|
|
Scheduled backup configurations to enable a scheduled backup of the metastore service.
|
|
params:
|
|
- arg_name: scheduled-backup-configs-from-file
|
|
api_field: service.scheduledBackup
|
|
type: "googlecloudsdk.calliope.arg_parsers:FileContents:"
|
|
processor: googlecloudsdk.command_lib.metastore.services.util:LoadScheduledBackupConfigsFromJsonFile
|
|
help_text: |
|
|
Path to a JSON file containing a mapping of scheduled backup configurations key-value pairs to apply to the scheduled backup.
|
|
|
|
For example:
|
|
|
|
backup_configs.json
|
|
{
|
|
"enabled": true,
|
|
"cron_schedule": "0 * * * *",
|
|
"backup_location": "gs://example-bucket",
|
|
"time_zone": "UTC"
|
|
}
|
|
- group:
|
|
help_text: |
|
|
Scheduled backup configurations to enable a scheduled backup of the metastore service.
|
|
params:
|
|
- arg_name: enable-scheduled-backup
|
|
api_field: service.scheduledBackup.enabled
|
|
action: store_true
|
|
help_text: |
|
|
Enable or disable scheduled backups for the Metastore service.
|
|
- arg_name: scheduled-backup-cron
|
|
api_field: service.scheduledBackup.cronSchedule
|
|
help_text: |
|
|
Specify the frequency of a scheduled backup of the metastore service.
|
|
- arg_name: scheduled-backup-location
|
|
api_field: service.scheduledBackup.backupLocation
|
|
help_text: |
|
|
Set the Cloud Storage URI to store the scheduled backups of the metastore service.
|
|
- arg_name: tags
|
|
api_field: service.tags.additionalProperties
|
|
metavar: KEY=VALUE
|
|
help_text: |
|
|
List of tag KEY=VALUE pairs to add.
|
|
type:
|
|
arg_dict:
|
|
flatten: true
|
|
spec:
|
|
- api_field: key
|
|
- api_field: value
|
|
- group:
|
|
release_tracks: [ALPHA, BETA]
|
|
hidden: true
|
|
help_text: |
|
|
Configuration required for migrating to BigQuery Metastore service.
|
|
params:
|
|
- arg_name: bigquery-project-id
|
|
api_field: service.bigqueryMetastoreMigrationConfig.bigqueryProjectId
|
|
help_text: |
|
|
Project ID where the BigQuery resources (e.g. datasets, tables, etc.)
|
|
should be created.
|
|
required: true
|
|
- arg_name: bigquery-dataset-location
|
|
api_field: service.bigqueryMetastoreMigrationConfig.bigqueryDatasetLocation
|
|
help_text: |
|
|
The location where the BigQuery resources (e.g. datasets, tables, etc.)
|
|
should be created (e.g. us-central1, us, eu, etc.).
|
|
required: true
|
|
- arg_name: desired-migration-state
|
|
api_field: service.bigqueryMetastoreMigrationConfig.desiredMigrationState
|
|
help_text: |
|
|
The desired state of the migration. Note that this also reflects the
|
|
current state of the migration. If an attempt to update to a new desired
|
|
state fails, the migration will revert to the previous state.
|
|
choices:
|
|
- arg_value: migrate
|
|
enum_value: MIGRATE
|
|
help_text: |
|
|
By setting the desired migration state to `MIGRATE`, metadata updates in
|
|
Dataproc Metastore will be replicated to the BigQuery Metastore service,
|
|
ensuring that it remains consistently synchronized with Dataproc
|
|
Metastore. Note that this includes initial backfill of existing metadata.
|
|
- arg_value: cutover
|
|
enum_value: CUTOVER
|
|
help_text: |
|
|
By setting the desired migration state to `CUTOVER`, all metadata
|
|
requests are routed to BigQuery Metastore service and Dataproc Metastore
|
|
only functions as a proxy. This state can be considered as the completion
|
|
of the migration.
|
|
- arg_value: cancel
|
|
enum_value: CANCEL
|
|
help_text: |
|
|
By setting the desired migration state to `CANCEL`, the migration is
|
|
effectively cancelled. If the previous migration state was `MIGRATE`,
|
|
then replication to BigQuery Metastore will be cancelled. If the previous
|
|
state was `CUTOVER`, then metadata requests will now be served from
|
|
Dataproc Metastore instead of BigQuery Metastore. Note that existing
|
|
metadata changes replicated to BigQuery Metastore service are not rolled
|
|
back.
|
|
required: true
|
|
labels:
|
|
api_field: service.labels
|
|
|
|
async:
|
|
collection: metastore.projects.locations.operations
|