davidkotnik/novafarma
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Add new gcloud commands, API clients, and third-party libraries across various services.

Browse Source
This commit is contained in:
David Kotnik
2026-01-01 20:26:35 +01:00
parent 5e23cbece0
commit a19e592eb7
25221 changed files with 8324611 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
gcloud auth application-default login/google-cloud-sdk/lib/surface/dataplex/tasks/__init__.py Normal file
View File

@@ -0,0 +1,28 @@
# -*- coding: utf-8 -*- #
# Copyright 2021 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Command group for Dataplex Task services."""
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from googlecloudsdk.calliope import base
@base.ReleaseTracks(base.ReleaseTrack.ALPHA, base.ReleaseTrack.GA)
class Tasks(base.Group):
"""Manage Dataplex Task services."""
category = base.DATA_ANALYTICS_CATEGORY

58
gcloud auth application-default login/google-cloud-sdk/lib/surface/dataplex/tasks/add_iam_policy_binding.py Normal file
View File

@@ -0,0 +1,58 @@
# -*- coding: utf-8 -*- #
# Copyright 2022 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Command to add-iam-policy-binding to a Dataplex task resource."""
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from googlecloudsdk.api_lib.dataplex import task
from googlecloudsdk.api_lib.util import exceptions as gcloud_exception
from googlecloudsdk.calliope import base
from googlecloudsdk.command_lib.dataplex import resource_args
from googlecloudsdk.command_lib.iam import iam_util
@base.ReleaseTracks(base.ReleaseTrack.ALPHA, base.ReleaseTrack.GA)
class AddIamPolicyBinding(base.Command):
"""Add IAM policy binding to a Dataplex task resource."""
detailed_help = {
'EXAMPLES':
"""\
To add an IAM policy binding for the role of `roles/dataplex.viewer`
for the user 'testuser@gmail.com' to task `test-task` within lake
`test-lake` in location `us-central`, run:
$ {command} test-task --location=us-central1 --lake=test-lake \
--role=roles/dataplex.viewer --member=user:testuser@gmail.com
See https://cloud.google.com/dataplex/docs/iam-roles for details of
policy role and member types.
""",
}
@staticmethod
def Args(parser):
resource_args.AddTaskResourceArg(parser, 'to add IAM policy binding to.')
iam_util.AddArgsForAddIamPolicyBinding(parser)
@gcloud_exception.CatchHTTPErrorRaiseHTTPException(
'Status code: {status_code}. {status_message}.')
def Run(self, args):
task_ref = args.CONCEPTS.task.Parse()
result = task.AddIamPolicyBinding(task_ref, args.member, args.role)
return result

380
gcloud auth application-default login/google-cloud-sdk/lib/surface/dataplex/tasks/create.yaml Normal file
View File

@@ -0,0 +1,380 @@
- release_tracks: [ALPHA, GA]
help_text:
brief: |
Create a Dataplex task resource.
description: |
Create a Dataplex task resource.
A task represents a user visible job that you want Dataplex to perform on a schedule. It
encapsulates your code, your parameters and the schedule.
This task ID must follow these rules:
o Must contain only lowercase letters, numbers, and hyphens.
o Must start with a letter.
o Must end with a number or a letter.
o Must be between 1-63 characters.
o Must be unique within the customer project / location.
examples: |
To create a Dataplex task `test-task` with ON_DEMAND trigger type,
`dataplex-demo-test@test-project.iam.gserviceaccount.com` as execution service account and
`gs://test-bucket/test-file.py` as spark python script file within lake `test-lake` in location `us-central1`.
$ {command} test-task --location=us-central1 --lake=test-lake \
--execution-service-account=dataplex-demo-test@test-project.iam.gserviceaccount.com \
--spark-python-script-file=gs://test-bucket/test-file.py \
--trigger-type=ON_DEMAND
To create a Dataplex task `test-task` with RECURRING trigger type starting every
hour at minute 0, `dataplex-demo-test@test-project.iam.gserviceaccount.com` as execution
service account and `gs://test-bucket/test-file.py` as spark python script file within lake `test-lake` in
location `us-central1`.
$ {command} test-task --location=us-central1 --lake=test-lake \
--execution-service-account=dataplex-demo-test@test-project.iam.gserviceaccount.com \
--spark-python-script-file=gs://test-bucket/test-file.py \
--trigger-type=RECURRING --trigger-schedule="0 * * * *"
request:
ALPHA:
api_version: v1
collection: dataplex.projects.locations.lakes.tasks
arguments:
resource:
help_text: |
Arguments and flags that specify the Dataplex Task you want to create.
spec: !REF googlecloudsdk.command_lib.dataplex.resources:task
params:
- arg_name: description
api_field: googleCloudDataplexV1Task.description
help_text: |
Description of the Dataplex task.
- arg_name: display-name
api_field: googleCloudDataplexV1Task.displayName
help_text: |
Display name of the Dataplex task.
- group: # trigger_spec
help_text: |
Spec related to Dataplex task scheduling and frequency settings.
required: true
params:
- arg_name: trigger-type
api_field: googleCloudDataplexV1Task.triggerSpec.type
required: true
help_text: |
Trigger type of the user-specified Dataplex Task.
choices:
- arg_value: on-demand
enum_value: ON_DEMAND
help_text: |
The `ON_DEMAND` trigger type runs the Dataplex task one time shortly after task creation.
- arg_value: recurring
enum_value: RECURRING
help_text: |
The `RECURRING` trigger type makes the task scheduled to run periodically.
- arg_name: trigger-start-time
api_field: googleCloudDataplexV1Task.triggerSpec.startTime
help_text: |
The first run of the task begins after this time. If not specified, an ON_DEMAND
task runs when it is submitted and a RECURRING task runs based on the trigger
schedule.
- arg_name: trigger-disabled
api_field: googleCloudDataplexV1Task.triggerSpec.disabled
type: bool
default: false
help_text: |
Prevent the task from executing.
This does not cancel already running tasks. It is intended to temporarily
disable RECURRING tasks.
- arg_name: trigger-max-retires
type: int
api_field: googleCloudDataplexV1Task.triggerSpec.maxRetries
help_text: |
Number of retry attempts before aborting.
Set to zero to never attempt to retry a failed task.
- arg_name: trigger-schedule
api_field: googleCloudDataplexV1Task.triggerSpec.schedule
help_text: |
Cron schedule (https://en.wikipedia.org/wiki/Cron) for running
tasks periodically.
- group: # execution_spec
help_text: |
Spec related to how a task is executed.
required: true
params:
- arg_name: execution-service-account
api_field: googleCloudDataplexV1Task.executionSpec.serviceAccount
required: true
help_text: |
Service account to use to execute a task.
- arg_name: execution-project
api_field: googleCloudDataplexV1Task.executionSpec.project
help_text: |
The project in which jobs are run.
By default, the project containing the Lake is used.
If a project is provided, the --execution-service-account must belong to this same
project.
- arg_name: execution-args
api_field: googleCloudDataplexV1Task.executionSpec.args.additionalProperties
metavar: KEY=VALUE
type:
arg_dict:
flatten: true
spec:
- api_field: key
- api_field: value
help_text: |
The arguments to pass to the task.
The args can use placeholders of the format ${placeholder} as
part of key/value string. These will be interpolated before passing the
args to the driver. Currently supported placeholders:
- ${task_id}
- ${job_time}
To pass positional args, set the key as TASK_ARGS. The value should be a
comma-separated string of all the positional arguments.
See https://cloud.google.com/sdk/gcloud/reference/topic/escaping for details on
using a delimiter other than a comma. In case of
other keys being present in the args, then TASK_ARGS will be passed as
the last argument.
- arg_name: max-job-execution-lifetime
api_field: googleCloudDataplexV1Task.executionSpec.maxJobExecutionLifetime
help_text: |
The maximum duration before the job execution expires.
- arg_name: kms-key
api_field: googleCloudDataplexV1Task.executionSpec.kmsKey
help_text: |
The Cloud KMS key to use for encryption, of the form:
projects/{project_number}/locations/{location_id}/keyRings/{key-ring-name}/cryptoKeys/{key-name}
- group: # task_config
help_text: |
Select which task you want to schedule and provide the required arguments for the task. The 2 types of tasks supported are:-
- spark tasks
- notebook tasks
mutex: true
required: true
params:
- group: # notebook_task_config
help_text: |
Config related to running custom notebook tasks.
params:
- arg_name: notebook
required: true
api_field: googleCloudDataplexV1Task.notebook.notebook
help_text: |
Path to input notebook. This can be the Google Cloud Storage URI of the notebook file
or the path to a Notebook Content.
The execution args are accessible as environment variables
(`TASK_key=value`).
- arg_name: notebook-file-uris
repeated: true
api_field: googleCloudDataplexV1Task.notebook.fileUris
help_text: |
Google Cloud Storage URIs of files to be placed in the working directory of each
executor.
- arg_name: notebook-archive-uris
repeated: true
api_field: googleCloudDataplexV1Task.notebook.archiveUris
help_text: |
Google Cloud Storage URIs of archives to be extracted into the working directory of
each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and
.zip.
- group: # notebook_infrastructure_spec
params:
- group: # notebook_batch_compute_resource
help_text: |
Compute resources needed for a Task when using Dataproc Serverless.
params:
- arg_name: notebook-batch-executors-count
api_field: googleCloudDataplexV1Task.notebook.infrastructureSpec.batch.executorsCount
type: int
help_text: |
Total number of job executors.
- arg_name: notebook-batch-max-executors-count
api_field: googleCloudDataplexV1Task.notebook.infrastructureSpec.batch.maxExecutorsCount
type: int
help_text: |
Max configurable executors.
If max_executors_count > executors_count, then auto-scaling is enabled.
- group: # notebook_container_image_runtime
help_text: |
Container Image Runtime Configuration.
params:
- arg_name: notebook-container-image
api_field: googleCloudDataplexV1Task.notebook.infrastructureSpec.containerImage.image
help_text: |
Optional custom container image for the job.
- arg_name: notebook-container-image-java-jars
api_field: googleCloudDataplexV1Task.notebook.infrastructureSpec.containerImage.javaJars
repeated: true
help_text: |
A list of Java JARS to add to the classpath.
Valid input includes Cloud Storage URIs to Jar binaries.
For example, gs://bucket-name/my/path/to/file.jar
- arg_name: notebook-container-image-properties
api_field: googleCloudDataplexV1Task.notebook.infrastructureSpec.containerImage.properties.additionalProperties
metavar: KEY=VALUE
type:
arg_dict:
flatten: true
spec:
- api_field: key
- api_field: value
help_text: |
The properties to set on daemon config files. Property keys are
specified in prefix:property format, for example core:hadoop.tmp.dir.
For more information, see Cluster properties
(https://cloud.google.com/dataproc/docs/concepts/cluster-properties)
- group: # notebook_vpc_network
help_text: |
Cloud VPC Network used to run the infrastructure.
params:
- group:
mutex: true
help_text: |
The Cloud VPC network identifier.
params:
- arg_name: notebook-vpc-network-name
api_field: googleCloudDataplexV1Task.notebook.infrastructureSpec.vpcNetwork.network
help_text: |
The Cloud VPC network in which the job is run. By default, the Cloud
VPC network named Default within the project is used.
- arg_name: notebook-vpc-sub-network-name
api_field: googleCloudDataplexV1Task.notebook.infrastructureSpec.vpcNetwork.subNetwork
help_text: |
The Cloud VPC sub-network in which the job is run.
- arg_name: notebook-vpc-network-tags
api_field: googleCloudDataplexV1Task.notebook.infrastructureSpec.vpcNetwork.networkTags
repeated: true
help_text: |
List of network tags to apply to the job.
- group: # spark_task_config
help_text: |
Config related to running custom Spark tasks.
params:
- group: # driver
required: true
mutex: true
help_text: |
The specification of the main method to call to drive the
job. Specify either the jar file that contains the main class or the
main class name.
params:
- arg_name: spark-main-jar-file-uri
api_field: googleCloudDataplexV1Task.spark.mainJarFileUri
help_text: |
The Google Cloud Storage URI of the jar file that contains the main class.
The execution args are passed in as a sequence of named process
arguments (`--key=value`).
- arg_name: spark-main-class
api_field: googleCloudDataplexV1Task.spark.mainClass
help_text: |
The name of the driver's main class. The jar file that contains the
class must be in the default CLASSPATH or specified in `jar_file_uris`.
The execution args are passed in as a sequence of named process
arguments (`--key=value`).
- arg_name: spark-python-script-file
api_field: googleCloudDataplexV1Task.spark.pythonScriptFile
help_text: |
The Google Cloud Storage URI of the main Python file to use as the driver. Must
be a .py file.
- arg_name: spark-sql-script-file
api_field: googleCloudDataplexV1Task.spark.sqlScriptFile
help_text: |
A reference to a query file. This can be the Google Cloud Storage URI of the query file
or it can the path to a SqlScript Content.
- arg_name: spark-sql-script
api_field: googleCloudDataplexV1Task.spark.sqlScript
help_text: |
The SQL query text.
- arg_name: spark-file-uris
repeated: true
api_field: googleCloudDataplexV1Task.spark.fileUris
help_text: |
Google Cloud Storage URIs of files to be placed in the working directory of each
executor.
- arg_name: spark-archive-uris
repeated: true
api_field: googleCloudDataplexV1Task.spark.archiveUris
help_text: |
Google Cloud Storage URIs of archives to be extracted into the working directory of
each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and
.zip.
- group: # infrastructure_spec
params:
- group: # batch_compute_resource
help_text: |
Compute resources needed for a Task when using Dataproc Serverless.
params:
- arg_name: batch-executors-count
api_field: googleCloudDataplexV1Task.spark.infrastructureSpec.batch.executorsCount
type: int
help_text: |
Total number of job executors.
- arg_name: batch-max-executors-count
api_field: googleCloudDataplexV1Task.spark.infrastructureSpec.batch.maxExecutorsCount
type: int
help_text: |
Max configurable executors.
If max_executors_count > executors_count, then auto-scaling is enabled.
- group: # container_image_runtime
help_text: |
Container Image Runtime Configuration.
params:
- arg_name: container-image
api_field: googleCloudDataplexV1Task.spark.infrastructureSpec.containerImage.image
help_text: |
Optional custom container image for the job.
- arg_name: container-image-java-jars
api_field: googleCloudDataplexV1Task.spark.infrastructureSpec.containerImage.javaJars
repeated: true
help_text: |
A list of Java JARS to add to the classpath.
Valid input includes Cloud Storage URIs to Jar binaries.
For example, gs://bucket-name/my/path/to/file.jar
- arg_name: container-image-python-packages
api_field: googleCloudDataplexV1Task.spark.infrastructureSpec.containerImage.pythonPackages
repeated: true
help_text: |
A list of python packages to be installed.
Valid formats include Cloud Storage URI to a PIP installable library.
For example, gs://bucket-name/my/path/to/lib.tar.gz
- arg_name: container-image-properties
api_field: googleCloudDataplexV1Task.spark.infrastructureSpec.containerImage.properties.additionalProperties
metavar: KEY=VALUE
type:
arg_dict:
flatten: true
spec:
- api_field: key
- api_field: value
help_text: |
The properties to set on daemon config files. Property keys are
specified in prefix:property format, for example core:hadoop.tmp.dir.
For more information, see Cluster properties
(https://cloud.google.com/dataproc/docs/concepts/cluster-properties)
- group: # vpc-network
help_text: |
Cloud VPC Network used to run the infrastructure.
params:
- group:
mutex: true
help_text: |
The Cloud VPC network identifier.
params:
- arg_name: vpc-network-name
api_field: googleCloudDataplexV1Task.spark.infrastructureSpec.vpcNetwork.network
help_text: |
The Cloud VPC network in which the job is run. By default, the Cloud
VPC network named Default within the project is used.
- arg_name: vpc-sub-network-name
api_field: googleCloudDataplexV1Task.spark.infrastructureSpec.vpcNetwork.subNetwork
help_text: |
The Cloud VPC sub-network in which the job is run.
- arg_name: vpc-network-tags
api_field: googleCloudDataplexV1Task.spark.infrastructureSpec.vpcNetwork.networkTags
repeated: true
help_text: |
List of network tags to apply to the job.
labels:
api_field: googleCloudDataplexV1Task.labels
async:
collection: dataplex.projects.locations.operations

22
gcloud auth application-default login/google-cloud-sdk/lib/surface/dataplex/tasks/delete.yaml Normal file
View File

@@ -0,0 +1,22 @@
- release_tracks: [ALPHA, GA]
help_text:
brief: |
Delete a Dataplex task resource.
description: |
Delete a Dataplex task resource.
examples: |
To delete a Dataplex task `test-task` within lake `test-lake` in location `us-central1`, run:
$ {command} test-task --location=us-central1 --lake=test-lake
request:
ALPHA:
api_version: v1
collection: dataplex.projects.locations.lakes.tasks
arguments:
resource:
help_text: |
Arguments and flags that define the Dataplex Task you want to delete.
spec: !REF googlecloudsdk.command_lib.dataplex.resources:task
async:
collection: dataplex.projects.locations.operations

23
gcloud auth application-default login/google-cloud-sdk/lib/surface/dataplex/tasks/describe.yaml Normal file
View File

@@ -0,0 +1,23 @@
- release_tracks: [ALPHA, GA]
help_text:
brief: |
Describe a Dataplex task resource.
description: |
Describe a Dataplex task resource.
Displays all details of a Dataplex task resource given a valid task ID.
examples: |
To describe a Dataplex task `test-task` within lake `test-lake` in location `us-central1`,
run:
$ {command} test-task --location=us-central1 --lake=test-lake
request:
ALPHA:
api_version: v1
collection: dataplex.projects.locations.lakes.tasks
method: get
arguments:
resource:
help_text: |
Arguments and flags that define the Dataplex Tasks you want to retrieve.
spec: !REF googlecloudsdk.command_lib.dataplex.resources:task

22
gcloud auth application-default login/google-cloud-sdk/lib/surface/dataplex/tasks/get_iam_policy.yaml Normal file
View File

@@ -0,0 +1,22 @@
- release_tracks: [ALPHA, GA]
help_text:
brief: |
Get the IAM policy for a Dataplex task resource.
description: |
Displays the IAM policy associated with a Dataplex task resource.
If formatted as JSON, the output can be edited and used as
a policy file for *set-iam-policy*. The output includes an "etag"
field identifying the version emitted and allowing detection of
concurrent policy updates.
examples: |
To print the IAM policy for Dataplex lake `test-task` within lake `test-lake` in location
`us-central1`, run:
$ {command} test-task --location=us-central1 --lake=test-lake
request:
collection: dataplex.projects.locations.lakes.tasks
arguments:
resource:
help_text: |
Arguments and flags that define the Dataplex Task IAM policy you want to retrieve.
spec: !REF googlecloudsdk.command_lib.dataplex.resources:task

28
gcloud auth application-default login/google-cloud-sdk/lib/surface/dataplex/tasks/jobs/__init__.py Normal file
View File

@@ -0,0 +1,28 @@
# -*- coding: utf-8 -*- #
# Copyright 2021 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Command group for Dataplex Jobs services."""
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from googlecloudsdk.calliope import base
@base.ReleaseTracks(base.ReleaseTrack.ALPHA, base.ReleaseTrack.GA)
class Jobs(base.Group):
"""Manage Dataplex Jobs services."""
category = base.DATA_ANALYTICS_CATEGORY

23
gcloud auth application-default login/google-cloud-sdk/lib/surface/dataplex/tasks/jobs/cancel.yaml Normal file
View File

@@ -0,0 +1,23 @@
- release_tracks: [ALPHA, GA]
help_text:
brief: |
Cancel a Dataplex Job running a particular task.
description: |
Cancel the operation that the given Dataplex Job is running task under a specific project,
location, lake and task.
examples: |
To cancel an job `test-job` running task `test-task` within lake `test-lake` in location
`us-central1`, run:
$ {command} test-job --location=us-central1 --lake=test-lake --task=test-task
request:
ALPHA:
api_version: v1
collection: dataplex.projects.locations.lakes.tasks.jobs
method: cancel
arguments:
resource:
help_text: |
Arguments and flags that define the Dataplex Job running a particular Task you want to cancel.
spec: !REF googlecloudsdk.command_lib.dataplex.resources:job

23
gcloud auth application-default login/google-cloud-sdk/lib/surface/dataplex/tasks/jobs/describe.yaml Normal file
View File

@@ -0,0 +1,23 @@
- release_tracks: [ALPHA, GA]
help_text:
brief: |
Describe a Dataplex job running a particular task.
description: |
Describe a Dataplex job running a particular task.
Displays all details of a Dataplex job given a valid job ID.
examples: |
To describe a Dataplex job `test-job` running a task `test-task` within lake `test-lake` in
location `us-central1`, run:
$ {command} test-job --location=us-central1 --lake=test-lake --task=test-task
request:
ALPHA:
api_version: v1
collection: dataplex.projects.locations.lakes.tasks.jobs
method: get
arguments:
resource:
help_text: |
Arguments and flags that define the Dataplex Job running a particular Task you want to retrieve.
spec: !REF googlecloudsdk.command_lib.dataplex.resources:job

34
gcloud auth application-default login/google-cloud-sdk/lib/surface/dataplex/tasks/jobs/list.yaml Normal file
View File

@@ -0,0 +1,34 @@
- release_tracks: [ALPHA, GA]
help_text:
brief: |
List job runs of a Dataplex task resource.
description: |
List Jobs runs of a Task under a specific project, location, lake and task.
examples: |
To list all the Dataplex job runs for a task `test-task` within lake `test-lake` in location
`us-central1`, run:
{command} --location=us-central1 --lake=test-lake --task=test-task
request:
ALPHA:
api_version: v1
collection: dataplex.projects.locations.lakes.tasks.jobs
arguments:
resource:
help_text: |
Arguments and flags that define the Dataplex Task to list the Jobs running the Task.
spec: !REF googlecloudsdk.command_lib.dataplex.resources:task
response:
id_field: name
output:
format: |
table(
name.basename():label=JOB_ID,
state:label=JOB_STATUS,
service:label=SERVICE,
startTime:label=START_TIME,
endTime:label=END_TIME
)

35
gcloud auth application-default login/google-cloud-sdk/lib/surface/dataplex/tasks/list.yaml Normal file
View File

@@ -0,0 +1,35 @@
- release_tracks: [ALPHA, GA]
help_text:
brief: |
List Dataplex task resources under a lake.
description: |
List Dataplex Task resources under a specific project, location, and lake.
examples: |
To list all Dataplex Task resources under the lake `test-lake` within project `test-project`
in location `us-central1`, run:
$ {command} --project=test-project --location=us-central1 --lake=test-lake
request:
ALPHA:
api_version: v1
collection: dataplex.projects.locations.lakes.tasks
arguments:
resource:
help_text: |
Arguments and flags that define the Dataplex Lake to list Tasks under it.
spec: !REF googlecloudsdk.command_lib.dataplex.resources:lake
response:
id_field: name
output:
format: |
table(
name.basename():label=NAME,
displayName:label=DISPLAY_NAME,
createTime:label=CREATE_TIME,
triggerSpec.type:label=TRIGGER_TYPE,
triggerSpec.schedule:label=TRIGGER_SCHEDULE,
labels:label=LABELS
)

59
gcloud auth application-default login/google-cloud-sdk/lib/surface/dataplex/tasks/remove_iam_policy_binding.py Normal file
View File

@@ -0,0 +1,59 @@
# -*- coding: utf-8 -*- #
# Copyright 2022 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Command to remove-iam-policy-binding from a Dataplex task resource."""
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from googlecloudsdk.api_lib.dataplex import task
from googlecloudsdk.api_lib.util import exceptions as gcloud_exception
from googlecloudsdk.calliope import base
from googlecloudsdk.command_lib.dataplex import resource_args
from googlecloudsdk.command_lib.iam import iam_util
@base.ReleaseTracks(base.ReleaseTrack.ALPHA, base.ReleaseTrack.GA)
class RemoveIamPolicyBinding(base.Command):
"""Remove IAM policy binding from a Dataplex task resource."""
detailed_help = {
'EXAMPLES':
"""\
To remove an IAM policy binding for the role `roles/dataplex.viewer`
for the user `testuser@gmail.com` from a task `test-task` within lake
`test-lake` in location `us-central1`, run:
$ {command} test-task --location=us-central1 --lake=test-lake \
--role=roles/dataplex.viewer --member=user:testuser@gmail.com
See https://cloud.google.com/dataplex/docs/iam-roles for details of
policy role and member types.
""",
}
@staticmethod
def Args(parser):
resource_args.AddTaskResourceArg(parser,
'to remove IAM policy binding from.')
iam_util.AddArgsForRemoveIamPolicyBinding(parser)
@gcloud_exception.CatchHTTPErrorRaiseHTTPException(
'Status code: {status_code}. {status_message}.')
def Run(self, args):
task_ref = args.CONCEPTS.task.Parse()
result = task.RemoveIamPolicyBinding(task_ref, args.member, args.role)
return result

66
gcloud auth application-default login/google-cloud-sdk/lib/surface/dataplex/tasks/run.py Normal file
View File

@@ -0,0 +1,66 @@
# -*- coding: utf-8 -*- #
# Copyright 2023 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Command to trigger one-time run of a Dataplex task."""
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
import argparse
from googlecloudsdk.api_lib.dataplex import task
from googlecloudsdk.api_lib.util import exceptions as gcloud_exception
from googlecloudsdk.calliope import arg_parsers
from googlecloudsdk.calliope import base
from googlecloudsdk.command_lib.dataplex import resource_args
from googlecloudsdk.command_lib.util.args import labels_util
@base.ReleaseTracks(base.ReleaseTrack.ALPHA, base.ReleaseTrack.GA)
class Run(base.Command):
"""Trigger one-time run of a Dataplex task."""
detailed_help = {
'EXAMPLES': """\
To trigger a one-time run of a Dataplex task `test-task` within
lake `test-lake` in location `us-central1`, run:
$ {command} test-task --location=us-central1 --lake=test-lake
""",
}
@staticmethod
def Args(parser):
resource_args.AddTaskResourceArg(parser, 'to run.')
parser.add_argument(
'ARGS',
metavar='execution-spec-args',
nargs=argparse.REMAINDER,
default=[],
help=(
'Execution spec arguments to pass to the driver. Follows the format'
' argKey=argValue.'
),
type=arg_parsers.ArgList(),
)
labels_util.AddCreateLabelsFlags(parser)
@gcloud_exception.CatchHTTPErrorRaiseHTTPException(
'Status code: {status_code}. {status_message}.'
)
def Run(self, args):
task_ref = args.CONCEPTS.task.Parse()
run_task_response = task.RunTask(task_ref, args)
return run_task_response

57
gcloud auth application-default login/google-cloud-sdk/lib/surface/dataplex/tasks/set_iam_policy.py Normal file
View File

@@ -0,0 +1,57 @@
# -*- coding: utf-8 -*- #
# Copyright 2022 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Command to set-iam-policy for a Dataplex task resource."""
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from googlecloudsdk.api_lib.dataplex import task
from googlecloudsdk.api_lib.util import exceptions as gcloud_exception
from googlecloudsdk.calliope import base
from googlecloudsdk.command_lib.dataplex import resource_args
from googlecloudsdk.command_lib.iam import iam_util
@base.ReleaseTracks(base.ReleaseTrack.ALPHA, base.ReleaseTrack.GA)
class SetIamPolicy(base.Command):
"""Set the IAM policy to a Dataplex task as defined in a JSON or YAML file.
See https://cloud.google.com/iam/docs/managing-policies for details of
the policy file format and contents.
"""
detailed_help = {
'EXAMPLES':
"""\
The following command will read an IAM policy defined in a JSON file
`policy.json` and set it for the Dataplex task `test-task` within
lake `test-lake` in location `us-central1`:
$ {command} --lake=test-lake --location=us-central1 test-task policy.json
""",
}
@staticmethod
def Args(parser):
resource_args.AddTaskResourceArg(parser, 'to set IAM policy to.')
iam_util.AddArgForPolicyFile(parser)
@gcloud_exception.CatchHTTPErrorRaiseHTTPException(
'Status code: {status_code}. {status_message}.')
def Run(self, args):
task_ref = args.CONCEPTS.task.Parse()
result = task.SetIamPolicyFromFile(task_ref, args.policy_file)
return result

343
gcloud auth application-default login/google-cloud-sdk/lib/surface/dataplex/tasks/update.yaml Normal file
View File

@@ -0,0 +1,343 @@
- release_tracks: [ALPHA, GA]
help_text:
brief: |
Update a Dataplex task resource.
description: |
Update a Dataplex task resource with the given configurations.
examples: |
To update a Dataplex task `test-task` within lake `test-lake` in location `us-central1` and
change the description to `Updated Description`, run:
$ {command} projects/test-project/locations/us-central1/lakes/test-lake/tasks/test-task \
--description='Updated Description'
request:
ALPHA:
api_version: v1
collection: dataplex.projects.locations.lakes.tasks
arguments:
resource:
help_text: |
Arguments and flags that specify the Dataplex Task you want to update.
spec: !REF googlecloudsdk.command_lib.dataplex.resources:task
params:
- arg_name: description
api_field: googleCloudDataplexV1Task.description
help_text: |
Description of the task.
- arg_name: display-name
api_field: googleCloudDataplexV1Task.displayName
help_text: |
Display name of the task.
- group: # trigger_spec
help_text: |
Spec related to how often and when a task should be triggered.
params:
- arg_name: trigger-start-time
api_field: googleCloudDataplexV1Task.triggerSpec.startTime
help_text: |
The first run of the task will be after this time.
If not specified, the task will run shortly after being submitted if
ON_DEMAND and based on the schedule if RECURRING.
- arg_name: trigger-disabled
api_field: googleCloudDataplexV1Task.triggerSpec.disabled
type: bool
default: false
help_text: |
Prevent the task from executing.
This does not cancel already running tasks. It is intended to temporarily
disable RECURRING tasks.
- arg_name: trigger-max-retires
type: int
api_field: googleCloudDataplexV1Task.triggerSpec.maxRetries
help_text: |
Number of retry attempts before aborting.
Set to zero to never attempt to retry a failed task.
- arg_name: trigger-schedule
api_field: googleCloudDataplexV1Task.triggerSpec.schedule
help_text: |
Cron schedule (https://en.wikipedia.org/wiki/Cron) for running
tasks periodically.
- group: # execution_spec
help_text: |
Spec related to how a task is executed.
params:
- arg_name: execution-args
api_field: googleCloudDataplexV1Task.executionSpec.args.additionalProperties
metavar: KEY=VALUE
type:
arg_dict:
flatten: true
spec:
- api_field: key
- api_field: value
help_text: |
The arguments to pass to the task.
The args can use placeholders of the format ${placeholder} as
part of key/value string. These will be interpolated before passing the
args to the driver. Currently supported placeholders:
- ${task_id}
- ${job_time}
To pass positional args, set the key as TASK_ARGS. The value should be a
comma-separated string of all the positional arguments. To use a
delimiter other than comma, refer to
https://cloud.google.com/sdk/gcloud/reference/topic/escaping. In case of
other keys being present in the args, then TASK_ARGS will be passed as
the last argument.
- arg_name: execution-service-account
api_field: googleCloudDataplexV1Task.executionSpec.serviceAccount
help_text: |
Service account to use to execute a task.
If not provided, the default Compute service account for the project is used.
- arg_name: execution-project
api_field: googleCloudDataplexV1Task.executionSpec.project
help_text: |
The project in which jobs are run.
By default, the project containing the Lake is used.
If a project is provided, the --execution-service-account must belong to this same
project.
- arg_name: max-job-execution-lifetime
api_field: googleCloudDataplexV1Task.executionSpec.maxJobExecutionLifetime
help_text: |
The maximum duration before the job execution expires.
- arg_name: kms-key
api_field: googleCloudDataplexV1Task.executionSpec.kmsKey
help_text: |
The Cloud KMS key to use for encryption, of the form:
projects/{project_number}/locations/{location_id}/keyRings/{key-ring-name}/cryptoKeys/{key-name}
- group: # task_config
mutex: true
help_text: |
Select which task you want to schedule and provide the required arguments:-
- spark tasks
- notebook tasks
params:
- group: # notebook_task_config
help_text: |
Config related to running custom Notebook tasks.
params:
- arg_name: notebook
repeated: true
api_field: googleCloudDataplexV1Task.notebook.notebook
help_text: |
Google Cloud Storage URIs of the notebook file or the path to a Notebook Content.
Path to input notebook.
- arg_name: notebook-file-uris
repeated: true
api_field: googleCloudDataplexV1Task.notebook.fileUris
help_text: |
Google Cloud Storage URIs of files to be placed in the working directory of each
executor.
- arg_name: notebook-archive-uris
repeated: true
api_field: googleCloudDataplexV1Task.notebook.archiveUris
help_text: |
Google Cloud Storage URIs of archives to be extracted into the working directory of
each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and
.zip.
- group: # notebook_infrastructure_spec
params:
- group: # notebook_batch_compute_resource
help_text: |
Compute resources needed for a Task when using Dataproc Serverless.
params:
- arg_name: notebook-batch-executors-count
api_field: googleCloudDataplexV1Task.notebook.infrastructureSpec.batch.executorsCount
type: int
help_text: |
Total number of job executors.
- arg_name: notebook-batch-max-executors-count
api_field: googleCloudDataplexV1Task.notebook.infrastructureSpec.batch.maxExecutorsCount
type: int
help_text: |
Max configurable executors.
If max_executors_count > executors_count, then auto-scaling is enabled.
- group: # notebook_container_image_runtime
help_text: |
Container Image Runtime Configuration.
params:
- arg_name: notebook-container-image
api_field: googleCloudDataplexV1Task.notebook.infrastructureSpec.containerImage.image
help_text: |
Optional custom container image for the job.
- arg_name: notebook-container-image-java-jars
api_field: googleCloudDataplexV1Task.notebook.infrastructureSpec.containerImage.javaJars
repeated: true
help_text: |
A list of Java JARS to add to the classpath.
Valid input includes Cloud Storage URIs to Jar binaries.
For example, gs://bucket-name/my/path/to/file.jar
- arg_name: notebook-container-image-properties
api_field: googleCloudDataplexV1Task.notebook.infrastructureSpec.containerImage.properties.additionalProperties
metavar: KEY=VALUE
type:
arg_dict:
flatten: true
spec:
- api_field: key
- api_field: value
help_text: |
Override to common configuration of open source components installed on
the Dataproc cluster.
The properties to set on daemon config files.
Property keys are specified in `prefix:property` format, for example
`core:hadoop.tmp.dir`.
For more information, see [Cluster
properties](https://cloud.google.com/dataproc/docs/concepts/cluster-properties).
- group: # notebook_vpc_network
help_text: |
Cloud VPC Network used to run the infrastructure.
params:
- group:
mutex: true
help_text: |
The Cloud VPC network identifier.
params:
- arg_name: notebook-vpc-network-name
api_field: googleCloudDataplexV1Task.notebook.infrastructureSpec.vpcNetwork.network
help_text: |
The Cloud VPC network in which the job is run. By default, the Cloud
VPC network named Default within the project is used.
- arg_name: notebook-vpc-sub-network-name
api_field: googleCloudDataplexV1Task.notebook.infrastructureSpec.vpcNetwork.subNetwork
help_text: |
The Cloud VPC sub-network in which the job is run.
- arg_name: notebook-vpc-network-tags
api_field: googleCloudDataplexV1Task.notebook.infrastructureSpec.vpcNetwork.networkTags
repeated: true
help_text: |
List of network tags to apply to the job.
- group: # spark_task_config
help_text: |
Config related to running custom Spark tasks.
params:
- group: # driver
mutex: true
help_text: |
The specification of the main method to call to drive the
job. Specify either the jar file that contains the main class or the
main class name.
params:
- arg_name: spark-main-jar-file-uri
api_field: googleCloudDataplexV1Task.spark.mainJarFileUri
help_text: |
The Google Cloud Storage URI of the jar file that contains the main class.
The execution args are passed in as a sequence of named process
arguments (`--key=value`).
- arg_name: spark-main-class
api_field: googleCloudDataplexV1Task.spark.mainClass
help_text: |
The name of the driver's main class. The jar file that contains the
class must be in the default CLASSPATH or specified in
- arg_name: spark-python-script-file
api_field: googleCloudDataplexV1Task.spark.pythonScriptFile
help_text: |
The Google Cloud Storage URI of the main Python file to use as the driver. Must
be a .py file.
- arg_name: spark-sql-script-file
api_field: googleCloudDataplexV1Task.spark.sqlScriptFile
help_text: |
A reference to a query file. This can be the Google Cloud Storage URI of the query file
or it can the path to a SqlScript Content.
- arg_name: spark-sql-script
api_field: googleCloudDataplexV1Task.spark.sqlScript
help_text: |
The SQL query text.
- arg_name: spark-file-uris
repeated: true
api_field: googleCloudDataplexV1Task.spark.fileUris
help_text: |
Google Cloud Storage URIs of files to be placed in the working directory of each
executor.
- arg_name: spark-archive-uris
repeated: true
api_field: googleCloudDataplexV1Task.spark.archiveUris
help_text: |
Google Cloud Storage URIs of archives to be extracted into the working directory of
each executor. Supported file types: .jar, .tar, .tar.gz, .tgz, and
.zip.
- group: # infrastructure_spec
params:
- group: # batch_compute_resource
help_text: |
Compute resources needed for a Task when using Dataproc Serverless.
params:
- arg_name: batch-executors-count
api_field: googleCloudDataplexV1Task.spark.infrastructureSpec.batch.executorsCount
type: int
help_text: |
Total number of job executors.
- arg_name: batch-max-executors-count
api_field: googleCloudDataplexV1Task.spark.infrastructureSpec.batch.maxExecutorsCount
type: int
help_text: |
Max configurable executors.
If max_executors_count > executors_count, then auto-scaling is enabled.
- group: # container_image_runtime
help_text: |
Container Image Runtime Configuration.
params:
- arg_name: container-image
api_field: googleCloudDataplexV1Task.spark.infrastructureSpec.containerImage.image
help_text: |
Optional custom container image for the job.
- arg_name: container-image-java-jars
api_field: googleCloudDataplexV1Task.spark.infrastructureSpec.containerImage.javaJars
repeated: true
help_text: |
A list of Java JARS to add to the classpath.
Valid input includes Cloud Storage URIs to Jar binaries.
For example, gs://bucket-name/my/path/to/file.jar
- arg_name: container-image-python-packages
api_field: googleCloudDataplexV1Task.spark.infrastructureSpec.containerImage.pythonPackages
repeated: true
help_text: |
A list of python packages to be installed.
Valid formats include Cloud Storage URI to a PIP installable library.
For example, gs://bucket-name/my/path/to/lib.tar.gz
- arg_name: container-image-properties
api_field: googleCloudDataplexV1Task.spark.infrastructureSpec.containerImage.properties.additionalProperties
metavar: KEY=VALUE
type:
arg_dict:
flatten: true
spec:
- api_field: key
- api_field: value
help_text: |
Override to common configuration of open source components installed on
the Dataproc cluster.
The properties to set on daemon config files.
Property keys are specified in `prefix:property` format, for example
`core:hadoop.tmp.dir`.
For more information, see [Cluster
properties](https://cloud.google.com/dataproc/docs/concepts/cluster-properties).
- group: # vpc-network
help_text: |
Cloud VPC Network used to run the infrastructure.
params:
- group:
help_text: |
The Cloud VPC network identifier.
params:
- arg_name: vpc-network-name
api_field: googleCloudDataplexV1Task.spark.infrastructureSpec.vpcNetwork.network
help_text: |
The Cloud VPC network in which the job is run. By default, the Cloud
VPC network named Default within the project is used.
- arg_name: vpc-sub-network-name
api_field: googleCloudDataplexV1Task.spark.infrastructureSpec.vpcNetwork.subNetwork
help_text: |
The Cloud VPC sub-network in which the job is run.
- arg_name: vpc-network-tags
api_field: googleCloudDataplexV1Task.spark.infrastructureSpec.vpcNetwork.networkTags
repeated: true
help_text: |
List of network tags to apply to the job.
labels:
api_field: googleCloudDataplexV1Task.labels
async:
collection: dataplex.projects.locations.operations
update:
# b/136698204
read_modify_update: true