feat: Add new gcloud commands, API clients, and third-party libraries across various services.

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataplex/entries/__init__.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2024 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command group for Dataplex Catalog Entries Resource."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+
+
+@base.DefaultUniverseOnly
+@base.ReleaseTracks(
+    base.ReleaseTrack.ALPHA, base.ReleaseTrack.BETA, base.ReleaseTrack.GA
+)
+class Entries(base.Group):
+  """Manage Dataplex Catalog Entries resources."""
+
+  category = base.DATA_ANALYTICS_CATEGORY

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataplex/entries/create.py

@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2024 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Create command for Dataplex Catalog Entries Resource."""
+
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.dataplex import entry as entry_api
+from googlecloudsdk.api_lib.util import exceptions as gcloud_exception
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.dataplex import flags
+from googlecloudsdk.command_lib.dataplex import resource_args
+
+
+@base.DefaultUniverseOnly
+@base.ReleaseTracks(
+    base.ReleaseTrack.ALPHA, base.ReleaseTrack.BETA, base.ReleaseTrack.GA
+)
+class Create(base.CreateCommand):
+  """Create a Dataplex Entry resource."""
+
+  detailed_help = {
+      'EXAMPLES': """\
+          To create a Dataplex entry with name `my-dataplex-entry` in location
+          `us-central1` in entry group `my-entry-group` and with entry type projects/my-project/locations/us-central1/entryTypes/my-type, run:
+
+            $ {command} my-dataplex-entry --location=us-central1 --entry_group=my-entry-group --entry-type projects/my-project/locations/us-central1/entryTypes/my-type
+
+          To create a Dataplex Entry with name `my-child-entry` and set its parent to an existing entry `my-parent-entry`, run:
+
+            $ {command} my-child-entry --location=us-central1 --entry_group=my-entry-group --entry-type projects/my-project/locations/us-central1/entryTypes/my-type --parent-entry projects/my-project/locations/us-central1/entryGroups/my-entry-group/entries/my-parent-entry
+
+          To create a Dataplex Entry with its description, display name, ancestors, labels and timestamps populated in its EntrySource, run:
+
+            $ {command} my-entry --location=us-central1 --entry_group=my-entry-group --entry-type projects/my-project/locations/us-central1/entryTypes/my-type --entry-source-description 'This is a description of the Entry.' --entry-source-display-name 'display name' --entry-source-ancestors '{"type":"projects/my-project/locations/us-central1/entryTypes/some-type", "name":"projects/my-project/locations/us-central1/entryGroups/my-entry-group/entries/ancestor-entry"}, {"type":"projects/my-project/locations/us-central1/entryTypes/another-type", "name":"projects/my-project/locations/us-central1/entryGroups/my-entry-group/entries/another-ancestor-entry"}' --entry-source-labels key1=value1,key2=value2 --entry-source-create-time 2024-01-01T09:39:25.160173Z --entry-source-update-time 2024-01-01T09:39:25.160173Z
+
+          To create a Dataplex Entry reading its aspects from a YAML file, run:
+
+            $ {command} my-entry --location=us-central1 --entry_group=my-entry-group --entry-type projects/my-project/locations/us-central1/entryTypes/my-type --aspects aspects.yaml
+
+          The file containing the aspects has the following format:
+
+            my-project.us-central1.my-aspect-type:
+              aspectType: my-project.us-central1.my-aspect-type
+              createTime: "2024-01-01T09:39:25.160173Z"
+              updateTime: "2024-01-01T09:39:25.160173Z"
+              data:
+                {}
+          """,
+  }
+
+  @staticmethod
+  def Args(parser):
+    resource_args.AddProjectArg(parser, 'to create the Entry.')
+    resource_args.AddEntryResourceArg(parser)
+    resource_args.AddEntryTypeResourceArg(parser)
+    resource_args.AddParentEntryResourceArg(parser)
+
+    parser.add_argument(
+        '--fully-qualified-name',
+        help=(
+            'A name for the entry that can reference it in an external system.'
+            ' The maximum size of the field is 4000 characters.'
+        ),
+    )
+    flags.AddEntrySourceArgs(parser, for_update=False)
+    flags.AddAspectFlags(
+        parser, update_aspects_name='aspects', remove_aspects_name=None
+    )
+
+  @gcloud_exception.CatchHTTPErrorRaiseHTTPException(
+      'Status code: {status_code}. {status_message}.'
+  )
+  def Run(self, args):
+    return entry_api.Create(args)

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataplex/entries/delete.yaml

@@ -0,0 +1,17 @@
+- release_tracks: [ALPHA, BETA, GA]
+  help_text:
+    brief: Delete a Dataplex entry.
+    description: Delete a Dataplex entry.
+    examples: |-
+      To delete the entry 'entry1', run:
+
+        $ {command} entry1 --entry-group=entry-group1 --location=us-central1 --project=test-project
+
+  request:
+    collection: dataplex.projects.locations.entryGroups.entries
+
+  arguments:
+    resource:
+      help_text: Entry to delete.
+      spec: !REF googlecloudsdk.command_lib.dataplex.resources:entry
+      is_positional: true

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataplex/entries/describe.yaml

@@ -0,0 +1,27 @@
+- release_tracks: [ALPHA, BETA, GA]
+  help_text:
+    brief: Describe a Dataplex entry.
+    description: |
+      Describe a Dataplex entry.
+
+      Displays the details of a Dataplex entry resource given a valid entry ID.
+    examples: |-
+      To describe a Dataplex entry `entry1` within entry group `entry-group1` in location `us-central1`,
+      run:
+
+        $ {command} entry1 --entry-group=entry-group1 --location=us-central1 --project=test-project
+
+  request:
+    collection: dataplex.projects.locations.entryGroups.entries
+    modify_request_hooks:
+    - googlecloudsdk.command_lib.dataplex.request_hooks:TransformEntryRootPath
+
+  arguments:
+    resource:
+      help_text: Arguments and flags that define the Dataplex Entry you want to describe.
+      spec: !REF googlecloudsdk.command_lib.dataplex.resources:entry
+      is_positional: true
+    params:
+    - !REF googlecloudsdk.command_lib.dataplex.flags:get_entry_view
+    - !REF googlecloudsdk.command_lib.dataplex.flags:aspect_types
+    - !REF googlecloudsdk.command_lib.dataplex.flags:paths

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataplex/entries/list.yaml

@@ -0,0 +1,40 @@
+- release_tracks: [ALPHA, BETA, GA]
+  help_text:
+    brief: List Dataplex entries.
+    description: List Dataplex entries in an entry group.
+    examples: |-
+      To List the entries in entry group 'entry-group1', run:
+
+        $ {command} --entry-group=entry-group1 --location=us-central1 --project=test-project
+
+  request:
+    collection: dataplex.projects.locations.entryGroups.entries
+
+  arguments:
+    resource:
+      help_text: Example Entry Group to list all contained Entries.
+      spec: !REF googlecloudsdk.command_lib.dataplex.resources:entry_group
+      is_positional: false
+
+  response:
+    id_field: name
+  output:
+    format: |
+      table(
+        name.basename():label=ENTRY_ID,
+        entrySource.location:label=SOURCE_LOCATION,
+        entrySource.display_name:label=DISPLAY_NAME,
+        entrySource.description:label=DESCRIPTION,
+        entrySource.resource:label=RESOURCE,
+        entrySource.system:label=SYSTEM,
+        entrySource.platform:label=PLATFORM,
+        entrySource.labels:label=LABELS,
+        entrySource.ancestors:label=ANCESTORS,
+        entryType:label=ENTRY_TYPE,
+        createTime:label=ENTRY_CREATE_TIME,
+        updateTime:label=ENTRY_UPDATE_TIME,
+        parent_entry:label=PARENT_ENTRY,
+        fullyQualifiedName:label=FULLY_QUALIFIED_NAME,
+        entrySource.create_time:label=SOURCE_CREATE_TIME,
+        entrySource.update_time:label=SOURCE_UPDATE_TIME
+      )

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataplex/entries/lookup.yaml

@@ -0,0 +1,38 @@
+- release_tracks: [ALPHA, BETA, GA]
+  help_text:
+    brief: Lookup a Dataplex entry.
+    description: |
+      Lookup a Dataplex entry.
+
+      Displays the details of a Dataplex entry resource given a valid entry ID.
+      Unlike `describe`, where the Dataplex permission is required for access, for `lookup` the
+      permission from the original source system is required on the entry instead. For example, if
+      the source is a BigQuery table, you need the `bigquery.tables.get` permission to lookup an
+      entry.
+    examples: |-
+      To lookup a Dataplex entry `entry1` within entry group `entry-group1` in location `us-central1`,
+      run:
+
+        $ {command} entry1 --entry-group=entry-group1 --location=us-central1 --project=test-project
+
+  request:
+    collection: dataplex.projects.locations
+    method: lookupEntry
+    modify_request_hooks:
+    - googlecloudsdk.command_lib.dataplex.request_hooks:TransformEntryRootPath
+
+  arguments:
+    params:
+    - arg_name: entry
+      help_text: Arguments and flags that define the Dataplex Entry you want to lookup.
+      is_positional: true
+      required: true
+      override_resource_collection: true
+      resource_spec: !REF googlecloudsdk.command_lib.dataplex.resources:entry
+      # Here we populate two fields from a single resource argument.
+      resource_method_params:
+        name: 'projects/{projectsId}/locations/{locationsId}'
+        entry: '{__relative_name__}'
+    - !REF googlecloudsdk.command_lib.dataplex.flags:get_entry_view
+    - !REF googlecloudsdk.command_lib.dataplex.flags:aspect_types
+    - !REF googlecloudsdk.command_lib.dataplex.flags:paths

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataplex/entries/remove_aspects.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2024 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""`gcloud dataplex entries remove-aspects` command."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.dataplex import entry as entry_api
+from googlecloudsdk.api_lib.util import exceptions as gcloud_exception
+from googlecloudsdk.calliope import base
+from googlecloudsdk.calliope import parser_arguments
+from googlecloudsdk.calliope import parser_extensions
+from googlecloudsdk.command_lib.dataplex import flags
+from googlecloudsdk.command_lib.dataplex import resource_args
+
+
+@base.DefaultUniverseOnly
+@base.ReleaseTracks(
+    base.ReleaseTrack.ALPHA, base.ReleaseTrack.BETA, base.ReleaseTrack.GA
+)
+class RemoveAspects(base.UpdateCommand):
+  """Remove aspects from a Dataplex Entry."""
+
+  detailed_help = {
+      'EXAMPLES': """\
+
+          To remove all aspects of type `test-project.us-central1.some-aspect-type` from the entry, run:
+
+            $ {command} entry1 --project=test-project --location=us-central1 --entry-group entry-group1 --keys='test-project.us-central1.some-aspect-type@*'
+
+          To remove all aspects on path `Schema.column1` from the entry, run:
+
+            $ {command} entry1 --project=test-project --location=us-central1 --entry-group entry-group1 --keys='*@Schema.column1'
+
+          To remove exact aspects `test-project.us-central1.some-aspect-type@Schema.column1` and `test-project.us-central1.some-aspect-type2@Schema.column2` from the entry, run:
+
+            $ {command} entry1 --project=test-project --location=us-central1 --entry-group entry-group1 --keys=test-project.us-central1.some-aspect-type@Schema.column1,test-project.us-central2.some-aspect-type@Schema.column2
+
+          """,
+  }
+
+  @staticmethod
+  def Args(parser: parser_arguments.ArgumentInterceptor):
+    resource_args.AddEntryResourceArg(parser)
+
+    flags.AddAspectFlags(
+        parser,
+        update_aspects_name=None,
+        remove_aspects_name='keys',
+        required=True,
+    )
+
+  @gcloud_exception.CatchHTTPErrorRaiseHTTPException(
+      'Status code: {status_code}. {status_message}.'
+  )
+  def Run(self, args: parser_extensions.Namespace):
+    # Under the hood we perorm UpdateEntry call affecting only aspects field.
+    return entry_api.Update(args, remove_aspects_arg_name='keys')

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataplex/entries/search.yaml

@@ -0,0 +1,52 @@
+- release_tracks: [ALPHA, BETA, GA]
+  help_text:
+    brief: Searches for Dataplex entries.
+    description: Searches for entries matching given query and scope.
+    examples: |-
+      To search project 'my-project' for Dataplex resources that match the simple predicate 'foo':
+
+        $ {command} 'foo' --project=my-project
+
+  generic:
+    # This removes `filter`, `sort-by`, `limit`, and `page-size` flags. `limit` and `page-size` are
+    # re-added later in this config. This seems to be the only way to remove flags `filter` and
+    # `sort-by` that duplicate functionality of flags that API already has and are exposed with other commands.
+    disable_paging_flags: true
+
+  request:
+    collection: dataplex.projects.locations
+    method: searchEntries
+    disable_resource_check: true
+
+  arguments:
+    params:
+    - arg_name: query
+      api_field: query
+      help_text: The query against which entries in scope should be matched.
+      is_positional: true
+      required: true
+    - arg_name: project
+      help_text: The project to which the request should be attributed.
+      api_field: name
+      processor: googlecloudsdk.command_lib.dataplex.processors:ProjectToSearchEntriesName
+      required: true
+    - arg_name: scope
+      help_text: The scope under which the search should be operating. Should either be organizations/<org_id> or projects/<project_ref>. If left unspecified, it will default to the organization where the project is located.
+      api_field: scope
+    - arg_name: order-by
+      help_text: |
+        Specifies the ordering of results, currently supported case-sensitive
+        choices are:
+
+        * `title [asc|desc]`, defaults to ascending if not specified.
+      api_field: orderBy
+    - arg_name: page-size
+      help_text: Maximum number of resources per page. No more than 500.
+      type: int
+      api_field: pageSize
+    - arg_name: limit
+      help_text: Maximum number of resources.
+      type: int
+  response:
+    modify_response_hooks:
+    - googlecloudsdk.command_lib.dataplex.response_hooks:ExtractEntryFromSearchEntriesResult

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataplex/entries/update.py

@@ -0,0 +1,92 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2024 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""`gcloud dataplex entries update` command."""
+
+from googlecloudsdk.api_lib.dataplex import entry as entry_api
+from googlecloudsdk.api_lib.util import exceptions as gcloud_exception
+from googlecloudsdk.calliope import base
+from googlecloudsdk.calliope import parser_arguments
+from googlecloudsdk.calliope import parser_extensions
+from googlecloudsdk.command_lib.dataplex import flags
+from googlecloudsdk.command_lib.dataplex import resource_args
+
+
+@base.DefaultUniverseOnly
+@base.ReleaseTracks(
+    base.ReleaseTrack.ALPHA, base.ReleaseTrack.BETA, base.ReleaseTrack.GA
+)
+class Update(base.UpdateCommand):
+  """Update a Dataplex Entry."""
+
+  detailed_help = {
+      'DESCRIPTION': 'Update specified fields in a given Dataplex Entry.',
+      'EXAMPLES': """\
+          To update fully qualified name (FQN) of an entry, run:
+
+            $ {command} entry1 --project=test-project --location=us-central1 --entry-group entry-group1 --fully-qualified-name='custom:a.b.c'
+
+          To update description of an entry, run:
+
+            $ {command} entry1 --project=test-project --location=us-central1 --entry-group entry-group1 --entry-source-description='Updated description' --entry-source-update-time='1998-09-04T12:00:00-0700'
+
+          To clear the description of an entry, run:
+
+            $ {command} entry1 --project=test-project --location=us-central1 --entry-group entry-group1 --clear-entry-source-description --entry-source-update-time='1998-09-04T12:00:00-0700'
+
+          To add or update aspects from the YAML/JSON file, run:
+
+            $ {command} entry1 --project=test-project --location=us-central1 --entry-group entry-group1 --update-aspects=path-to-a-file-with-aspects.json
+
+          To remove all aspects of type `test-project.us-central1.some-aspect-type` from the entry, run:
+
+            $ {command} entry1 --project=test-project --location=us-central1 --entry-group entry-group1 --remove-aspects='test-project.us-central1.some-aspect-type@*'
+
+          To remove all aspects on path `Schema.column1` from the entry, run:
+
+            $ {command} entry1 --project=test-project --location=us-central1 --entry-group entry-group1 --remove-aspects='*@Schema.column1'
+
+          To remove exact aspects `test-project.us-central1.some-aspect-type@Schema.column1` and `test-project.us-central1.some-aspect-type2@Schema.column2` from the entry, run:
+
+            $ {command} entry1 --project=test-project --location=us-central1 --entry-group entry-group1 --remove-aspects=test-project.us-central1.some-aspect-type@Schema.column1,test-project.us-central2.some-aspect-type@Schema.column2
+
+          """,
+  }
+
+  @staticmethod
+  def Args(parser: parser_arguments.ArgumentInterceptor):
+    resource_args.AddEntryResourceArg(parser)
+
+    fully_qualified_name = parser.add_mutually_exclusive_group()
+    fully_qualified_name.add_argument(
+        '--fully-qualified-name',
+        help=(
+            'FQN, a name for the entry that can reference it in an external'
+            ' system.'
+        ),
+    )
+    fully_qualified_name.add_argument(
+        '--clear-fully-qualified-name',
+        action='store_true',
+        help='Clear the FQN for the Entry.',
+    )
+
+    flags.AddEntrySourceArgs(parser, for_update=True)
+    flags.AddAspectFlags(parser)
+
+  @gcloud_exception.CatchHTTPErrorRaiseHTTPException(
+      'Status code: {status_code}. {status_message}.'
+  )
+  def Run(self, args: parser_extensions.Namespace):
+    return entry_api.Update(args)

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataplex/entries/update_aspects.py

@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2024 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""`gcloud dataplex entries update-aspects` command."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.dataplex import entry as entry_api
+from googlecloudsdk.api_lib.util import exceptions as gcloud_exception
+from googlecloudsdk.calliope import base
+from googlecloudsdk.calliope import parser_arguments
+from googlecloudsdk.calliope import parser_extensions
+from googlecloudsdk.command_lib.dataplex import flags
+from googlecloudsdk.command_lib.dataplex import resource_args
+
+
+@base.DefaultUniverseOnly
+@base.ReleaseTracks(
+    base.ReleaseTrack.ALPHA, base.ReleaseTrack.BETA, base.ReleaseTrack.GA
+)
+class UpdateAspects(base.UpdateCommand):
+  """Add or update aspects for a Dataplex Entry."""
+
+  detailed_help = {
+      'EXAMPLES': """\
+
+          To add or update aspects for the Dataplex entry `entry1` within the entry group `entry-group1` in location `us-central1` from the YAML/JSON file, run:
+
+            $ {command} entry1 --project=test-project --location=us-central1 --entry-group entry-group1 --aspects=path-to-a-file-with-aspects.json
+
+          """,
+  }
+
+  @staticmethod
+  def Args(parser: parser_arguments.ArgumentInterceptor):
+    resource_args.AddEntryResourceArg(parser)
+
+    flags.AddAspectFlags(
+        parser,
+        update_aspects_name='aspects',
+        remove_aspects_name=None,
+        required=True,
+    )
+
+  @gcloud_exception.CatchHTTPErrorRaiseHTTPException(
+      'Status code: {status_code}. {status_message}.'
+  )
+  def Run(self, args: parser_extensions.Namespace):
+    # Under the hood we perorm UpdateEntry call affecting only aspects field.
+    return entry_api.Update(args, update_aspects_arg_name='aspects')