feat: Add new gcloud commands, API clients, and third-party libraries across various services.

gcloud auth application-default login/google-cloud-sdk/lib/surface/dlp/datasources/__init__.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2018 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""The gcloud dlp datasources command group."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+
+
+@base.ReleaseTracks(base.ReleaseTrack.ALPHA)
+class Datasources(base.Group):
+  # pylint: disable=line-too-long
+  """Cloud DLP Commands for analyzing Google Cloud data repositories.
+
+  Cloud DLP Commands for inspecting and analyzing sensitive data in Google Cloud
+  data repositories.
+
+  See [Inspecting Storage and Databases for Sensitive Data]
+  (https://cloud.google.com/dlp/docs/inspecting-storage)
+  for more details.
+  """
+  # pylint: enable=line-too-long

gcloud auth application-default login/google-cloud-sdk/lib/surface/dlp/datasources/bigquery/__init__.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2018 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""The gcloud dlp datasources bigquery command group."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+
+
+@base.ReleaseTracks(base.ReleaseTrack.ALPHA)
+class BigQuery(base.Group):
+  """Cloud DLP Commands for analyzing data in BigQuery tables.
+
+  Cloud DLP commands for inspecting and analyzing content in BigQuery tables.
+  """

gcloud auth application-default login/google-cloud-sdk/lib/surface/dlp/datasources/bigquery/analyze.yaml

@@ -0,0 +1,102 @@
+# Copyright 2018 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+- release_tracks: [ALPHA]
+  help_text:
+    brief: Schedule a risk analysis job for content in a BigQuery table.
+    description: |
+      Schedules a job to compute risk analysis metrics on content in a
+      BigQuery table.
+    examples: |
+      To create a job `my-bq-analysis` to analyze records in a BigQuery table
+      `myproject.myds.mytable`, run:
+
+        $ {command}  `myproject.myds.mytable` --job-id my-ds-job --output-topics my-topic --numerical-stat-field col2
+
+  request:
+    collection: dlp.projects.dlpJobs
+    display_resource_type: job
+    method: create
+    api_version: v2
+    modify_request_hooks:
+    - googlecloudsdk.command_lib.dlp.hooks:SetRequestParent
+
+  arguments:
+    params:
+    - group:
+        mutex: true
+        required: true
+        help_text: Privacy analysis metrics.
+        params:
+        - arg_name: categorical-stat-field
+          api_field: googlePrivacyDlpV2CreateDlpJobRequest.riskJob.privacyMetric.categoricalStatsConfig.field.name
+          help_text: |
+            An individual column to compute numerical stats on, including number
+            of distinct values and value count distribution.
+        - arg_name: numerical-stat-field
+          api_field: googlePrivacyDlpV2CreateDlpJobRequest.riskJob.privacyMetric.numericalStatsConfig.field.name
+          help_text: |
+            Individual column to compute numerical stats on. Supported types are
+            integer, float, date, datetime, timestamp, time.
+        - group:
+            help_text: l-diversity analysis options.
+            params:
+            - arg_name: quasi-ids
+              api_field: googlePrivacyDlpV2CreateDlpJobRequest.riskJob.privacyMetric.lDiversityConfig.quasiIds
+              type: googlecloudsdk.command_lib.dlp.hooks:PrivacyField
+              required: true
+              help_text: |
+                A set of quasi-identifiers indicating how equivalence classes are
+                defined for the l-diversity computation. When multiple fields are
+                specified, they are considered a single composite key.
+
+            - arg_name: sensitive-attribute
+              api_field: googlePrivacyDlpV2CreateDlpJobRequest.riskJob.privacyMetric.lDiversityConfig.sensitiveAttribute.name
+              help_text: |
+                Sensitive field for computing the l-diversity value.
+
+    - group:
+        mutex: true
+        required: true
+        params:
+        - arg_name: output-topics
+          api_field: googlePrivacyDlpV2CreateDlpJobRequest.riskJob.actions
+          type: googlecloudsdk.command_lib.dlp.hooks:PubSubTopicAction
+          help_text: |
+            Publishes the results of a Cloud DLP job to one or more Cloud Pub/Sub
+            topics.
+
+            Note: The topic must have given publishing access rights to the DLP
+            API service account executing the Cloud DLP job.
+        - arg_name: output-table
+          api_field: googlePrivacyDlpV2CreateDlpJobRequest.riskJob.actions
+          type: googlecloudsdk.command_lib.dlp.hooks:BigQueryTableAction
+          help_text: |
+            Publishes results of a Cloud DLP job to a BigQuery table.
+            BigQuery tables are uniquely identified by their project_id,
+            dataset_id, and table_id in the format
+            `<project_id>.<dataset_id>.<table_id>`. If no table_id is specified,
+            DLP will create a new table.
+
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:big_query_table
+      arg_name: input_table
+      api_field: googlePrivacyDlpV2CreateDlpJobRequest.riskJob.sourceTable
+      required: true
+      is_positional: true
+      processor: googlecloudsdk.command_lib.dlp.hooks:ExtractBqTableFromInputConfig
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:job_id
+    resource:
+      help_text: |
+        The Cloud DLP project to create Job In.
+      spec: !REF googlecloudsdk.command_lib.dlp.resources:project
+      is_parent_resource: true

gcloud auth application-default login/google-cloud-sdk/lib/surface/dlp/datasources/bigquery/inspect.yaml

@@ -0,0 +1,64 @@
+# Copyright 2018 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+- release_tracks: [ALPHA]
+  help_text:
+    brief: Schedules a job to inspect content in a BigQuery table.
+    description: |
+      Schedules a job to inspect content in a BigQuery table for sensitive data.
+
+       See [Inspecting Storage and Databases for Sensitive Data](https://cloud.google.com/dlp/docs/inspecting-storage)
+       for more details.
+    examples: |
+      The following command creates a job `my-bq-job` to scan records in
+      BigQuery table `myproject.myds.mytable`:
+
+        $ {command}  `myproject.myds.mytable` --job-id my-ds-job --min-time '2018-01-01T12:00:00Z' --max-time '2018-01-31T12:00:00Z' --output-topics my-topic --max-findings-per-item 3 --max-findings 1000 --info-types PHONE_NUMBER,EMAIL_ADDRESS --min-likelihood very-likely --include-quote --exclude-info-types
+
+  request:
+    collection: dlp.projects.dlpJobs
+    display_resource_type: job
+    method: create
+    api_version: v2
+    modify_request_hooks:
+    - googlecloudsdk.command_lib.dlp.hooks:SetRequestParent
+    - googlecloudsdk.command_lib.dlp.hooks:UpdateIdentifyingFields
+
+  arguments:
+    additional_arguments_hook: googlecloudsdk.command_lib.dlp.hooks:GetIdentifyingFieldsArg
+    params:
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:big_query_table
+      arg_name: input_table
+      api_field: googlePrivacyDlpV2CreateDlpJobRequest.inspectJob.storageConfig.bigQueryOptions
+      required: true
+      is_positional: true
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:max_item_findings
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:min_time
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:max_time
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:jobs_output_group
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:info_type
+      api_field: googlePrivacyDlpV2CreateDlpJobRequest.inspectJob.inspectConfig.infoTypes
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:min_likelihood
+      api_field: googlePrivacyDlpV2CreateDlpJobRequest.inspectJob.inspectConfig.minLikelihood
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:include_quote
+      api_field: googlePrivacyDlpV2CreateDlpJobRequest.inspectJob.inspectConfig.includeQuote
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:max_findings
+      api_field: googlePrivacyDlpV2CreateDlpJobRequest.inspectJob.inspectConfig.limits.maxFindingsPerRequest
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:exclude_info_types
+      api_field: googlePrivacyDlpV2CreateDlpJobRequest.inspectJob.inspectConfig.excludeInfoTypes
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:job_id
+    resource:
+      help_text: |
+        The Cloud DLP project to create Job In.
+      spec: !REF googlecloudsdk.command_lib.dlp.resources:project
+      is_parent_resource: true

gcloud auth application-default login/google-cloud-sdk/lib/surface/dlp/datasources/datastore/__init__.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2018 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""The gcloud dlp datasources datastore command group."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+
+
+@base.ReleaseTracks(base.ReleaseTrack.ALPHA)
+class Datastore(base.Group):
+  """Cloud DLP Commands for analyzing data in Cloud Datastore repositories.
+
+  Cloud DLP commands for inspecting and analyzing content in Cloud Datastore
+  repositories.
+  """
+
+
+
+

gcloud auth application-default login/google-cloud-sdk/lib/surface/dlp/datasources/datastore/inspect.yaml

@@ -0,0 +1,61 @@
+# Copyright 2018 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+- release_tracks: [ALPHA]
+  help_text:
+    brief: Schedules a job to inspect content in a Cloud Datastore.
+    description: |
+      Schedules a job to inspect content in a Cloud Datastore for sensitive data.
+
+       See [Inspecting Storage and Databases for Sensitive Data](https://cloud.google.com/dlp/docs/inspecting-storage)
+       for more details.
+    examples: |
+      The following command creates a job `my-ds-job` to scan records in
+      Cloud Datastore table `myds:mykind`:
+
+        $ {command}  `myds:mykind` --job-id my-ds-job --min-time '2018-01-01T12:00:00Z' --max-time '2018-01-31T12:00:00Z' --output-topics my-topic --max-findings-per-item 3 --max-findings 1000 --info-types PHONE_NUMBER,EMAIL_ADDRESS --min-likelihood very-likely --include-quote --exclude-info-types
+
+  request:
+    collection: dlp.projects.dlpJobs
+    display_resource_type: job
+    method: create
+    api_version: v2
+    modify_request_hooks:
+    - googlecloudsdk.command_lib.dlp.hooks:SetRequestParent
+  arguments:
+    params:
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:datastore_kind
+      arg_name: datastore_kind
+      api_field: googlePrivacyDlpV2CreateDlpJobRequest.inspectJob.storageConfig.datastoreOptions
+      is_positional: true
+      required: true
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:max_item_findings
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:min_time
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:max_time
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:jobs_output_group
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:info_type
+      api_field: googlePrivacyDlpV2CreateDlpJobRequest.inspectJob.inspectConfig.infoTypes
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:min_likelihood
+      api_field: googlePrivacyDlpV2CreateDlpJobRequest.inspectJob.inspectConfig.minLikelihood
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:include_quote
+      api_field: googlePrivacyDlpV2CreateDlpJobRequest.inspectJob.inspectConfig.includeQuote
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:max_findings
+      api_field: googlePrivacyDlpV2CreateDlpJobRequest.inspectJob.inspectConfig.limits.maxFindingsPerRequest
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:exclude_info_types
+      api_field: googlePrivacyDlpV2CreateDlpJobRequest.inspectJob.inspectConfig.excludeInfoTypes
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:job_id
+    resource:
+      help_text: |
+        The Cloud DLP project to create Job In.
+      spec: !REF googlecloudsdk.command_lib.dlp.resources:project
+      is_parent_resource: true

gcloud auth application-default login/google-cloud-sdk/lib/surface/dlp/datasources/gcs/__init__.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2018 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""The gcloud dlp datasources gcs command group."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+
+
+@base.ReleaseTracks(base.ReleaseTrack.ALPHA)
+class Gcs(base.Group):
+  """Cloud DLP Commands for analyzing data in Google Cloud Storage.
+
+  Cloud DLP commands for inspecting content in a Google Cloud Storage bucket.
+  """
+
+
+
+

gcloud auth application-default login/google-cloud-sdk/lib/surface/dlp/datasources/gcs/inspect.yaml

@@ -0,0 +1,69 @@
+# Copyright 2018 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+- release_tracks: [ALPHA]
+  help_text:
+    brief: Schedules a job to inspect content in a Google Cloud Storage bucket.
+    description: |
+      Schedules a job to inspect content in a Google Cloud Storage bucket for
+      sensitive data.
+
+      See [Inspecting Storage and Databases for Sensitive Data](https://cloud.google.com/dlp/docs/inspecting-storage)
+      for more details.
+    examples: |
+      The following command creates a job `my-gcs-job` to scan files in the
+      Google Cloud Storage path `gs://testproject/dlp/*`:
+
+        $ {command}  `gs://testproject/dlp/*` --job-id my-gcs-job --file-size-limit 10000 --min-time '2018-01-01T12:00:00Z' --max-time '2018-01-31T12:00:00Z' --output-topics my-topic --max-findings-per-item 3 --max-findings 1000 --info-types PHONE_NUMBER,EMAIL_ADDRESS --min-likelihood very-likely --include-quote --exclude-info-types
+
+  request:
+    collection: dlp.projects.dlpJobs
+    display_resource_type: job
+    method: create
+    api_version: v2
+    modify_request_hooks:
+    - googlecloudsdk.command_lib.dlp.hooks:SetRequestParent
+    - googlecloudsdk.command_lib.util.hooks.request_modifiers:SetFieldFromArg:api_field=googlePrivacyDlpV2CreateDlpJobRequest.inspectJob.storageConfig.cloudStorageOptions.bytesLimitPerFile,arg_name=file_size_limit
+  arguments:
+    params:
+    - arg_name: file-size-limit
+      api_field: googlePrivacyDlpV2CreateDlpJobRequest.inspectJob.storageConfig.cloudStorageOptions.bytesLimitPerFile
+      type: int
+      default: 1024
+      help_text: |
+        Integer indicating max number of bytes to scan from a file. If a
+        scanned file's size is bigger than this value, then the rest of the
+        bytes are omitted. The default value is 1024.
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:gcs_path
+      api_field: googlePrivacyDlpV2CreateDlpJobRequest.inspectJob.storageConfig.cloudStorageOptions
+      is_positional: true
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:max_item_findings
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:min_time
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:max_time
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:jobs_output_group
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:info_type
+      api_field: googlePrivacyDlpV2CreateDlpJobRequest.inspectJob.inspectConfig.infoTypes
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:min_likelihood
+      api_field: googlePrivacyDlpV2CreateDlpJobRequest.inspectJob.inspectConfig.minLikelihood
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:include_quote
+      api_field: googlePrivacyDlpV2CreateDlpJobRequest.inspectJob.inspectConfig.includeQuote
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:max_findings
+      api_field: googlePrivacyDlpV2CreateDlpJobRequest.inspectJob.inspectConfig.limits.maxFindingsPerRequest
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:exclude_info_types
+      api_field: googlePrivacyDlpV2CreateDlpJobRequest.inspectJob.inspectConfig.excludeInfoTypes
+    - _REF_: googlecloudsdk.command_lib.dlp.flags:job_id
+    resource:
+      help_text: |
+        The Cloud DLP project to create Job In.
+      spec: !REF googlecloudsdk.command_lib.dlp.resources:project
+      is_parent_resource: true