feat: Add new gcloud commands, API clients, and third-party libraries across various services.

gcloud auth application-default login/google-cloud-sdk/lib/surface/spanner/migrate/__init__.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2023 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""The migrate command group for spanner."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+
+
+@base.ReleaseTracks(base.ReleaseTrack.ALPHA)
+class Migration(base.Group):
+  """Migrate external databases to Cloud Spanner."""

gcloud auth application-default login/google-cloud-sdk/lib/surface/spanner/migrate/cleanup.py

@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2023 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Cleanup migration resources given a data migration job id."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import textwrap
+
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.spanner import flags
+from googlecloudsdk.command_lib.spanner import migration_backend
+
+
+class Cleanup(base.BinaryBackedCommand):
+  """Cleanup migration resources given a data migration job id."""
+
+  detailed_help = {
+      'EXAMPLES': textwrap.dedent("""\
+        To cleanup resources for a data migration job, specify the jobId and the
+        boolean flags for the resources that need to be cleaned up. For sharded
+        migrations, specific data shard Ids can also be provided.
+
+          For a all shards of a sharded migration, or a non-sharded migration:
+          $ {command} --job-id="XXX" --target-profile="XXX" --datastream --dataflow --pub-sub --monitoring
+
+          For a subset of shards of a sharded migration:
+          $ {command} --job-id="XXX" --data-shard-ids="lorem,epsum" --target-profile="XXX" --datastream --dataflow --pub-sub --monitoring
+
+      """),
+  }
+
+  @staticmethod
+  def Args(parser):
+    """Register the flags for this command."""
+    flags.GetSpannerMigrationJobIdFlag().AddToParser(parser)
+    flags.GetSpannerMigrationDataShardIdsFlag().AddToParser(parser)
+    flags.GetSpannerMigrationTargetProfileFlag().AddToParser(parser)
+    flags.GetSpannerMigrationCleanupDatastreamResourceFlag().AddToParser(parser)
+    flags.GetSpannerMigrationCleanupDataflowResourceFlag().AddToParser(parser)
+    flags.GetSpannerMigrationCleanupPubsubResourceFlag().AddToParser(parser)
+    flags.GetSpannerMigrationCleanupMonitoringResourceFlag().AddToParser(parser)
+    flags.GetSpannerMigrationLogLevelFlag().AddToParser(parser)
+
+  def Run(self, args):
+    """Run the schema-and-data command."""
+    command_executor = migration_backend.SpannerMigrationWrapper()
+    env_vars = migration_backend.GetEnvArgsForCommand(
+        extra_vars={'GCLOUD_HB_PLUGIN': 'true'}
+    )
+    response = command_executor(
+        command='cleanup',
+        job_id=args.job_id,
+        data_shard_ids=args.data_shard_ids,
+        target_profile=args.target_profile,
+        datastream=args.datastream,
+        dataflow=args.dataflow,
+        pub_sub=args.pub_sub,
+        monitoring=args.monitoring,
+        log_level=args.log_level,
+        env=env_vars,
+    )
+    self.exit_code = response.exit_code
+    return self._DefaultOperationResponseHandler(response)

gcloud auth application-default login/google-cloud-sdk/lib/surface/spanner/migrate/data.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2023 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Migrate data from a source database to Cloud Spanner given a schema."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import textwrap
+
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.spanner import flags
+from googlecloudsdk.command_lib.spanner import migration_backend
+
+
+@base.DefaultUniverseOnly
+class Data(base.BinaryBackedCommand):
+  """Migrate data from a source database to Cloud Spanner given a schema."""
+
+  detailed_help = {
+      'EXAMPLES':
+          textwrap.dedent("""\
+        To copy data to Cloud Spanner given a session file:
+
+          $ {command} --session=./session.json --source=postgresql < ~/cart.pg_dump --target-profile='instance=spanner-instance'
+      """),
+  }
+
+  @staticmethod
+  def Args(parser):
+    """Register the flags for this command."""
+    flags.GetSpannerMigrationSourceFlag().AddToParser(parser)
+    flags.GetSpannerMigrationSessionFlag().AddToParser(parser)
+    flags.GetSpannerMigrationPrefixFlag().AddToParser(parser)
+    flags.GetSpannerMigrationSkipForeignKeysFlag().AddToParser(parser)
+    flags.GetSpannerMigrationSourceProfileFlag().AddToParser(parser)
+    flags.GetSpannerMigrationTargetFlag().AddToParser(parser)
+    flags.GetSpannerMigrationTargetProfileFlag().AddToParser(parser)
+    flags.GetSpannerMigrationWriteLimitFlag().AddToParser(parser)
+    flags.GetSpannerMigrationDryRunFlag().AddToParser(parser)
+    flags.GetSpannerMigrationLogLevelFlag().AddToParser(parser)
+    flags.GetSpannerMigrationProjectFlag().AddToParser(parser)
+    flags.GetSpannerMigrationDataflowTemplateFlag().AddToParser(parser)
+
+  def Run(self, args):
+    """Run the data command."""
+    command_executor = migration_backend.SpannerMigrationWrapper()
+    env_vars = migration_backend.GetEnvArgsForCommand(
+        extra_vars={'GCLOUD_HB_PLUGIN': 'true'})
+    response = command_executor(
+        command='data',
+        source=args.source,
+        session=args.session,
+        prefix=args.prefix,
+        skip_foreign_keys=args.skip_foreign_keys,
+        source_profile=args.source_profile,
+        target=args.target,
+        target_profile=args.target_profile,
+        write_limit=args.write_limit,
+        dry_run=args.dry_run,
+        log_level=args.log_level,
+        project=args.project,
+        dataflow_template=args.dataflow_template,
+        env=env_vars,
+    )
+    self.exit_code = response.exit_code
+    return self._DefaultOperationResponseHandler(response)

gcloud auth application-default login/google-cloud-sdk/lib/surface/spanner/migrate/schema.py

@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2023 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Migrate schema from a source database to Cloud Spanner."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import textwrap
+
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.spanner import flags
+from googlecloudsdk.command_lib.spanner import migration_backend
+
+
+@base.DefaultUniverseOnly
+class Schema(base.BinaryBackedCommand):
+  """Migrate schema from a source database to Cloud Spanner."""
+
+  detailed_help = {
+      'EXAMPLES':
+          textwrap.dedent("""\
+        To generate schema file from the source database:
+
+          $ {command} --source=postgresql < ~/cart.pg_dump
+      """),
+  }
+
+  @staticmethod
+  def Args(parser):
+    """Register the flags for this command."""
+    flags.GetSpannerMigrationSourceFlag().AddToParser(parser)
+    flags.GetSpannerMigrationPrefixFlag().AddToParser(parser)
+    flags.GetSpannerMigrationSourceProfileFlag().AddToParser(parser)
+    flags.GetSpannerMigrationTargetFlag().AddToParser(parser)
+    flags.GetSpannerMigrationTargetProfileFlag().AddToParser(parser)
+    flags.GetSpannerMigrationDryRunFlag().AddToParser(parser)
+    flags.GetSpannerMigrationLogLevelFlag().AddToParser(parser)
+    flags.GetSpannerMigrationProjectFlag().AddToParser(parser)
+
+  def Run(self, args):
+    """Run the schema command."""
+    command_executor = migration_backend.SpannerMigrationWrapper()
+    env_vars = migration_backend.GetEnvArgsForCommand(
+        extra_vars={'GCLOUD_HB_PLUGIN': 'true'})
+    response = command_executor(
+        command='schema',
+        source=args.source,
+        prefix=args.prefix,
+        source_profile=args.source_profile,
+        target=args.target,
+        target_profile=args.target_profile,
+        dry_run=args.dry_run,
+        log_level=args.log_level,
+        project=args.project,
+        env=env_vars,
+    )
+    self.exit_code = response.exit_code
+    return self._DefaultOperationResponseHandler(response)

gcloud auth application-default login/google-cloud-sdk/lib/surface/spanner/migrate/schema_and_data.py

@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2023 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Migrate data from a source database to Cloud Spanner given a schema."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import textwrap
+
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.spanner import flags
+from googlecloudsdk.command_lib.spanner import migration_backend
+
+
+@base.DefaultUniverseOnly
+class SchemaAndData(base.BinaryBackedCommand):
+  """Migrate data from a source database to Cloud Spanner given a schema."""
+
+  detailed_help = {
+      'EXAMPLES':
+          textwrap.dedent("""\
+        To generate schema and copy data to Cloud Spanner from a source database:
+
+          $ {command} --source=postgresql < ~/cart.pg_dump --target-profile='instance=spanner-instance'
+      """),
+  }
+
+  @staticmethod
+  def Args(parser):
+    """Register the flags for this command."""
+    flags.GetSpannerMigrationSourceFlag().AddToParser(parser)
+    flags.GetSpannerMigrationPrefixFlag().AddToParser(parser)
+    flags.GetSpannerMigrationSkipForeignKeysFlag().AddToParser(parser)
+    flags.GetSpannerMigrationSourceProfileFlag().AddToParser(parser)
+    flags.GetSpannerMigrationTargetFlag().AddToParser(parser)
+    flags.GetSpannerMigrationTargetProfileFlag().AddToParser(parser)
+    flags.GetSpannerMigrationWriteLimitFlag().AddToParser(parser)
+    flags.GetSpannerMigrationDryRunFlag().AddToParser(parser)
+    flags.GetSpannerMigrationLogLevelFlag().AddToParser(parser)
+    flags.GetSpannerMigrationProjectFlag().AddToParser(parser)
+    flags.GetSpannerMigrationDataflowTemplateFlag().AddToParser(parser)
+
+  def Run(self, args):
+    """Run the schema-and-data command."""
+    command_executor = migration_backend.SpannerMigrationWrapper()
+    env_vars = migration_backend.GetEnvArgsForCommand(
+        extra_vars={'GCLOUD_HB_PLUGIN': 'true'})
+    response = command_executor(
+        command='schema-and-data',
+        source=args.source,
+        prefix=args.prefix,
+        skip_foreign_keys=args.skip_foreign_keys,
+        source_profile=args.source_profile,
+        target=args.target,
+        target_profile=args.target_profile,
+        write_limit=args.write_limit,
+        dry_run=args.dry_run,
+        log_level=args.log_level,
+        project=args.project,
+        dataflow_template=args.dataflow_template,
+        env=env_vars,
+    )
+    self.exit_code = response.exit_code
+    return self._DefaultOperationResponseHandler(response)

gcloud auth application-default login/google-cloud-sdk/lib/surface/spanner/migrate/web.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2023 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Run the web UI assistant for schema migrations."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import textwrap
+
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.spanner import flags
+from googlecloudsdk.command_lib.spanner import migration_backend
+
+
+@base.DefaultUniverseOnly
+class Web(base.BinaryBackedCommand):
+  """Run the web UI assistant for schema migrations."""
+
+  detailed_help = {
+      'EXAMPLES':
+          textwrap.dedent("""\
+        To run the web UI assistant:
+
+          $ {command}
+      """),
+  }
+
+  @staticmethod
+  def Args(parser):
+    """Register the flags for this command."""
+    flags.GetSpannerMigrationWebPortFlag().AddToParser(parser)
+    flags.GetSpannerMigrationWebOpenFlag().AddToParser(parser)
+    flags.GetSpannerMigrationLogLevelFlag().AddToParser(parser)
+    flags.GetSpannerMigrationDataflowTemplateFlag().AddToParser(parser)
+
+  def Run(self, args):
+    """Run the web UI assistant."""
+    command_executor = migration_backend.SpannerMigrationWrapper()
+    env_vars = migration_backend.GetEnvArgsForCommand(
+        extra_vars={'GCLOUD_HB_PLUGIN': 'true'})
+    response = command_executor(
+        command='web',
+        open_flag=args.open,
+        port=args.port,
+        log_level=args.log_level,
+        dataflow_template=args.dataflow_template,
+        env=env_vars,
+    )
+    self.exit_code = response.exit_code
+    return self._DefaultOperationResponseHandler(response)