feat: Add new gcloud commands, API clients, and third-party libraries across various services.

gcloud auth application-default login/google-cloud-sdk/lib/surface/composer/environments/storage/__init__.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2018 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""The command group for managing a Cloud Composer environment's storage."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+
+
+class Storage(base.Group):
+  """Manage Cloud Storage objects stored as part of Cloud Composer environments.
+
+  The {command} command group lets you manage DAGs, Airflow plugins and data
+  used inside your Cloud Composer Environments.
+  """
+
+Storage.category = base.COMPOSER_CATEGORY

gcloud auth application-default login/google-cloud-sdk/lib/surface/composer/environments/storage/dags/__init__.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2018 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""The command group for managing a Cloud Composer environment's DAGs."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+
+
+class Dags(base.Group):
+  """Manage DAGs for Cloud Composer environments.
+
+  The {command} command group lets you import, export, list, and delete DAGs for
+  your Cloud Composer environment.
+  """

gcloud auth application-default login/google-cloud-sdk/lib/surface/composer/environments/storage/dags/delete.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2018 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command to delete Airflow DAGs for a Cloud Composer environment."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.composer import flags
+from googlecloudsdk.command_lib.composer import resource_args
+from googlecloudsdk.command_lib.composer import storage_util
+from googlecloudsdk.core.console import console_io
+
+
+DETAILED_HELP = {
+    'EXAMPLES':
+        '''\
+          To delete the dags in the path ``path/to/dags'', for the
+          environment named ``environment-1'' in the location ``us-east1'', run:
+
+            $ {command} path/to/dags --environment=environment-1 --location=us-east1
+        '''
+}
+
+
+class Delete(base.Command):
+  """Delete DAG files from an Cloud Composer environment's Cloud Storage bucket.
+  """
+
+  detailed_help = DETAILED_HELP
+
+  @staticmethod
+  def Args(parser):
+    resource_args.AddEnvironmentResourceArg(
+        parser, 'whose DAGs to delete', positional=False)
+    flags.AddDeleteTargetPositional(parser, 'dags')
+
+  def Run(self, args):
+    env_ref = args.CONCEPTS.environment.Parse()
+    subtarget = '[{}] in '.format(args.target) if args.target else ''
+    console_io.PromptContinue(
+        'Recursively deleting all contents from {}the \'dags/\' '
+        'subdirectory of environment [{}]'.format(subtarget,
+                                                  env_ref.RelativeName()),
+        cancel_on_no=True)
+    return storage_util.Delete(
+        env_ref, args.target or '*', 'dags', release_track=self.ReleaseTrack())

gcloud auth application-default login/google-cloud-sdk/lib/surface/composer/environments/storage/dags/export.py

@@ -0,0 +1,129 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2018 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command to export files into a Cloud Composer environment's bucket."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import posixpath
+
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.composer import flags
+from googlecloudsdk.command_lib.composer import resource_args
+from googlecloudsdk.command_lib.composer import storage_util
+
+
+class Export(base.Command):
+  """Export DAGs from an environment into local storage or Cloud Storage.
+
+  If the SOURCE is a directory, it and its contents are are exported
+  recursively. If no SOURCE is provided, the entire contents of the
+  environment's DAGs directory will be exported. Colliding files in the
+  DESTINATION will be overwritten. If a file exists in the DESTINATION but
+  there is no corresponding file to overwrite it, it is untouched.
+
+  ## EXAMPLES
+  Suppose the environment `myenv`'s Cloud Storage bucket has the following
+  structure:
+
+    gs://the-bucket
+    |
+    +-- dags
+    |   |
+    |   +-- file1.py
+    |   +-- file2.py
+    |   |
+    |   +-- subdir1
+    |   |   |
+    |   |   +-- file3.py
+    |   |   +-- file4.py
+
+  And the local directory '/foo' has the following
+  structure:
+
+    /foo
+    |
+    +-- file1.py
+    +-- fileX.py
+    |   |
+    |   +-- subdir1
+    |   |   |
+    |   |   +-- file3.py
+    |   |   +-- fileY.py
+
+  The following command:
+
+    {command} myenv --destination=/foo
+
+  would result in the following structure in the local '/foo' directory:
+
+    /foo
+    |
+    +-- file1.py
+    +-- file2.py
+    +-- fileX.py
+    |   |
+    |   +-- subdir1
+    |   |   |
+    |   |   +-- file3.py
+    |   |   +-- file4.py
+    |   |   +-- fileY.py
+
+  The local files '/foo/file1.py' and '/foo/subdir1/file3.py' will be
+  overwritten with the contents of the corresponding files in the Cloud Storage
+  bucket.
+
+  If instead we had run
+
+    {command} myenv --source=subdir1/file3.py --destination=/foo
+
+  the resulting local directory structure would be the following:
+
+    /foo
+    |
+    +-- file1.py
+    +-- file3.py
+    +-- fileX.py
+    |   |
+    |   +-- subdir1
+    |   |   |
+    |   |   +-- file3.py
+    |   |   +-- fileY.py
+
+  No local files would be overwritten since
+  'gs://the-bucket/dags/subdir1/file3.py' was written to '/foo/file3.py'
+  instead of 'foo/subdir1/file3.py'.
+  """
+
+  SUBDIR_BASE = 'dags'
+
+  @staticmethod
+  def Args(parser):
+    resource_args.AddEnvironmentResourceArg(
+        parser, 'from whose Cloud Storage bucket to export DAGs',
+        positional=False)
+    flags.AddExportSourceFlag(parser, Export.SUBDIR_BASE)
+    flags.AddExportDestinationFlag(parser)
+
+  def Run(self, args):
+    storage_util.WarnIfWildcardIsPresent(args.source, '--source')
+    env_ref = args.CONCEPTS.environment.Parse()
+    source_path = posixpath.join(Export.SUBDIR_BASE,
+                                 (args.source or '*').strip(posixpath.sep))
+    return storage_util.Export(
+        env_ref, source_path,
+        args.destination,
+        release_track=self.ReleaseTrack())

gcloud auth application-default login/google-cloud-sdk/lib/surface/composer/environments/storage/dags/import.py

@@ -0,0 +1,139 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2018 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command to import files into a Cloud Composer environment's bucket."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import posixpath
+
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.composer import flags
+from googlecloudsdk.command_lib.composer import resource_args
+from googlecloudsdk.command_lib.composer import storage_util
+
+
+class Import(base.Command):
+  """Import DAGs from local storage or Cloud Storage into an environment.
+
+  If the SOURCE is a directory, it and its contents are imported recursively.
+  Colliding files in the environment's Cloud Storage bucket will be
+  overwritten. If a file exists in the bucket but is not present in the SOURCE,
+  it is not removed.
+
+  ## EXAMPLES
+  Suppose the '/foo' directory in the local filesystem has the following
+  structure:
+
+    foo
+    |
+    +-- subdir1
+    |   |
+    |   +-- file1.txt
+    |   +-- file2.txt
+    |
+    +-- subdir2
+    |   |
+    |   +-- file3.txt
+    |   +-- file4.txt
+
+  And the environment `myenv`'s Cloud Storage bucket has the following
+  structure:
+
+    gs://the-bucket
+    |
+    +-- dags
+    |   |
+    |   +-- foo
+    |   |   |
+    |   |   +-- subdir1
+    |   |   |   |
+    |   |   |   +-- bar.txt
+
+  The following command:
+
+    {command} myenv --source=/foo
+
+  would result in the following structure in `myenv`'s Cloud Storage bucket:
+
+    gs://the-bucket
+    |
+    +-- dags
+    |   |
+    |   +-- foo
+    |   |   |
+    |   |   +-- subdir1
+    |   |   |   |
+    |   |   |   +-- bar.txt
+    |   |   |   +-- file1.txt
+    |   |   |   +-- file2.txt
+    |   |   |
+    |   |   +-- subdir2
+    |   |   |   |
+    |   |   |   +-- file3.txt
+    |   |   |   +-- file4.txt
+
+  If instead we had run
+
+    {command} myenv --source=/foo --destination=bar
+
+  the resulting bucket structure would be the following:
+
+    gs://the-bucket
+    |
+    +-- dags
+    |   |
+    |   +-- foo
+    |   |   |
+    |   |   +-- subdir1
+    |   |   |   |
+    |   |   |   +-- bar.txt
+    |   |
+    |   +-- bar
+    |   |   |
+    |   |   +-- foo
+    |   |   |   |
+    |   |   |   +-- subdir1
+    |   |   |   |   |
+    |   |   |   |   +-- file1.txt
+    |   |   |   |   +-- file2.txt
+    |   |   |   |
+    |   |   |   +-- subdir2
+    |   |   |   |   |
+    |   |   |   |   +-- file3.txt
+    |   |   |   |   +-- file4.txt
+  """
+
+  SUBDIR_BASE = 'dags'
+
+  @staticmethod
+  def Args(parser):
+    resource_args.AddEnvironmentResourceArg(
+        parser, 'into whose Cloud Storage bucket to import DAGs',
+        positional=False)
+    flags.AddImportSourceFlag(parser, Import.SUBDIR_BASE)
+    flags.AddImportDestinationFlag(parser, Import.SUBDIR_BASE)
+
+  def Run(self, args):
+    storage_util.WarnIfWildcardIsPresent(args.source, '--source')
+    env_ref = args.CONCEPTS.environment.Parse()
+    gcs_subdir = Import.SUBDIR_BASE
+    if args.destination:
+      gcs_subdir = posixpath.join(gcs_subdir,
+                                  args.destination.strip(posixpath.sep))
+    gcs_subdir = posixpath.join(gcs_subdir, '')
+    return storage_util.Import(
+        env_ref, args.source, gcs_subdir, release_track=self.ReleaseTrack())

gcloud auth application-default login/google-cloud-sdk/lib/surface/composer/environments/storage/dags/list.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2018 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command to list Airflow DAGs for a Cloud Composer environment."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.composer import resource_args
+from googlecloudsdk.command_lib.composer import storage_util
+
+
+DETAILED_HELP = {
+    'EXAMPLES':
+        """\
+        To list the dags for the Cloud Composer environment
+        ``environment-1'' and location ``us-central1'', run:
+
+          $ {command} --environment=environment-1 --location=us-central1
+        """
+}
+
+
+class List(base.Command):
+  """List the DAG files for a Cloud Composer environment."""
+
+  detailed_help = DETAILED_HELP
+
+  @staticmethod
+  def Args(parser):
+    resource_args.AddEnvironmentResourceArg(
+        parser, 'for which to list DAGs', positional=False)
+    parser.display_info.AddFormat('table(name)')
+
+  def Run(self, args):
+    env_ref = args.CONCEPTS.environment.Parse()
+    return storage_util.List(env_ref, 'dags', release_track=self.ReleaseTrack())

gcloud auth application-default login/google-cloud-sdk/lib/surface/composer/environments/storage/data/__init__.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2018 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""The command group for managing a Cloud Composer environment's data."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+
+
+class Data(base.Group):
+  """Manage data for Cloud Composer environments.
+
+  The {command} command group lets you import, export, list, and delete data for
+  your Cloud Composer environment.
+  """

gcloud auth application-default login/google-cloud-sdk/lib/surface/composer/environments/storage/data/delete.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2018 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command to delete Airflow data for a Cloud Composer environment."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.composer import flags
+from googlecloudsdk.command_lib.composer import resource_args
+from googlecloudsdk.command_lib.composer import storage_util
+from googlecloudsdk.core.console import console_io
+
+
+DETAILED_HELP = {
+    'EXAMPLES':
+        '''\
+          To delete the data from the path ``path/to/data'', for the
+          environment named ``environment-1'' in the location ``us-east1'', run:
+
+            $ {command} path/to/data --environment=environment-1 --location=us-east1
+        '''
+}
+
+
+class Delete(base.Command):
+  """Delete data from an Cloud Composer environment's Cloud Storage bucket.
+  """
+
+  detailed_help = DETAILED_HELP
+
+  @staticmethod
+  def Args(parser):
+    resource_args.AddEnvironmentResourceArg(
+        parser, 'whose data to delete.', positional=False)
+    flags.AddDeleteTargetPositional(parser, 'data')
+
+  def Run(self, args):
+    env_ref = args.CONCEPTS.environment.Parse()
+    subtarget = '[{}] in '.format(args.target) if args.target else ''
+    console_io.PromptContinue(
+        'Recursively deleting all contents from {}the \'data/\' '
+        'subdirectory of environment [{}]'.format(subtarget,
+                                                  env_ref.RelativeName()),
+        cancel_on_no=True)
+    return storage_util.Delete(
+        env_ref, args.target or '*', 'data', release_track=self.ReleaseTrack())

gcloud auth application-default login/google-cloud-sdk/lib/surface/composer/environments/storage/data/export.py

@@ -0,0 +1,129 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2018 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command to export files into a Cloud Composer environment's bucket."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import posixpath
+
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.composer import flags
+from googlecloudsdk.command_lib.composer import resource_args
+from googlecloudsdk.command_lib.composer import storage_util
+
+
+class Export(base.Command):
+  """Export data from an environment into local storage or Cloud Storage.
+
+  If the SOURCE is a directory, it and its contents are are exported
+  recursively. If no SOURCE is provided, the entire contents of the
+  environment's data directory will be exported. Colliding files in the
+  DESTINATION will be overwritten. If a file exists in the DESTINATION but
+  there is no corresponding file to overwrite it, it is untouched.
+
+  ## EXAMPLES
+  Suppose the environment `myenv`'s Cloud Storage bucket has the following
+  structure:
+
+    gs://the-bucket
+    |
+    +-- data
+    |   |
+    |   +-- file1.txt
+    |   +-- file2.txt
+    |   |
+    |   +-- subdir1
+    |   |   |
+    |   |   +-- file3.txt
+    |   |   +-- file4.txt
+
+  And the local directory '/foo' has the following
+  structure:
+
+    /foo
+    |
+    +-- file1.txt
+    +-- fileX.txt
+    |   |
+    |   +-- subdir1
+    |   |   |
+    |   |   +-- file3.txt
+    |   |   +-- fileY.txt
+
+  The following command:
+
+    {command} myenv --destination=/foo
+
+  would result in the following structure in the local '/foo' directory:
+
+    /foo
+    |
+    +-- file1.txt
+    +-- file2.txt
+    +-- fileX.txt
+    |   |
+    |   +-- subdir1
+    |   |   |
+    |   |   +-- file3.txt
+    |   |   +-- file4.txt
+    |   |   +-- fileY.txt
+
+  The local files '/foo/file1.txt' and '/foo/subdir1/file3.txt' will be
+  overwritten with the contents of the corresponding files in the Cloud Storage
+  bucket.
+
+  If instead we had run
+
+    {command} myenv --source=subdir1/file3.txt --destination=/foo
+
+  the resulting local directory structure would be the following:
+
+    /foo
+    |
+    +-- file1.txt
+    +-- file3.txt
+    +-- fileX.txt
+    |   |
+    |   +-- subdir1
+    |   |   |
+    |   |   +-- file3.txt
+    |   |   +-- fileY.txt
+
+  No local files would be overwritten since
+  'gs://the-bucket/dags/subdir1/file3.txt' was written to '/foo/file3.txt'
+  instead of 'foo/subdir1/file3.txt'.
+  """
+
+  SUBDIR_BASE = 'data'
+
+  @staticmethod
+  def Args(parser):
+    resource_args.AddEnvironmentResourceArg(
+        parser, 'from whose Cloud Storage bucket to export data.',
+        positional=False)
+    flags.AddExportSourceFlag(parser, Export.SUBDIR_BASE)
+    flags.AddExportDestinationFlag(parser)
+
+  def Run(self, args):
+    storage_util.WarnIfWildcardIsPresent(args.source, '--source')
+    env_ref = args.CONCEPTS.environment.Parse()
+    source_path = posixpath.join(Export.SUBDIR_BASE,
+                                 (args.source or '*').strip(posixpath.sep))
+    return storage_util.Export(
+        env_ref, source_path,
+        args.destination,
+        release_track=self.ReleaseTrack())

gcloud auth application-default login/google-cloud-sdk/lib/surface/composer/environments/storage/data/import.py

@@ -0,0 +1,139 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2018 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command to import files into a Cloud Composer environment's bucket."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import posixpath
+
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.composer import flags
+from googlecloudsdk.command_lib.composer import resource_args
+from googlecloudsdk.command_lib.composer import storage_util
+
+
+class Import(base.Command):
+  """Import data from local storage or Cloud Storage into an environment.
+
+  If the SOURCE is a directory, it and its contents are imported recursively.
+  Colliding files in the environment's Cloud Storage bucket will be
+  overwritten. If a file exists in the bucket but is not present in the SOURCE,
+  it is not removed.
+
+  ## EXAMPLES
+  Suppose the '/foo' directory in the local filesystem has the following
+  structure:
+
+    foo
+    |
+    +-- subdir1
+    |   |
+    |   +-- file1.txt
+    |   +-- file2.txt
+    |
+    +-- subdir2
+    |   |
+    |   +-- file3.txt
+    |   +-- file4.txt
+
+  And the environment `myenv`'s Cloud Storage bucket has the following
+  structure:
+
+    gs://the-bucket
+    |
+    +-- data
+    |   |
+    |   +-- foo
+    |   |   |
+    |   |   +-- subdir1
+    |   |   |   |
+    |   |   |   +-- bar.txt
+
+  The following command:
+
+    {command} myenv --source=/foo
+
+  would result in the following structure in `myenv`'s Cloud Storage bucket:
+
+    gs://the-bucket
+    |
+    +-- data
+    |   |
+    |   +-- foo
+    |   |   |
+    |   |   +-- subdir1
+    |   |   |   |
+    |   |   |   +-- bar.txt
+    |   |   |   +-- file1.txt
+    |   |   |   +-- file2.txt
+    |   |   |
+    |   |   +-- subdir2
+    |   |   |   |
+    |   |   |   +-- file3.txt
+    |   |   |   +-- file4.txt
+
+  If instead we had run
+
+    {command} myenv --source=/foo --destination=bar
+
+  the resulting bucket structure would be the following:
+
+    gs://the-bucket
+    |
+    +-- data
+    |   |
+    |   +-- foo
+    |   |   |
+    |   |   +-- subdir1
+    |   |   |   |
+    |   |   |   +-- bar.txt
+    |   |
+    |   +-- bar
+    |   |   |
+    |   |   +-- foo
+    |   |   |   |
+    |   |   |   +-- subdir1
+    |   |   |   |   |
+    |   |   |   |   +-- file1.txt
+    |   |   |   |   +-- file2.txt
+    |   |   |   |
+    |   |   |   +-- subdir2
+    |   |   |   |   |
+    |   |   |   |   +-- file3.txt
+    |   |   |   |   +-- file4.txt
+  """
+
+  SUBDIR_BASE = 'data'
+
+  @staticmethod
+  def Args(parser):
+    resource_args.AddEnvironmentResourceArg(
+        parser, 'into whose Cloud Storage bucket to import data.',
+        positional=False)
+    flags.AddImportSourceFlag(parser, Import.SUBDIR_BASE)
+    flags.AddImportDestinationFlag(parser, Import.SUBDIR_BASE)
+
+  def Run(self, args):
+    storage_util.WarnIfWildcardIsPresent(args.source, '--source')
+    env_ref = args.CONCEPTS.environment.Parse()
+    gcs_subdir = Import.SUBDIR_BASE
+    if args.destination:
+      gcs_subdir = posixpath.join(gcs_subdir,
+                                  args.destination.strip(posixpath.sep))
+    gcs_subdir = posixpath.join(gcs_subdir, '')
+    return storage_util.Import(
+        env_ref, args.source, gcs_subdir, release_track=self.ReleaseTrack())

gcloud auth application-default login/google-cloud-sdk/lib/surface/composer/environments/storage/data/list.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2018 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command to list data for a Cloud Composer environment."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.composer import resource_args
+from googlecloudsdk.command_lib.composer import storage_util
+
+
+DETAILED_HELP = {
+    'EXAMPLES':
+        """\
+        To list the data from the Cloud Composer environment
+        ``environment-1'' and location ``us-central1'', run:
+
+          $ {command} --environment=environment-1 --location=us-central1
+        """
+}
+
+
+class List(base.Command):
+  """List the data for a Cloud Composer environment."""
+
+  detailed_help = DETAILED_HELP
+
+  @staticmethod
+  def Args(parser):
+    resource_args.AddEnvironmentResourceArg(
+        parser, 'for which to list data.', positional=False)
+    parser.display_info.AddFormat('table(name)')
+
+  def Run(self, args):
+    env_ref = args.CONCEPTS.environment.Parse()
+    return storage_util.List(env_ref, 'data', release_track=self.ReleaseTrack())

gcloud auth application-default login/google-cloud-sdk/lib/surface/composer/environments/storage/plugins/__init__.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2018 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""The command group for managing a Cloud Composer environment's plugins."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+
+
+class Plugins(base.Group):
+  """Manage plugins for Cloud Composer environments.
+
+  The {command} command group lets you import, export, list, and delete plugins
+  for your Cloud Composer environment.
+  """

gcloud auth application-default login/google-cloud-sdk/lib/surface/composer/environments/storage/plugins/delete.py

@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2018 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command to delete Airflow plugins for a Cloud Composer environment."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.composer import flags
+from googlecloudsdk.command_lib.composer import resource_args
+from googlecloudsdk.command_lib.composer import storage_util
+from googlecloudsdk.core.console import console_io
+
+
+DETAILED_HELP = {
+    'EXAMPLES':
+        '''\
+          To delete the plugins in the path ``path/to/plugins'', for the
+          environment named ``environment-1'' in the location ``us-east1'', run:
+
+            $ {command} path/to/plugins --environment=environment-1 --location=us-east1
+        '''
+}
+
+
+class Delete(base.Command):
+  """Delete plugins from an Cloud Composer environment's Cloud Storage bucket.
+  """
+
+  detailed_help = DETAILED_HELP
+
+  @staticmethod
+  def Args(parser):
+    resource_args.AddEnvironmentResourceArg(
+        parser, 'whose plugins to delete.', positional=False)
+    flags.AddDeleteTargetPositional(parser, 'plugins')
+
+  def Run(self, args):
+    env_ref = args.CONCEPTS.environment.Parse()
+    subtarget = '[{}] in '.format(args.target) if args.target else ''
+    console_io.PromptContinue(
+        'Recursively deleting all contents from {}the \'plugins/\' '
+        'subdirectory of environment [{}]'.format(subtarget,
+                                                  env_ref.RelativeName()),
+        cancel_on_no=True)
+    return storage_util.Delete(
+        env_ref,
+        args.target or '*',
+        'plugins',
+        release_track=self.ReleaseTrack())

gcloud auth application-default login/google-cloud-sdk/lib/surface/composer/environments/storage/plugins/export.py

@@ -0,0 +1,129 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2018 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command to export files into a Cloud Composer environment's bucket."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import posixpath
+
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.composer import flags
+from googlecloudsdk.command_lib.composer import resource_args
+from googlecloudsdk.command_lib.composer import storage_util
+
+
+class Export(base.Command):
+  """Export plugins from an environment into local storage or Cloud Storage.
+
+  If the SOURCE is a directory, it and its contents are are exported
+  recursively. If no SOURCE is provided, the entire contents of the
+  environment's plugins directory will be exported. Colliding files in the
+  DESTINATION will be overwritten. If a file exists in the DESTINATION but
+  there is no corresponding file to overwrite it, it is untouched.
+
+  ## EXAMPLES
+  Suppose the environment `myenv`'s Cloud Storage bucket has the following
+  structure:
+
+    gs://the-bucket
+    |
+    +-- plugins
+    |   |
+    |   +-- file1.py
+    |   +-- file2.py
+    |   |
+    |   +-- subdir1
+    |   |   |
+    |   |   +-- file3.py
+    |   |   +-- file4.py
+
+  And the local directory '/foo' has the following
+  structure:
+
+    /foo
+    |
+    +-- file1.py
+    +-- fileX.py
+    |   |
+    |   +-- subdir1
+    |   |   |
+    |   |   +-- file3.py
+    |   |   +-- fileY.py
+
+  The following command:
+
+    {command} myenv --destination=/foo
+
+  would result in the following structure in the local '/foo' directory:
+
+    /foo
+    |
+    +-- file1.py
+    +-- file2.py
+    +-- fileX.py
+    |   |
+    |   +-- subdir1
+    |   |   |
+    |   |   +-- file3.py
+    |   |   +-- file4.py
+    |   |   +-- fileY.py
+
+  The local files '/foo/file1.py' and '/foo/subdir1/file3.py' will be
+  overwritten with the contents of the corresponding files in the Cloud Storage
+  bucket.
+
+  If instead we had run
+
+    {command} myenv file2.py subdir1/file3.py --destination=/foo
+
+  the resulting local directory structure would be the following:
+
+    /foo
+    |
+    +-- file1.py
+    +-- file3.py
+    +-- fileX.py
+    |   |
+    |   +-- subdir1
+    |   |   |
+    |   |   +-- file3.py
+    |   |   +-- fileY.py
+
+  No local files would be overwritten since
+  'gs://the-bucket/dags/subdir1/file3.py' was written to '/foo/file3.py'
+  instead of 'foo/subdir1/file3.py'.
+  """
+
+  SUBDIR_BASE = 'plugins'
+
+  @staticmethod
+  def Args(parser):
+    resource_args.AddEnvironmentResourceArg(
+        parser, 'from whose Cloud Storage bucket to export plugins.',
+        positional=False)
+    flags.AddExportSourceFlag(parser, Export.SUBDIR_BASE)
+    flags.AddExportDestinationFlag(parser)
+
+  def Run(self, args):
+    storage_util.WarnIfWildcardIsPresent(args.source, '--source')
+    env_ref = args.CONCEPTS.environment.Parse()
+    source_path = posixpath.join(Export.SUBDIR_BASE,
+                                 (args.source or '*').strip(posixpath.sep))
+    return storage_util.Export(
+        env_ref, source_path,
+        args.destination,
+        release_track=self.ReleaseTrack())

gcloud auth application-default login/google-cloud-sdk/lib/surface/composer/environments/storage/plugins/import.py

@@ -0,0 +1,139 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2018 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command to import files into a Cloud Composer environment's bucket."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import posixpath
+
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.composer import flags
+from googlecloudsdk.command_lib.composer import resource_args
+from googlecloudsdk.command_lib.composer import storage_util
+
+
+class Import(base.Command):
+  """Import plugins from local storage or Cloud Storage into an environment.
+
+  If the SOURCE is a directory, it and its contents are imported recursively.
+  Colliding files in the environment's Cloud Storage bucket will be
+  overwritten. If a file exists in the bucket but is not present in the SOURCE,
+  it is not removed.
+
+  ## EXAMPLES
+  Suppose the '/foo' directory in the local filesystem has the following
+  structure:
+
+    foo
+    |
+    +-- subdir1
+    |   |
+    |   +-- file1.txt
+    |   +-- file2.txt
+    |
+    +-- subdir2
+    |   |
+    |   +-- file3.txt
+    |   +-- file4.txt
+
+  And the environment `myenv`'s Cloud Storage bucket has the following
+  structure:
+
+    gs://the-bucket
+    |
+    +-- plugins
+    |   |
+    |   +-- foo
+    |   |   |
+    |   |   +-- subdir1
+    |   |   |   |
+    |   |   |   +-- bar.txt
+
+  The following command:
+
+    {command} myenv --source=/foo
+
+  would result in the following structure in `myenv`'s Cloud Storage bucket:
+
+    gs://the-bucket
+    |
+    +-- plugins
+    |   |
+    |   +-- foo
+    |   |   |
+    |   |   +-- subdir1
+    |   |   |   |
+    |   |   |   +-- bar.txt
+    |   |   |   +-- file1.txt
+    |   |   |   +-- file2.txt
+    |   |   |
+    |   |   +-- subdir2
+    |   |   |   |
+    |   |   |   +-- file3.txt
+    |   |   |   +-- file4.txt
+
+  If instead we had run
+
+    {command} myenv --source=/foo --destination=bar
+
+  the resulting bucket structure would be the following:
+
+    gs://the-bucket
+    |
+    +-- plugins
+    |   |
+    |   +-- foo
+    |   |   |
+    |   |   +-- subdir1
+    |   |   |   |
+    |   |   |   +-- bar.txt
+    |   |
+    |   +-- bar
+    |   |   |
+    |   |   +-- foo
+    |   |   |   |
+    |   |   |   +-- subdir1
+    |   |   |   |   |
+    |   |   |   |   +-- file1.txt
+    |   |   |   |   +-- file2.txt
+    |   |   |   |
+    |   |   |   +-- subdir2
+    |   |   |   |   |
+    |   |   |   |   +-- file3.txt
+    |   |   |   |   +-- file4.txt
+  """
+
+  SUBDIR_BASE = 'plugins'
+
+  @staticmethod
+  def Args(parser):
+    resource_args.AddEnvironmentResourceArg(
+        parser, 'into whose Cloud Storage bucket to import plugins.',
+        positional=False)
+    flags.AddImportSourceFlag(parser, Import.SUBDIR_BASE)
+    flags.AddImportDestinationFlag(parser, Import.SUBDIR_BASE)
+
+  def Run(self, args):
+    storage_util.WarnIfWildcardIsPresent(args.source, '--source')
+    env_ref = args.CONCEPTS.environment.Parse()
+    gcs_subdir = Import.SUBDIR_BASE
+    if args.destination:
+      gcs_subdir = posixpath.join(gcs_subdir,
+                                  args.destination.strip(posixpath.sep))
+    gcs_subdir = posixpath.join(gcs_subdir, '')
+    return storage_util.Import(
+        env_ref, args.source, gcs_subdir, release_track=self.ReleaseTrack())

gcloud auth application-default login/google-cloud-sdk/lib/surface/composer/environments/storage/plugins/list.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2018 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command to list Airflow plugins for a Cloud Composer environment."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.composer import resource_args
+from googlecloudsdk.command_lib.composer import storage_util
+
+DETAILED_HELP = {
+    'EXAMPLES':
+        """\
+        To list the plugins for the Cloud Composer environment
+        ``environment-1'' and location ``us-central1'', run:
+
+          $ {command} --environment=environment-1 --location=us-central1
+        """
+}
+
+
+class List(base.Command):
+  """List the plugins for a Cloud Composer environment."""
+
+  detailed_help = DETAILED_HELP
+
+  @staticmethod
+  def Args(parser):
+    resource_args.AddEnvironmentResourceArg(
+        parser, 'for which to list plugins.', positional=False)
+    parser.display_info.AddFormat('table(name)')
+
+  def Run(self, args):
+    env_ref = args.CONCEPTS.environment.Parse()
+    return storage_util.List(
+        env_ref, 'plugins', release_track=self.ReleaseTrack())