200 lines
7.3 KiB
Python
200 lines
7.3 KiB
Python
# -*- coding: utf-8 -*- #
|
|
# Copyright 2013 Google LLC. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""Command to list Cloud Storage objects."""
|
|
|
|
from __future__ import absolute_import
|
|
from __future__ import division
|
|
from __future__ import unicode_literals
|
|
|
|
from googlecloudsdk.calliope import base
|
|
from googlecloudsdk.command_lib.storage import copying
|
|
from googlecloudsdk.command_lib.storage import paths
|
|
from googlecloudsdk.command_lib.storage import storage_parallel
|
|
from googlecloudsdk.core import log
|
|
|
|
|
|
@base.Hidden
|
|
@base.ReleaseTracks(base.ReleaseTrack.ALPHA)
|
|
@base.UniverseCompatible
|
|
@base.Deprecate(is_removed=False, warning='This command is deprecated. '
|
|
'Use `gcloud alpha storage cp` instead.')
|
|
class Copy(base.Command):
|
|
"""Upload, download, and copy Cloud Storage objects."""
|
|
|
|
detailed_help = {
|
|
'DESCRIPTION': """\
|
|
Copy files between your local file system and Cloud Storage or from one
|
|
Cloud Storage location to another.
|
|
""",
|
|
'EXAMPLES': """\
|
|
|
|
Uploading files:
|
|
|
|
To upload a single file to a remote location:
|
|
|
|
$ *{command}* path/to/file.txt gs://mybucket/file.txt
|
|
$ *{command}* path/to/file.txt gs://mybucket/
|
|
|
|
The above two commands both create gs://mybucket/file.txt.
|
|
|
|
To upload multiple files to a remote location:
|
|
|
|
$ *{command}* path/to/a.txt other/path/b.txt gs://mybucket/remote-dir/
|
|
|
|
The above command will create gs://mybucket/remote-dir/a.txt and
|
|
gs://mybucket/remote-dir/b.txt. If remote-dir does not exist, this command will create
|
|
remote-dir.
|
|
|
|
To upload a directory my-dir and all its sub-directories and files:
|
|
|
|
$ *{command}* --recursive my-dir gs://mybucket/remote-dir/
|
|
|
|
If my-dir has a subdirectory sub-dir and sub-dir has a file a.txt, the above
|
|
command will create gs://mybucket/remote-dir/my-dir/sub-dir/a.txt. The structure of directory
|
|
is kept.
|
|
|
|
The following command also uploads all files in my-dir and sub-directories recursively:
|
|
|
|
$ *{command}* my-dir/** gs://mybucket/remote-dir/
|
|
|
|
The above command flattens the directory strucutre and creates gs://mybucket/remote-dir/a.txt.
|
|
|
|
To upload all files in a directory, ignoring the subdirectories:
|
|
|
|
$ *{command}* my-dir/* gs://mybucket/remote-dir/
|
|
|
|
If my-dir has a file a.txt and a subdirectory sub-dir. The above command will ceate
|
|
gs://mybucket/remote-dir/a.txt.
|
|
|
|
We can combine the wildcards to upload all text files in a directory and all subdirectories
|
|
recursively:
|
|
|
|
$ *{command}* my-dir/**/*.txt gs://mybucket/remote-dir/
|
|
|
|
Downloading files:
|
|
|
|
To download a single file:
|
|
|
|
$ *{command}* gs://mybucket/file.txt local-dir/
|
|
$ *{command}* gs://mybucket/file.txt local-dir/file.txt
|
|
|
|
The above two commands both create local-dir/file.txt.
|
|
|
|
To download multiple files:
|
|
|
|
$ *{command}* gs://mybucket/a.txt gs://mybucket/b.txt local-dir/
|
|
|
|
The above command creates local-dir/a.txt and local-dir/b.txt.
|
|
|
|
To download a directory and all its sub-directories and files:
|
|
|
|
$ *{command}* --recursive gs://mybucket/remote-dir/ local-dir/
|
|
|
|
The above command creates local-dir/remote-dir/ which contains all files and subdirectories
|
|
of gs://mybucket/remote-dir/. The structure of directory is kept.
|
|
|
|
The following command also downloads all files in gs://mybucket/remote-dir/ to local-dir:
|
|
|
|
$ *{command}* gs://mybucket/remote-dir/** local-dir/
|
|
|
|
If remote-dir contains files a.txt and sub-dir/b.txt, the above command flattens the
|
|
directory structure and creates local-dir/a.txt and local-dir/b.txt.
|
|
|
|
To download all files, ignoring the subdirectories::
|
|
|
|
$ *{command}* gs://mybucket/remote-dir/* local-dir/
|
|
|
|
We can combine the wildcards to download all text files under remote-dir and its
|
|
subdirectories:
|
|
|
|
$ *{command}* gs://mybucket/remote-dir/**/*.txt local-dir/
|
|
|
|
Coping between Cloud Storage locations:
|
|
|
|
To copy a single file to another location:
|
|
|
|
$ *{command}* gs://mybucket/file.txt gs://otherbucket/file.txt
|
|
$ *{command}* gs://mybucket/file.txt gs://otherbucket/
|
|
|
|
The above two commands both create gs://otherbucket/file.txt.
|
|
|
|
To copy multiple files to a new location:
|
|
|
|
$ *{command}* gs://mybucket/a.txt gs://mybucket/b.txt gs://otherbucket/target-dir/
|
|
|
|
The above command creates gs://otherbucket/target-dir/a.txt and
|
|
gs://otherbucket/target-dir/b.txt. If target-dir does not exist, this command will create
|
|
target-dir.
|
|
|
|
To copy all files and subdirectories in one location to another:
|
|
|
|
$ *{command}* --recursive gs://mybucket/source-dir/ gs://otherbucket/target-dir/
|
|
|
|
If source-dir has a subdirectory sub-dir and sub-dir has a file a.txt, the above
|
|
command will create gs://mybucket/target-dir/source-dir/sub-dir/a.txt. The structure of
|
|
directory is kept.
|
|
|
|
The following command also copies all files in source-dir and its sub-directories:
|
|
|
|
$ *{command}* gs://mybucket/source-dir/** gs://mybucket/target-dir/
|
|
|
|
The above command flattens the directory strucutre and creates gs://mybucket/target-dir/a.txt.
|
|
|
|
To copy all files in a directory, ignoring the subdirectories:
|
|
|
|
$ *{command}* gs://mybucket/source-dir/* gs://mybucket/target-dir/
|
|
|
|
If source-dir has a file a.txt and a subdirectory sub-dir. The above command will ceate
|
|
gs://mybucket/target-dir/a.txt.
|
|
|
|
We can combine the wildcards to copy all text in one location and all its sub-directories:
|
|
|
|
$ *{command}* gs://mybucket/source-dir/**/*.txt gs://mybucket/target-dir/
|
|
""",
|
|
}
|
|
|
|
@staticmethod
|
|
def Args(parser):
|
|
parser.add_argument(
|
|
'source',
|
|
nargs='+',
|
|
help='The source file to copy.')
|
|
parser.add_argument(
|
|
'destination',
|
|
help='The destination to copy the file to.')
|
|
parser.add_argument(
|
|
'--recursive',
|
|
action='store_true',
|
|
help='Recursively copy the contents of any directories that match the '
|
|
'path expression.')
|
|
parser.add_argument(
|
|
'--num-threads',
|
|
type=int,
|
|
hidden=True,
|
|
default=16,
|
|
help='The number of threads to use for the copy.')
|
|
|
|
def Run(self, args):
|
|
|
|
sources = [paths.Path(p) for p in args.source]
|
|
dest = paths.Path(args.destination)
|
|
copier = copying.CopyTaskGenerator()
|
|
tasks = copier.GetCopyTasks(sources, dest, recursive=args.recursive)
|
|
storage_parallel.ExecuteTasks(
|
|
tasks, num_threads=args.num_threads, progress_bar_label='Copying Files')
|
|
log.status.write('Copied [{}] file{}.\n'.format(
|
|
len(tasks), 's' if len(tasks) > 1 else ''))
|