795 lines
34 KiB
Python
795 lines
34 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Copyright 2013 Google Inc. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""Integration tests for rm command."""
|
|
|
|
from __future__ import absolute_import
|
|
from __future__ import print_function
|
|
from __future__ import division
|
|
from __future__ import unicode_literals
|
|
|
|
import os
|
|
import re
|
|
import sys
|
|
from unittest import mock
|
|
|
|
from gslib.exception import NO_URLS_MATCHED_PREFIX
|
|
from gslib.exception import NO_URLS_MATCHED_TARGET
|
|
import gslib.tests.testcase as testcase
|
|
from gslib.tests.testcase.base import MAX_BUCKET_LENGTH
|
|
from gslib.tests.testcase.integration_testcase import SkipForS3
|
|
import gslib.tests.util as util
|
|
from gslib.tests.util import GenerationFromURI as urigen
|
|
from gslib.tests.util import ObjectToURI as suri
|
|
from gslib.tests.util import SetBotoConfigForTest
|
|
from gslib.tests.util import SetEnvironmentForTest
|
|
from gslib.utils import shim_util
|
|
from gslib.utils.retry_util import Retry
|
|
|
|
MACOS_WARNING = (
|
|
'If you experience problems with multiprocessing on MacOS, they might be '
|
|
'related to https://bugs.python.org/issue33725. You can disable '
|
|
'multiprocessing by editing your .boto config or by adding the following '
|
|
'flag to your command: `-o "GSUtil:parallel_process_count=1"`. Note that '
|
|
'multithreading is still available even if you disable multiprocessing.')
|
|
|
|
|
|
class TestRm(testcase.GsUtilIntegrationTestCase):
|
|
"""Integration tests for rm command."""
|
|
|
|
def _CleanRmUiOutputBeforeChecking(self, stderr):
|
|
"""Excludes everything coming from the UI to avoid assert errors.
|
|
|
|
Args:
|
|
stderr: The cumulative stderr output.
|
|
Returns:
|
|
The cumulative stderr output without the expected UI output.
|
|
"""
|
|
if self._use_gcloud_storage:
|
|
return self._CleanOutputLinesForGcloudStorage(stderr)
|
|
ui_output_pattern = '[^\n\r]*objects][^\n\r]*[\n\r]'
|
|
final_message_pattern = 'Operation completed over[^\n]*'
|
|
ui_spinner_list = ['\\\r', '|\r', '/\r', '-\r']
|
|
ui_lines_list = (re.findall(ui_output_pattern, stderr) +
|
|
re.findall(final_message_pattern, stderr) +
|
|
ui_spinner_list)
|
|
for ui_line in ui_lines_list:
|
|
stderr = stderr.replace(ui_line, '')
|
|
return stderr
|
|
|
|
def _CleanOutputLinesForGcloudStorage(self, stderr):
|
|
"""Remove irrelevant lines from the output lines."""
|
|
stderr_lines = stderr.splitlines()
|
|
valid_lines = []
|
|
strings_to_remove = set(['Removing objects:', 'Removing Buckets:', ' '])
|
|
for line in stderr_lines:
|
|
if line in strings_to_remove:
|
|
continue
|
|
# Gcloud storage logs a '.' for each resource in non-interactive mode.
|
|
# This might get added to the start of a valid line.
|
|
# e.g. ".Removing gs://bucket/obj"
|
|
cleaned_line = line.lstrip('.')
|
|
if cleaned_line:
|
|
valid_lines.append(cleaned_line)
|
|
return os.linesep.join(valid_lines)
|
|
|
|
def _RunRemoveCommandAndCheck(self,
|
|
command_and_args,
|
|
objects_to_remove=None,
|
|
buckets_to_remove=None,
|
|
stdin=None):
|
|
"""Tests a remove command in the presence of eventual listing consistency.
|
|
|
|
Eventual listing consistency means that a remove command may not see all
|
|
of the objects to be removed at once. When removing multiple objects
|
|
(or buckets via -r), some calls may return no matches and multiple calls
|
|
to the rm command may be necessary to reach the desired state. This function
|
|
retries the rm command, incrementally tracking what has been removed and
|
|
ensuring that the exact set of objects/buckets are removed across all
|
|
retried calls.
|
|
|
|
The caller is responsible for confirming the existence of buckets/objects
|
|
prior to calling this function.
|
|
|
|
Args:
|
|
command_and_args: List of strings representing the rm command+args to run.
|
|
objects_to_remove: List of object URL strings (optionally including
|
|
generation) that should be removed by the command, if any.
|
|
buckets_to_remove: List of bucket URL strings that should be removed by
|
|
the command, if any.
|
|
stdin: String of data to pipe to the process as standard input (for
|
|
testing -I option).
|
|
"""
|
|
bucket_strings = []
|
|
for bucket_to_remove in buckets_to_remove or []:
|
|
bucket_strings.append('Removing %s/...' % bucket_to_remove)
|
|
object_strings = []
|
|
for object_to_remove in objects_to_remove or []:
|
|
object_strings.append('Removing %s...' % object_to_remove)
|
|
expected_stderr_lines = set(object_strings + bucket_strings)
|
|
|
|
if not self.multiregional_buckets and self.default_provider == 'gs':
|
|
stderr = self.RunGsUtil(command_and_args,
|
|
return_stderr=True,
|
|
expected_status=None,
|
|
stdin=stdin)
|
|
num_objects = len(object_strings)
|
|
# Asserting for operation completion
|
|
if '-q' not in command_and_args and not self._use_gcloud_storage:
|
|
if '-m' in command_and_args:
|
|
self.assertIn('[%d/%d objects]' % (num_objects, num_objects), stderr)
|
|
else:
|
|
self.assertIn('[%d objects]' % num_objects, stderr)
|
|
|
|
stderr = self._CleanRmUiOutputBeforeChecking(stderr)
|
|
stderr_set = set(stderr.splitlines())
|
|
if '' in stderr_set:
|
|
stderr_set.remove('') # Avoid groups represented by an empty string.
|
|
if MACOS_WARNING in stderr_set:
|
|
stderr_set.remove(MACOS_WARNING)
|
|
if self._use_gcloud_storage:
|
|
for to_check in expected_stderr_lines:
|
|
self.assertIn(to_check, stderr)
|
|
else:
|
|
self.assertEqual(stderr_set, expected_stderr_lines)
|
|
else:
|
|
cumulative_stderr_lines = set()
|
|
|
|
@Retry(AssertionError, tries=5, timeout_secs=1)
|
|
def _RunRmCommandAndCheck():
|
|
"""Runs/retries the command updating+checking cumulative output."""
|
|
stderr = self.RunGsUtil(command_and_args,
|
|
return_stderr=True,
|
|
expected_status=None,
|
|
stdin=stdin)
|
|
stderr = self._CleanRmUiOutputBeforeChecking(stderr)
|
|
update_lines = True
|
|
# Retry 404's and 409's due to eventual listing consistency, but don't
|
|
# add the output to the set.
|
|
if (NO_URLS_MATCHED_PREFIX in stderr or
|
|
'409 BucketNotEmpty' in stderr or
|
|
'409 VersionedBucketNotEmpty' in stderr):
|
|
update_lines = False
|
|
|
|
# For recursive deletes of buckets, it is possible that the bucket is
|
|
# deleted before the objects are all present in the listing, in which
|
|
# case we will never see all of the expected "Removing object..."
|
|
# messages. Since this is still a successful outcome, just return
|
|
# successfully.
|
|
if self._use_gcloud_storage:
|
|
bucket_not_found_string = 'not found'
|
|
else:
|
|
bucket_not_found_string = 'bucket does not exist'
|
|
if '-r' in command_and_args and 'bucket does not exist' in stderr:
|
|
for bucket_to_remove in buckets_to_remove:
|
|
matching_bucket = re.match(
|
|
r'.*404\s+%s\s+bucket does not exist' %
|
|
re.escape(bucket_to_remove), stderr)
|
|
if matching_bucket:
|
|
for line in cumulative_stderr_lines:
|
|
if 'Removing %s/...' % bucket_to_remove in line:
|
|
return
|
|
if 'Removing %s/...' % bucket_to_remove in stderr:
|
|
return
|
|
|
|
if update_lines:
|
|
cumulative_stderr_lines.update(
|
|
set([s for s in stderr.splitlines() if s]))
|
|
|
|
# Ensure all of the expected strings are present.
|
|
self.assertEqual(cumulative_stderr_lines, expected_stderr_lines)
|
|
|
|
_RunRmCommandAndCheck()
|
|
|
|
def test_all_versions_current(self):
|
|
"""Test that 'rm -a' for an object with a current version works."""
|
|
bucket_uri = self.CreateVersionedBucket()
|
|
key_uri = self.StorageUriCloneReplaceName(bucket_uri, 'foo')
|
|
self.StorageUriSetContentsFromString(key_uri, 'bar')
|
|
g1 = urigen(key_uri)
|
|
self.StorageUriSetContentsFromString(key_uri, 'baz')
|
|
g2 = urigen(key_uri)
|
|
|
|
def _Check1(stderr_lines):
|
|
stderr = self.RunGsUtil(
|
|
['-m', 'rm', '-a', suri(key_uri)], return_stderr=True)
|
|
stderr_lines.update(set(stderr.splitlines()))
|
|
stderr = '\n'.join(stderr_lines)
|
|
self.assertEqual(stderr.count('Removing %s://' % self.default_provider),
|
|
2)
|
|
self.assertIn('Removing %s#%s...' % (suri(key_uri), g1), stderr)
|
|
self.assertIn('Removing %s#%s...' % (suri(key_uri), g2), stderr)
|
|
|
|
all_stderr_lines = set()
|
|
if self.multiregional_buckets:
|
|
self.AssertNObjectsInBucket(bucket_uri, 2, versioned=True)
|
|
|
|
@Retry(AssertionError, tries=3, timeout_secs=1)
|
|
# Use @Retry as hedge against bucket listing eventual consistency.
|
|
def _CheckWithRetries(stderr_lines):
|
|
_Check1(stderr_lines)
|
|
|
|
_CheckWithRetries(all_stderr_lines)
|
|
else:
|
|
_Check1(all_stderr_lines)
|
|
|
|
self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True)
|
|
|
|
def test_all_versions_no_current(self):
|
|
"""Test that 'rm -a' for an object without a current version works."""
|
|
bucket_uri = self.CreateVersionedBucket()
|
|
key_uri = self.StorageUriCloneReplaceName(bucket_uri, 'foo')
|
|
self.StorageUriSetContentsFromString(key_uri, 'bar')
|
|
g1 = urigen(key_uri)
|
|
self.StorageUriSetContentsFromString(key_uri, 'baz')
|
|
g2 = urigen(key_uri)
|
|
self._RunRemoveCommandAndCheck(
|
|
['-m', 'rm', '-a', suri(key_uri)],
|
|
objects_to_remove=[
|
|
'%s#%s' % (suri(key_uri), g1),
|
|
'%s#%s' % (suri(key_uri), g2)
|
|
])
|
|
self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True)
|
|
|
|
def test_fails_for_missing_obj(self):
|
|
bucket_uri = self.CreateVersionedBucket()
|
|
stderr = self.RunGsUtil(
|
|
['rm', '-a', '%s' % suri(bucket_uri, 'foo')],
|
|
return_stderr=True,
|
|
expected_status=1)
|
|
if self._use_gcloud_storage:
|
|
no_url_matched_target = no_url_matched_target = (
|
|
'The following URLs matched no objects or files:\n%s')
|
|
else:
|
|
no_url_matched_target = NO_URLS_MATCHED_TARGET
|
|
self.assertIn(no_url_matched_target % suri(bucket_uri, 'foo'), stderr)
|
|
|
|
def test_remove_recursive_prefix(self):
|
|
bucket_uri = self.CreateBucket()
|
|
obj_uri = self.CreateObject(bucket_uri=bucket_uri,
|
|
object_name='a/b/c',
|
|
contents=b'foo')
|
|
if self.multiregional_buckets:
|
|
self.AssertNObjectsInBucket(bucket_uri, 1)
|
|
|
|
stderr = self.RunGsUtil(['rm', '-r', suri(bucket_uri, 'a')],
|
|
return_stderr=True)
|
|
self.assertIn('Removing %s' % suri(obj_uri), stderr)
|
|
|
|
def test_remove_all_versions_recursive_on_bucket(self):
|
|
"""Test that 'rm -r' works on bucket."""
|
|
bucket_uri = self.CreateVersionedBucket()
|
|
k1_uri = self.StorageUriCloneReplaceName(bucket_uri, 'foo')
|
|
k2_uri = self.StorageUriCloneReplaceName(bucket_uri, 'foo2')
|
|
self.StorageUriSetContentsFromString(k1_uri, 'bar')
|
|
self.StorageUriSetContentsFromString(k2_uri, 'bar2')
|
|
k1g1 = urigen(k1_uri)
|
|
k2g1 = urigen(k2_uri)
|
|
self.StorageUriSetContentsFromString(k1_uri, 'baz')
|
|
self.StorageUriSetContentsFromString(k2_uri, 'baz2')
|
|
k1g2 = urigen(k1_uri)
|
|
k2g2 = urigen(k2_uri)
|
|
|
|
if self.multiregional_buckets:
|
|
self.AssertNObjectsInBucket(bucket_uri, 4, versioned=True)
|
|
|
|
self._RunRemoveCommandAndCheck(['rm', '-r', suri(bucket_uri)],
|
|
objects_to_remove=[
|
|
'%s#%s' % (suri(k1_uri), k1g1),
|
|
'%s#%s' % (suri(k1_uri), k1g2),
|
|
'%s#%s' % (suri(k2_uri), k2g1),
|
|
'%s#%s' % (suri(k2_uri), k2g2)
|
|
],
|
|
buckets_to_remove=[suri(bucket_uri)])
|
|
|
|
# Use @Retry as hedge against bucket listing eventual consistency.
|
|
@Retry(AssertionError, tries=3, timeout_secs=1)
|
|
def _Check():
|
|
# Bucket should no longer exist.
|
|
stderr = self.RunGsUtil(['ls', '-a', suri(bucket_uri)],
|
|
return_stderr=True,
|
|
expected_status=1)
|
|
if self._use_gcloud_storage:
|
|
if self._use_gcloud_storage:
|
|
# GCS and S3 responses.
|
|
self.assertTrue('not found: 404' in stderr or
|
|
'NoSuchBucket' in stderr)
|
|
else:
|
|
self.assertIn('bucket does not exist', stderr)
|
|
|
|
_Check()
|
|
|
|
def test_remove_all_versions_recursive_on_subdir(self):
|
|
"""Test that 'rm -r' works on subdir."""
|
|
bucket_uri = self.CreateVersionedBucket()
|
|
k1_uri = self.StorageUriCloneReplaceName(bucket_uri, 'dir/foo')
|
|
k2_uri = self.StorageUriCloneReplaceName(bucket_uri, 'dir/foo2')
|
|
self.StorageUriSetContentsFromString(k1_uri, 'bar')
|
|
self.StorageUriSetContentsFromString(k2_uri, 'bar2')
|
|
k1g1 = urigen(k1_uri)
|
|
k2g1 = urigen(k2_uri)
|
|
self.StorageUriSetContentsFromString(k1_uri, 'baz')
|
|
self.StorageUriSetContentsFromString(k2_uri, 'baz2')
|
|
k1g2 = urigen(k1_uri)
|
|
k2g2 = urigen(k2_uri)
|
|
|
|
if self.multiregional_buckets:
|
|
self.AssertNObjectsInBucket(bucket_uri, 4, versioned=True)
|
|
|
|
self._RunRemoveCommandAndCheck(
|
|
['rm', '-r', '%s' % suri(bucket_uri, 'dir')],
|
|
objects_to_remove=[
|
|
'%s#%s' % (suri(k1_uri), k1g1),
|
|
'%s#%s' % (suri(k1_uri), k1g2),
|
|
'%s#%s' % (suri(k2_uri), k2g1),
|
|
'%s#%s' % (suri(k2_uri), k2g2)
|
|
])
|
|
self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True)
|
|
|
|
def test_rm_seek_ahead(self):
|
|
object_uri = self.CreateObject(contents=b'foo')
|
|
with SetBotoConfigForTest([('GSUtil', 'task_estimation_threshold', '1'),
|
|
('GSUtil', 'task_estimation_force', 'True')]):
|
|
stderr = self.RunGsUtil(['-m', 'rm', suri(object_uri)],
|
|
return_stderr=True)
|
|
self.assertIn('Estimated work for this command: objects: 1\n', stderr)
|
|
|
|
def test_rm_seek_ahead_stdin_args(self):
|
|
object_uri = self.CreateObject(contents=b'foo')
|
|
with SetBotoConfigForTest([('GSUtil', 'task_estimation_threshold', '1'),
|
|
('GSUtil', 'task_estimation_force', 'True')]):
|
|
stderr = self.RunGsUtil(['-m', 'rm', '-I'],
|
|
stdin=suri(object_uri),
|
|
return_stderr=True)
|
|
self.assertNotIn('Estimated work', stderr)
|
|
|
|
def test_missing_first_force(self):
|
|
bucket_uri = self.CreateBucket()
|
|
object_uri = self.CreateObject(bucket_uri=bucket_uri,
|
|
object_name='present',
|
|
contents=b'foo')
|
|
if self.multiregional_buckets:
|
|
self.AssertNObjectsInBucket(bucket_uri, 1)
|
|
if not self._use_gcloud_storage:
|
|
# Gcloud storage continues on missing objects by default.
|
|
# So we will skip running this for gcloud storage.
|
|
self.RunGsUtil(
|
|
['rm', '%s' % suri(bucket_uri, 'missing'),
|
|
suri(object_uri)],
|
|
expected_status=1)
|
|
stderr = self.RunGsUtil(
|
|
['rm', '-f',
|
|
'%s' % suri(bucket_uri, 'missing'),
|
|
suri(object_uri)],
|
|
return_stderr=True,
|
|
expected_status=1)
|
|
self.assertEqual(stderr.count('Removing %s://' % self.default_provider), 1)
|
|
self.RunGsUtil(['stat', suri(object_uri)], expected_status=1)
|
|
|
|
def test_some_missing(self):
|
|
"""Test that 'rm -a' fails when some but not all uris don't exist."""
|
|
bucket_uri = self.CreateVersionedBucket()
|
|
key_uri = self.StorageUriCloneReplaceName(bucket_uri, 'foo')
|
|
self.StorageUriSetContentsFromString(key_uri, 'bar')
|
|
if self.multiregional_buckets:
|
|
self.AssertNObjectsInBucket(bucket_uri, 1, versioned=True)
|
|
stderr = self.RunGsUtil(
|
|
['rm', '-a',
|
|
suri(key_uri),
|
|
'%s' % suri(bucket_uri, 'missing')],
|
|
return_stderr=True,
|
|
expected_status=1)
|
|
self.assertEqual(stderr.count('Removing %s://' % self.default_provider), 1)
|
|
if self._use_gcloud_storage:
|
|
self.assertIn(
|
|
'The following URLs matched no objects or files:\n%s' %
|
|
suri(bucket_uri, 'missing'), stderr)
|
|
else:
|
|
self.assertIn(NO_URLS_MATCHED_TARGET % suri(bucket_uri, 'missing'),
|
|
stderr)
|
|
|
|
def test_some_missing_force(self):
|
|
"""Test that 'rm -af' succeeds despite hidden first uri."""
|
|
bucket_uri = self.CreateVersionedBucket()
|
|
key_uri = self.StorageUriCloneReplaceName(bucket_uri, 'foo')
|
|
self.StorageUriSetContentsFromString(key_uri, 'bar')
|
|
if self.multiregional_buckets:
|
|
self.AssertNObjectsInBucket(bucket_uri, 1, versioned=True)
|
|
stderr = self.RunGsUtil(
|
|
['rm', '-af',
|
|
suri(key_uri),
|
|
'%s' % suri(bucket_uri, 'missing')],
|
|
return_stderr=True,
|
|
expected_status=1)
|
|
self.assertEqual(stderr.count('Removing %s://' % self.default_provider), 1)
|
|
self.AssertNObjectsInBucket(bucket_uri, 0)
|
|
|
|
def test_folder_objects_deleted(self):
|
|
"""Test for 'rm -r' of a folder with a dir_$folder$ marker."""
|
|
bucket_uri = self.CreateVersionedBucket()
|
|
key_uri = self.StorageUriCloneReplaceName(bucket_uri, 'abc/o1')
|
|
self.StorageUriSetContentsFromString(key_uri, 'foobar')
|
|
folder_uri = self.StorageUriCloneReplaceName(bucket_uri, 'abc_$folder$')
|
|
self.StorageUriSetContentsFromString(folder_uri, '')
|
|
|
|
def _RemoveAndCheck():
|
|
self.RunGsUtil(['rm', '-r', '%s' % suri(bucket_uri, 'abc')],
|
|
expected_status=None)
|
|
self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True)
|
|
|
|
if self.multiregional_buckets:
|
|
self.AssertNObjectsInBucket(bucket_uri, 2, versioned=True)
|
|
# This could fail due to eventual listing consistency, so use retry and
|
|
# expected_status=None to guard against No URLs matched exceptions.
|
|
@Retry(AssertionError, tries=3, timeout_secs=1)
|
|
def _RemoveAndCheckWrapper():
|
|
_RemoveAndCheck()
|
|
|
|
_RemoveAndCheckWrapper()
|
|
else:
|
|
_RemoveAndCheck()
|
|
# Bucket should not be deleted (Should not get ServiceException).
|
|
bucket_uri.get_location(validate=False)
|
|
|
|
def test_folder_objects_deleted_with_wildcard(self):
|
|
"""Test for 'rm -r' of a folder with a dir_$folder$ marker."""
|
|
bucket_uri = self.CreateVersionedBucket()
|
|
key_uri = self.StorageUriCloneReplaceName(bucket_uri, 'abc/o1')
|
|
self.StorageUriSetContentsFromString(key_uri, 'foobar')
|
|
folder_uri = self.StorageUriCloneReplaceName(bucket_uri, 'abc_$folder$')
|
|
self.StorageUriSetContentsFromString(folder_uri, '')
|
|
|
|
if self.multiregional_buckets:
|
|
self.AssertNObjectsInBucket(bucket_uri, 2, versioned=True)
|
|
self._RunRemoveCommandAndCheck(
|
|
['rm', '-r', '%s' % suri(bucket_uri, '*')],
|
|
objects_to_remove=[
|
|
'%s#%s' % (suri(key_uri), urigen(key_uri)),
|
|
'%s#%s' % (suri(folder_uri), urigen(folder_uri))
|
|
])
|
|
self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True)
|
|
# Bucket should not be deleted (Should not get ServiceException).
|
|
bucket_uri.get_location(validate=False)
|
|
|
|
def test_folder_objects_deleted_with_double_wildcard(self):
|
|
"""Test for 'rm -r' of a folder with a dir_$folder$ marker."""
|
|
bucket_uri = self.CreateVersionedBucket()
|
|
key_uri = self.StorageUriCloneReplaceName(bucket_uri, 'abc/o1')
|
|
self.StorageUriSetContentsFromString(key_uri, 'foobar')
|
|
folder_uri = self.StorageUriCloneReplaceName(bucket_uri, 'abc_$folder$')
|
|
self.StorageUriSetContentsFromString(folder_uri, '')
|
|
|
|
if self.multiregional_buckets:
|
|
self.AssertNObjectsInBucket(bucket_uri, 2, versioned=True)
|
|
self._RunRemoveCommandAndCheck(
|
|
['rm', '-r', '%s' % suri(bucket_uri, '**')],
|
|
objects_to_remove=[
|
|
'%s#%s' % (suri(key_uri), urigen(key_uri)),
|
|
'%s#%s' % (suri(folder_uri), urigen(folder_uri))
|
|
])
|
|
self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True)
|
|
# Bucket should not be deleted (Should not get ServiceException).
|
|
bucket_uri.get_location(validate=False)
|
|
|
|
@SkipForS3('Listing/removing S3 DeleteMarkers is not supported')
|
|
def test_recursive_bucket_rm(self):
|
|
"""Test for 'rm -r' of a bucket."""
|
|
bucket_uri = self.CreateBucket()
|
|
object_uri = self.CreateObject(bucket_uri, contents=b'foo')
|
|
if self.multiregional_buckets:
|
|
self.AssertNObjectsInBucket(bucket_uri, 1)
|
|
self._RunRemoveCommandAndCheck(
|
|
['rm', '-r', suri(bucket_uri)],
|
|
objects_to_remove=['%s#%s' % (suri(object_uri), urigen(object_uri))],
|
|
buckets_to_remove=[suri(bucket_uri)])
|
|
|
|
# Use @Retry as hedge against bucket listing eventual consistency.
|
|
@Retry(AssertionError, tries=3, timeout_secs=1)
|
|
def _Check1():
|
|
# Bucket should be deleted.
|
|
stderr = self.RunGsUtil(['ls', '-Lb', suri(bucket_uri)],
|
|
return_stderr=True,
|
|
expected_status=1,
|
|
force_gsutil=True)
|
|
self.assertIn('bucket does not exist', stderr)
|
|
|
|
_Check1()
|
|
|
|
# Now try same thing, but for a versioned bucket with multiple versions of
|
|
# an object present.
|
|
bucket_uri = self.CreateVersionedBucket()
|
|
self.CreateObject(bucket_uri, 'obj', contents=b'z')
|
|
self.CreateObject(bucket_uri, 'obj', contents=b'z')
|
|
final_uri = self.CreateObject(bucket_uri, 'obj', contents=b'z')
|
|
if self.multiregional_buckets:
|
|
self.AssertNObjectsInBucket(bucket_uri, 3, versioned=True)
|
|
self._RunRemoveCommandAndCheck(['rm', suri(bucket_uri, '**')],
|
|
objects_to_remove=['%s' % final_uri])
|
|
|
|
stderr = self.RunGsUtil(['rb', suri(bucket_uri)],
|
|
return_stderr=True,
|
|
expected_status=1,
|
|
force_gsutil=True)
|
|
self.assertIn('not empty', stderr)
|
|
|
|
# Now try with rm -r.
|
|
@Retry(AssertionError, tries=3, timeout_secs=1)
|
|
def _Check2():
|
|
self.RunGsUtil(['rm', '-r', suri(bucket_uri)])
|
|
# Bucket should be deleted.
|
|
stderr = self.RunGsUtil(['ls', '-Lb', suri(bucket_uri)],
|
|
return_stderr=True,
|
|
expected_status=1,
|
|
force_gsutil=True)
|
|
self.assertIn('bucket does not exist', stderr)
|
|
|
|
_Check2()
|
|
|
|
def test_recursive_bucket_rm_with_wildcarding(self):
|
|
"""Tests removing all objects and buckets matching a bucket wildcard."""
|
|
buri_base = 'gsutil-test-%s' % self.GetTestMethodName()
|
|
buri_base = buri_base[:MAX_BUCKET_LENGTH - 20]
|
|
buri_base = '%s-%s' % (buri_base, self.MakeRandomTestString())
|
|
buri_base = 'aaa-' + buri_base
|
|
buri_base = util.MakeBucketNameValid(buri_base)
|
|
buri1 = self.CreateBucket(bucket_name='%s-tbuck1' % buri_base)
|
|
buri2 = self.CreateBucket(bucket_name='%s-tbuck2' % buri_base)
|
|
buri3 = self.CreateBucket(bucket_name='%s-tb3' % buri_base)
|
|
ouri1 = self.CreateObject(bucket_uri=buri1, object_name='o1', contents=b'z')
|
|
ouri2 = self.CreateObject(bucket_uri=buri2, object_name='o2', contents=b'z')
|
|
self.CreateObject(bucket_uri=buri3, object_name='o3', contents=b'z')
|
|
|
|
if self.multiregional_buckets:
|
|
self.AssertNObjectsInBucket(buri1, 1)
|
|
self.AssertNObjectsInBucket(buri2, 1)
|
|
self.AssertNObjectsInBucket(buri3, 1)
|
|
|
|
self._RunRemoveCommandAndCheck(
|
|
['rm', '-r',
|
|
'%s://%s-tbu*' % (self.default_provider, buri_base)],
|
|
objects_to_remove=[
|
|
'%s#%s' % (suri(ouri1), urigen(ouri1)),
|
|
'%s#%s' % (suri(ouri2), urigen(ouri2))
|
|
],
|
|
buckets_to_remove=[suri(buri1), suri(buri2)])
|
|
|
|
self.AssertNObjectsInBucket(buri3, 1)
|
|
|
|
def test_rm_quiet(self):
|
|
"""Test that 'rm -q' outputs no progress indications."""
|
|
bucket_uri = self.CreateBucket()
|
|
key_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo')
|
|
if self.multiregional_buckets:
|
|
self.AssertNObjectsInBucket(bucket_uri, 1)
|
|
self._RunRemoveCommandAndCheck(['-q', 'rm', suri(key_uri)], [])
|
|
self.AssertNObjectsInBucket(bucket_uri, 0)
|
|
|
|
@SkipForS3('The boto lib used for S3 does not handle objects '
|
|
'starting with slashes if we use V4 signature')
|
|
def test_rm_object_with_prefix_slash(self):
|
|
"""Tests removing a bucket that has an object starting with slash.
|
|
|
|
The boto lib used for S3 does not handle objects starting with slashes
|
|
if we use V4 signature. Hence we are testing objects with prefix
|
|
slashes separately.
|
|
"""
|
|
bucket_uri = self.CreateVersionedBucket()
|
|
ouri1 = self.CreateObject(bucket_uri=bucket_uri,
|
|
object_name='/dirwithslash/foo',
|
|
contents=b'z')
|
|
if self.multiregional_buckets:
|
|
self.AssertNObjectsInBucket(bucket_uri, 1, versioned=True)
|
|
|
|
self._RunRemoveCommandAndCheck(
|
|
['rm', '-r', suri(bucket_uri)],
|
|
objects_to_remove=['%s#%s' % (suri(ouri1), urigen(ouri1))],
|
|
buckets_to_remove=[suri(bucket_uri)])
|
|
|
|
def test_rm_object_with_slashes(self):
|
|
"""Tests removing a bucket that has objects with slashes."""
|
|
bucket_uri = self.CreateVersionedBucket()
|
|
ouri1 = self.CreateObject(bucket_uri=bucket_uri,
|
|
object_name='h/e/l//lo',
|
|
contents=b'z')
|
|
ouri2 = self.CreateObject(bucket_uri=bucket_uri,
|
|
object_name='dirnoslash/foo',
|
|
contents=b'z')
|
|
ouri3 = self.CreateObject(bucket_uri=bucket_uri,
|
|
object_name='dirnoslash/foo2',
|
|
contents=b'z')
|
|
if self.multiregional_buckets:
|
|
self.AssertNObjectsInBucket(bucket_uri, 3, versioned=True)
|
|
|
|
self._RunRemoveCommandAndCheck(['rm', '-r', suri(bucket_uri)],
|
|
objects_to_remove=[
|
|
'%s#%s' % (suri(ouri1), urigen(ouri1)),
|
|
'%s#%s' % (suri(ouri2), urigen(ouri2)),
|
|
'%s#%s' % (suri(ouri3), urigen(ouri3))
|
|
],
|
|
buckets_to_remove=[suri(bucket_uri)])
|
|
|
|
@SkipForS3('The boto lib used for S3 does not handle objects '
|
|
'starting with slashes if we use V4 signature')
|
|
def test_slasher_horror_film(self):
|
|
"""Tests removing a bucket with objects that are filled with slashes."""
|
|
bucket_uri = self.CreateVersionedBucket()
|
|
ouri1 = self.CreateObject(bucket_uri=bucket_uri,
|
|
object_name='h/e/l//lo',
|
|
contents=b'Halloween')
|
|
ouri2 = self.CreateObject(bucket_uri=bucket_uri,
|
|
object_name='/h/e/l/l/o',
|
|
contents=b'A Nightmare on Elm Street')
|
|
ouri3 = self.CreateObject(bucket_uri=bucket_uri,
|
|
object_name='//h//e/l//l/o',
|
|
contents=b'Friday the 13th')
|
|
ouri4 = self.CreateObject(bucket_uri=bucket_uri,
|
|
object_name='//h//e//l//l//o',
|
|
contents=b'I Know What You Did Last Summer')
|
|
ouri5 = self.CreateObject(bucket_uri=bucket_uri,
|
|
object_name='/',
|
|
contents=b'Scream')
|
|
ouri6 = self.CreateObject(bucket_uri=bucket_uri,
|
|
object_name='//',
|
|
contents=b'Child\'s Play')
|
|
ouri7 = self.CreateObject(bucket_uri=bucket_uri,
|
|
object_name='///',
|
|
contents=b'The Prowler')
|
|
ouri8 = self.CreateObject(bucket_uri=bucket_uri,
|
|
object_name='////',
|
|
contents=b'Black Christmas')
|
|
ouri9 = self.CreateObject(
|
|
bucket_uri=bucket_uri,
|
|
object_name='everything/is/better/with/slashes///////',
|
|
contents=b'Maniac')
|
|
|
|
if self.multiregional_buckets:
|
|
self.AssertNObjectsInBucket(bucket_uri, 9, versioned=True)
|
|
|
|
# We add a slash to URIs with a trailing slash,
|
|
# because ObjectToURI (suri) removes one trailing slash.
|
|
objects_to_remove = [
|
|
'%s#%s' % (suri(ouri1), urigen(ouri1)),
|
|
'%s#%s' % (suri(ouri2), urigen(ouri2)),
|
|
'%s#%s' % (suri(ouri3), urigen(ouri3)),
|
|
'%s#%s' % (suri(ouri4), urigen(ouri4)),
|
|
'%s#%s' % (suri(ouri5) + '/', urigen(ouri5)),
|
|
'%s#%s' % (suri(ouri6) + '/', urigen(ouri6)),
|
|
'%s#%s' % (suri(ouri7) + '/', urigen(ouri7)),
|
|
'%s#%s' % (suri(ouri8) + '/', urigen(ouri8)),
|
|
'%s#%s' % (suri(ouri9) + '/', urigen(ouri9))
|
|
]
|
|
|
|
self._RunRemoveCommandAndCheck(
|
|
['-m', 'rm', '-r', suri(bucket_uri)],
|
|
objects_to_remove=objects_to_remove,
|
|
buckets_to_remove=[suri(bucket_uri)])
|
|
|
|
@SkipForS3('GCS versioning headers not supported by S3')
|
|
def test_rm_failing_precondition(self):
|
|
"""Test for '-h x-goog-if-generation-match:value rm' of an object."""
|
|
bucket_uri = self.CreateBucket()
|
|
object_uri = self.CreateObject(bucket_uri, contents=b'foo')
|
|
stderr = self.RunGsUtil(
|
|
['-h', 'x-goog-if-generation-match:12345', 'rm',
|
|
suri(object_uri)],
|
|
return_stderr=True,
|
|
expected_status=1)
|
|
if self._use_gcloud_storage:
|
|
self.assertRegex(stderr, r'pre-conditions you specified did not hold')
|
|
else:
|
|
self.assertRegex(stderr, r'PreconditionException: 412')
|
|
|
|
def test_stdin_args(self):
|
|
"""Tests rm with the -I option."""
|
|
buri1 = self.CreateVersionedBucket()
|
|
ouri1 = self.CreateObject(bucket_uri=buri1,
|
|
object_name='foo',
|
|
contents=b'foocontents')
|
|
self.CreateObject(bucket_uri=buri1,
|
|
object_name='bar',
|
|
contents=b'barcontents')
|
|
ouri3 = self.CreateObject(bucket_uri=buri1,
|
|
object_name='baz',
|
|
contents=b'bazcontents')
|
|
buri2 = self.CreateVersionedBucket()
|
|
ouri4 = self.CreateObject(bucket_uri=buri2,
|
|
object_name='moo',
|
|
contents=b'moocontents')
|
|
if self.multiregional_buckets:
|
|
self.AssertNObjectsInBucket(buri1, 3, versioned=True)
|
|
self.AssertNObjectsInBucket(buri2, 1, versioned=True)
|
|
|
|
objects_to_remove = [
|
|
'%s#%s' % (suri(ouri1), urigen(ouri1)),
|
|
'%s#%s' % (suri(ouri3), urigen(ouri3)),
|
|
'%s#%s' % (suri(ouri4), urigen(ouri4))
|
|
]
|
|
stdin = '\n'.join(objects_to_remove)
|
|
self._RunRemoveCommandAndCheck(['rm', '-I'],
|
|
objects_to_remove=objects_to_remove,
|
|
stdin=stdin)
|
|
self.AssertNObjectsInBucket(buri1, 1, versioned=True)
|
|
self.AssertNObjectsInBucket(buri2, 0, versioned=True)
|
|
|
|
def test_rm_nonexistent_bucket_recursive(self):
|
|
stderr = self.RunGsUtil([
|
|
'rm', '-rf',
|
|
'%s://%s' % (self.default_provider, self.nonexistent_bucket_name)
|
|
],
|
|
return_stderr=True,
|
|
expected_status=1)
|
|
if self._use_gcloud_storage:
|
|
# GCS and S3 responses.
|
|
self.assertTrue('not found: 404' in stderr or 'NoSuchBucket' in stderr)
|
|
else:
|
|
self.assertIn('Encountered non-existent bucket', stderr)
|
|
|
|
def test_rm_multiple_nonexistent_objects(self):
|
|
bucket_uri = self.CreateBucket()
|
|
nonexistent_object1 = suri(bucket_uri, 'nonexistent1')
|
|
nonexistent_object2 = suri(bucket_uri, 'nonexistent2')
|
|
stderr = self.RunGsUtil(
|
|
['rm', '-rf', nonexistent_object1, nonexistent_object2],
|
|
return_stderr=True,
|
|
expected_status=1)
|
|
if self._use_gcloud_storage:
|
|
self.assertIn(
|
|
'The following URLs matched no objects or files:\n{}\n{}'.format(
|
|
nonexistent_object1, nonexistent_object2), stderr)
|
|
else:
|
|
self.assertIn('2 files/objects could not be removed.', stderr)
|
|
|
|
|
|
class TestRmUnitTestsWithShim(testcase.ShimUnitTestBase):
|
|
"""Unit tests for gsutil rm with shim."""
|
|
|
|
def test_shim_translates_flags(self):
|
|
bucket_uri = self.CreateBucket()
|
|
with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'),
|
|
('GSUtil', 'hidden_shim_mode', 'dry_run')]):
|
|
with SetEnvironmentForTest({
|
|
'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True',
|
|
'CLOUDSDK_ROOT_DIR': 'fake_dir',
|
|
}):
|
|
mock_log_handler = self.RunCommand(
|
|
'rm',
|
|
['-r', '-R', '-a', '-f', suri(bucket_uri)],
|
|
return_log_handler=True)
|
|
info_lines = '\n'.join(mock_log_handler.messages['info'])
|
|
self.assertIn(
|
|
'Gcloud Storage Command: {} storage rm'
|
|
' -r -r -a --continue-on-error {}'.format(
|
|
shim_util._get_gcloud_binary_path('fake_dir'),
|
|
suri(bucket_uri)), info_lines)
|
|
|
|
@mock.patch.object(sys, 'stdin')
|
|
def test_shim_translates_stdin_flag(self, mock_stdin):
|
|
bucket_uri = self.CreateBucket()
|
|
object_uri = self.CreateObject(bucket_uri, 'foo', 'abcd')
|
|
mock_stdin.__iter__.return_value = [suri(object_uri)]
|
|
with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'),
|
|
('GSUtil', 'hidden_shim_mode', 'dry_run')]):
|
|
with SetEnvironmentForTest({
|
|
'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True',
|
|
'CLOUDSDK_ROOT_DIR': 'fake_dir',
|
|
}):
|
|
|
|
mock_log_handler = self.RunCommand('rm', ['-I'],
|
|
return_log_handler=True)
|
|
info_lines = '\n'.join(mock_log_handler.messages['info'])
|
|
self.assertIn(
|
|
'Gcloud Storage Command: {} storage rm'
|
|
' -I'.format(shim_util._get_gcloud_binary_path('fake_dir')),
|
|
info_lines)
|